[med-svn] [bcl2fastq2] 01/02: New upstream version 2.19.1.403
Andreas Tille
tille at debian.org
Tue Jun 20 14:08:15 UTC 2017
This is an automated email from the git hooks/post-receive script.
tille pushed a commit to branch master
in repository bcl2fastq2.
commit 14a87995b007023b1beed1899d40572bae7a3250
Author: Andreas Tille <tille at debian.org>
Date: Tue Jun 20 16:00:50 2017 +0200
New upstream version 2.19.1.403
---
CODING | 11 +
COPYRIGHT | 38 +
Illumina_EULA.pdf | Bin 0 -> 102191 bytes
build0.sh | 9 +
build13.sh | 2 +
build7.sh | 2 +
data/InterOp/parseIndexMetrics.py | 101 ++
data/generator/mkdata.sh | 192 +++
redist/zlib-ng-1.9.9.tar.gz | Bin 0 -> 792481 bytes
src/CMakeLists.txt | 208 ++++
src/CODING | 6 +
src/Changes | 380 ++++++
src/INSTALL | 113 ++
src/cmake/bcl2fastq_redist_macros.cmake | 106 ++
src/cmake/boost.cmake | 143 +++
src/cmake/bootstrap/common.sh | 51 +
src/cmake/bootstrap/installBoost.sh | 60 +
src/cmake/bootstrap/installCmake.sh | 98 ++
src/cmake/cppunit.cmake | 101 ++
src/cmake/cxxConfigure.cmake | 181 +++
src/cmake/cxxExecutable.cmake | 48 +
src/cmake/cxxLibrary.cmake | 73 ++
src/cmake/globals.cmake | 64 +
src/cmake/macros.cmake | 84 ++
src/cmake/modules/FindMarkdown.cmake | 58 +
src/cmake/modules/UseMarkdown.cmake | 118 ++
src/cmake/postInstall/CMakeLists.txt | 20 +
src/cmake/preInstall/CMakeLists.txt | 23 +
.../checkTargetPathsWritable/CMakeLists.txt | 31 +
.../checkTargetPathWritable.cmake | 37 +
.../preInstall/copyrightAndChanges/CMakeLists.txt | 24 +
src/cmake/zlib-ng.cmake | 30 +
src/configure | 431 +++++++
src/css/CMakeLists.txt | 28 +
src/css/Report.css.xml | 50 +
src/cxx/CMakeLists.txt | 76 ++
src/cxx/CODING | 10 +
src/cxx/Doxyfile.in | 276 +++++
src/cxx/bcl2fastq.sln | 32 +
src/cxx/bcl2fastq.vcxproj | 326 +++++
src/cxx/bcl2fastq.vcxproj.filters | 489 ++++++++
src/cxx/bcl2fastq.vcxproj.vspscc | 10 +
src/cxx/bin/CMakeLists.txt | 38 +
src/cxx/bin/bcl2fastq.cpp | 155 +++
src/cxx/include/common/CsvGrammar.hh | 124 ++
src/cxx/include/common/CsvGrammar.hpp | 108 ++
src/cxx/include/common/Debug.hh | 69 ++
src/cxx/include/common/DirectoryValidator.hh | 84 ++
src/cxx/include/common/Endianness.hh | 52 +
src/cxx/include/common/Exceptions.hh | 222 ++++
src/cxx/include/common/FastIo.hh | 58 +
src/cxx/include/common/FastIo.hpp | 62 +
src/cxx/include/common/FileSystem.hh | 45 +
src/cxx/include/common/InstallationPath.hh | 42 +
src/cxx/include/common/Logger.hh | 153 +++
src/cxx/include/common/Options.hh | 141 +++
src/cxx/include/common/Program.hh | 50 +
src/cxx/include/common/Program.hpp | 136 +++
src/cxx/include/common/ProgramInfo.hh | 88 ++
src/cxx/include/common/SampleMetadata.hh | 69 ++
src/cxx/include/common/StaticMemPool.hh | 91 ++
src/cxx/include/common/StaticMemPool.hpp | 118 ++
src/cxx/include/common/SystemCompatibility.hh | 54 +
src/cxx/include/common/SystemCompatibility.hpp | 60 +
src/cxx/include/common/Threads.hh | 199 ++++
src/cxx/include/common/Threads.hpp | 271 +++++
src/cxx/include/common/Timer.hh | 53 +
src/cxx/include/common/Types.hh | 143 +++
src/cxx/include/config/Bcl2FastqOptions.hh | 330 ++++++
src/cxx/include/config/SampleSheetCsv.hh | 227 ++++
src/cxx/include/conversion/AdapterLocator.hh | 121 ++
src/cxx/include/conversion/AdapterLocator.hpp | 394 +++++++
src/cxx/include/conversion/BclBaseConversion.hh | 168 +++
src/cxx/include/conversion/BclLoader.hh | 464 ++++++++
src/cxx/include/conversion/BclReader.hh | 395 +++++++
src/cxx/include/conversion/Converter.hh | 405 +++++++
src/cxx/include/conversion/Demultiplexer.hh | 242 ++++
src/cxx/include/conversion/Demultiplexer.hpp | 80 ++
src/cxx/include/conversion/FastqBuffer.hh | 54 +
src/cxx/include/conversion/FastqCreator.hh | 408 +++++++
src/cxx/include/conversion/FastqCreator.hpp | 808 +++++++++++++
src/cxx/include/conversion/FastqIterator.hh | 197 ++++
src/cxx/include/conversion/FastqIterator.hpp | 75 ++
src/cxx/include/conversion/FastqWriter.hh | 189 +++
src/cxx/include/conversion/SampleIndex.hh | 243 ++++
src/cxx/include/conversion/Stage.hh | 94 ++
src/cxx/include/conversion/Task.hh | 80 ++
src/cxx/include/conversion/TaskQueue.hh | 56 +
src/cxx/include/conversion/TaskQueue.hpp | 89 ++
src/cxx/include/conversion/ThreadPool.hh | 80 ++
src/cxx/include/conversion/ThreadSafeQueue.hh | 101 ++
src/cxx/include/conversion/ThreadSafeQueue.hpp | 216 ++++
src/cxx/include/data/AggregatedBclFileReader.hh | 56 +
src/cxx/include/data/BclBuffer.hh | 125 ++
src/cxx/include/data/BclFile.hh | 113 ++
src/cxx/include/data/BclFileReader.hh | 88 ++
src/cxx/include/data/CbclFile.hh | 192 +++
src/cxx/include/data/ControlFile.hh | 109 ++
src/cxx/include/data/CycleBCIFile.hh | 155 +++
src/cxx/include/data/FastqFile.hh | 109 ++
src/cxx/include/data/FileReader.hh | 191 +++
src/cxx/include/data/FileReader.hpp | 61 +
src/cxx/include/data/FilterFile.hh | 126 ++
src/cxx/include/data/Instrument.hh | 75 ++
src/cxx/include/data/InteropFile.hh | 135 +++
src/cxx/include/data/PositionsFile.hh | 136 +++
src/cxx/include/data/RawBclBuffer.hh | 208 ++++
src/cxx/include/data/TileBclFileReader.hh | 49 +
src/cxx/include/io/FileBufWithReopen.hh | 170 +++
src/cxx/include/io/FileBufWithReopen.hpp | 258 ++++
src/cxx/include/io/GzipCompressor.hh | 116 ++
src/cxx/include/io/GzipDecompressor.hh | 191 +++
src/cxx/include/io/GzipDecompressor.hpp | 129 ++
src/cxx/include/io/SyncFile.hh | 265 +++++
src/cxx/include/io/Utility.hh | 45 +
src/cxx/include/io/Xml.hh | 58 +
src/cxx/include/io/Xml.hpp | 47 +
src/cxx/include/layout/BCIndex.hh | 111 ++
src/cxx/include/layout/Barcode.hh | 209 ++++
src/cxx/include/layout/Barcode.hpp | 129 ++
src/cxx/include/layout/BarcodeCollisionDetector.hh | 73 ++
src/cxx/include/layout/BarcodeTranslationTable.hh | 181 +++
src/cxx/include/layout/ConfigXml.hh | 111 ++
src/cxx/include/layout/CycleInfo.hh | 78 ++
src/cxx/include/layout/FileExistenceVerifier.hh | 85 ++
src/cxx/include/layout/FlowcellInfo.hh | 130 ++
src/cxx/include/layout/LaneInfo.hh | 185 +++
src/cxx/include/layout/Layout.hh | 417 +++++++
src/cxx/include/layout/ReadInfo.hh | 207 ++++
src/cxx/include/layout/ReadMetadata.hh | 65 +
src/cxx/include/layout/RunInfoXml.hh | 120 ++
src/cxx/include/layout/SampleInfo.hh | 159 +++
src/cxx/include/layout/TileInfo.hh | 134 +++
src/cxx/include/layout/UseBasesMask.hh | 90 ++
src/cxx/include/stats/BarcodeHits.hh | 155 +++
src/cxx/include/stats/BarcodeStats.hh | 106 ++
src/cxx/include/stats/ConversionStatsXml.hh | 84 ++
src/cxx/include/stats/DemultiplexingStatsXml.hh | 58 +
src/cxx/include/stats/DemuxReportGenerator.hh | 75 ++
src/cxx/include/stats/Json.hh | 119 ++
src/cxx/include/stats/TileStats.hpp | 237 ++++
src/cxx/lib/CMakeLists.txt | 48 +
src/cxx/lib/common/CMakeLists.txt | 23 +
src/cxx/lib/common/CsvGrammar.cpp | 89 ++
src/cxx/lib/common/DirectoryValidator.cpp | 94 ++
src/cxx/lib/common/Exceptions.cpp | 192 +++
src/cxx/lib/common/FileSystem.cpp | 64 +
src/cxx/lib/common/InstallationPath.cpp | 69 ++
src/cxx/lib/common/Logger.cpp | 183 +++
src/cxx/lib/common/Options.cpp | 223 ++++
src/cxx/lib/common/ProgramInfo.cpp | 87 ++
src/cxx/lib/common/SystemCompatibility.cpp | 63 +
src/cxx/lib/common/config.h.in | 178 +++
src/cxx/lib/common/cppunit/CMakeLists.txt | 23 +
src/cxx/lib/common/cppunit/RegistryNames.txt | 1 +
src/cxx/lib/common/cppunit/testExceptions.cpp | 40 +
src/cxx/lib/common/cppunit/testExceptions.hh | 48 +
src/cxx/lib/config/Bcl2FastqOptions.cpp | 608 ++++++++++
src/cxx/lib/config/CMakeLists.txt | 23 +
src/cxx/lib/config/SampleSheetCsv.cpp | 1014 ++++++++++++++++
src/cxx/lib/config/cppunit/CMakeLists.txt | 23 +
src/cxx/lib/config/cppunit/RegistryNames.txt | 1 +
src/cxx/lib/config/cppunit/testSampleSheetCsv.cpp | 310 +++++
src/cxx/lib/config/cppunit/testSampleSheetCsv.hh | 51 +
src/cxx/lib/conversion/BclLoader.cpp | 1028 ++++++++++++++++
src/cxx/lib/conversion/BclReader.cpp | 767 ++++++++++++
src/cxx/lib/conversion/CMakeLists.txt | 23 +
src/cxx/lib/conversion/Converter.cpp | 1168 ++++++++++++++++++
src/cxx/lib/conversion/Demultiplexer.cpp | 381 ++++++
src/cxx/lib/conversion/FastqBuffer.cpp | 34 +
src/cxx/lib/conversion/FastqCreator.cpp | 122 ++
src/cxx/lib/conversion/FastqWriter.cpp | 278 +++++
src/cxx/lib/conversion/SampleIndex.cpp | 311 +++++
src/cxx/lib/conversion/Stage.cpp | 70 ++
src/cxx/lib/conversion/Task.cpp | 38 +
src/cxx/lib/conversion/ThreadPool.cpp | 102 ++
src/cxx/lib/conversion/cppunit/CMakeLists.txt | 23 +
src/cxx/lib/conversion/cppunit/RegistryNames.txt | 1 +
.../lib/conversion/cppunit/testAdapterLocator.cpp | 192 +++
.../lib/conversion/cppunit/testAdapterLocator.hh | 64 +
src/cxx/lib/data/AggregatedBclFileReader.cpp | 165 +++
src/cxx/lib/data/BclFile.cpp | 241 ++++
src/cxx/lib/data/CMakeLists.txt | 23 +
src/cxx/lib/data/CbclFile.cpp | 365 ++++++
src/cxx/lib/data/ControlFile.cpp | 110 ++
src/cxx/lib/data/CycleBCIFile.cpp | 173 +++
src/cxx/lib/data/FastqFile.cpp | 112 ++
src/cxx/lib/data/FileReader.cpp | 177 +++
src/cxx/lib/data/FilterFile.cpp | 158 +++
src/cxx/lib/data/InteropFile.cpp | 117 ++
src/cxx/lib/data/PositionsFile.cpp | 679 +++++++++++
src/cxx/lib/data/TileBclFileReader.cpp | 78 ++
src/cxx/lib/io/CMakeLists.txt | 23 +
src/cxx/lib/io/GzipCompressor.cpp | 157 +++
src/cxx/lib/io/GzipDecompressor.cpp | 209 ++++
src/cxx/lib/io/SyncFile.cpp | 328 ++++++
src/cxx/lib/io/Utility.cpp | 95 ++
src/cxx/lib/io/Xml.cpp | 220 ++++
src/cxx/lib/io/cppunit/CMakeLists.txt | 23 +
src/cxx/lib/io/cppunit/RegistryNames.txt | 1 +
src/cxx/lib/io/cppunit/testGzipCompressor.cpp | 178 +++
src/cxx/lib/io/cppunit/testGzipCompressor.hh | 67 ++
src/cxx/lib/layout/BCIndex.cpp | 145 +++
src/cxx/lib/layout/Barcode.cpp | 168 +++
src/cxx/lib/layout/BarcodeCollisionDetector.cpp | 210 ++++
src/cxx/lib/layout/BarcodeTranslationTable.cpp | 201 ++++
src/cxx/lib/layout/CMakeLists.txt | 23 +
src/cxx/lib/layout/ConfigXml.cpp | 127 ++
src/cxx/lib/layout/CycleInfo.cpp | 45 +
src/cxx/lib/layout/FileExistenceVerifier.cpp | 217 ++++
src/cxx/lib/layout/FlowcellInfo.cpp | 77 ++
src/cxx/lib/layout/LaneInfo.cpp | 112 ++
src/cxx/lib/layout/Layout.cpp | 1246 ++++++++++++++++++++
src/cxx/lib/layout/ReadInfo.cpp | 157 +++
src/cxx/lib/layout/RunInfoXml.cpp | 223 ++++
src/cxx/lib/layout/SampleInfo.cpp | 110 ++
src/cxx/lib/layout/TileInfo.cpp | 85 ++
src/cxx/lib/layout/UseBasesMask.cpp | 350 ++++++
src/cxx/lib/layout/cppunit/CMakeLists.txt | 23 +
src/cxx/lib/layout/cppunit/RegistryNames.txt | 5 +
.../cppunit/testBarcodeCollisionDetector.cpp | 177 +++
.../layout/cppunit/testBarcodeCollisionDetector.hh | 51 +
src/cxx/lib/layout/cppunit/testConfigXml.cpp | 115 ++
src/cxx/lib/layout/cppunit/testConfigXml.hh | 47 +
src/cxx/lib/layout/cppunit/testLayout.cpp | 640 ++++++++++
src/cxx/lib/layout/cppunit/testLayout.hh | 104 ++
src/cxx/lib/layout/cppunit/testRunInfoXml.cpp | 98 ++
src/cxx/lib/layout/cppunit/testRunInfoXml.hh | 47 +
src/cxx/lib/layout/cppunit/testUseBasesMask.cpp | 399 +++++++
src/cxx/lib/layout/cppunit/testUseBasesMask.hh | 68 ++
src/cxx/lib/stats/CMakeLists.txt | 20 +
src/cxx/lib/stats/ConversionStatsXml.cpp | 110 ++
src/cxx/lib/stats/DemultiplexingStatsXml.cpp | 70 ++
src/cxx/lib/stats/DemuxReportGenerator.cpp | 184 +++
src/cxx/lib/stats/Json.cpp | 107 ++
src/cxx/lib/stats/TileStats.cpp | 32 +
src/cxx/libexec/CMakeLists.txt | 37 +
src/cxx/unittest/CMakeLists.txt | 24 +
src/cxx/unittest/RegistryName.cpp | 63 +
src/cxx/unittest/RegistryName.hh | 29 +
src/cxx/unittest/check-source.sh | 42 +
src/cxx/unittest/cppunitTest.cpp | 52 +
src/xsl/CMakeLists.txt | 26 +
src/xsl/common/Utils.xsl | 163 +++
src/xsl/demux/FlowcellSummaryPage.xsl | 115 ++
src/xsl/demux/GenerateReport.xsl | 200 ++++
src/xsl/demux/LaneSummary.xsl | 238 ++++
src/xsl/demux/NameUtils.xsl | 70 ++
src/xsl/demux/PathUtils.xsl | 63 +
249 files changed, 37537 insertions(+)
diff --git a/CODING b/CODING
new file mode 100644
index 0000000..3287e2d
--- /dev/null
+++ b/CODING
@@ -0,0 +1,11 @@
+The source tree is constructed to satisfy the following conditions:
+
+1. Easy to maintain installation scripts. In majority of cases, no changes
+ to install scripts should be needed when new files get added.
+2. Easy to understand the relationship between the source tree structure and
+ the resulting installation tree structure
+3. Fast developer access (quick branching/tagging). In particular, the ability
+ to avoid branching/tagging of rarely-changhing large binary or data files
+
+Please see the corresponding CODING file in each folder for more details
+on folder purpose and content naming conventions.
diff --git a/COPYRIGHT b/COPYRIGHT
new file mode 100644
index 0000000..000fe1a
--- /dev/null
+++ b/COPYRIGHT
@@ -0,0 +1,38 @@
+BCL to FASTQ file converter
+Copyright (c) 2007-2017 Illumina, Inc.
+
+This software is covered by the accompanying EULA, and certain third party copyright/licenses, and any user of this
+source file is bound by the terms therein.
+
+The bcl2fastq distribution includes the following code libraries,
+and are distributed according to the licensing terms governing each
+library.
+
+******************************************************************
+
+CMake - Cross Platform Makefile Generator
+Copyright 2000-2009 Kitware, Inc., Insight Software Consortium
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+* Neither the names of Kitware, Inc., the Insight Software Consortium, nor the names of their contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROF [...]
+(C)2008-09 Copyright Kitware, Inc.
+
+******************************************************************
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute, execute, and transmit the Software, and to prepare derivative works of the Software, and to permit third-parties to whom the Software is furnished to do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including the above license grant, this restriction and the following disclaimer, must be included in all copies of the Software, in whole or in part, and all derivative works of the Software, unless such copies or derivative works are solely in the form of machine-executable object code generated by a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+******************************************************************
+
+
diff --git a/Illumina_EULA.pdf b/Illumina_EULA.pdf
new file mode 100755
index 0000000..c9f351a
Binary files /dev/null and b/Illumina_EULA.pdf differ
diff --git a/build0.sh b/build0.sh
new file mode 100755
index 0000000..274f88f
--- /dev/null
+++ b/build0.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# Get major.minor.patch version from src/configure.
+B2F_MAJOR=`sed -r '/bcl2fastq_version_major=/I!d;s/.*"([0-9]*)"/\1/' src/configure `
+B2F_MINOR=`sed -r '/bcl2fastq_version_minor=/I!d;s/.*"([0-9]*)"/\1/' src/configure `
+B2F_PATCH=`sed -r '/bcl2fastq_version_patch=/I!d;s/.*"([0-9]*)"/\1/' src/configure `
+# Set build version in src/configure.
+sed -i.bak 's:bcl2fastq_version_build="[0-9]*":bcl2fastq_version_build="403":' src/configure
+"##teamcity[buildNumber '$B2F_MAJOR.$B2F_MINOR.$B2F_PATCH.403']"
+chmod +x src/configure
diff --git a/build13.sh b/build13.sh
new file mode 100755
index 0000000..c0a2681
--- /dev/null
+++ b/build13.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+ curl -X POST "http://svc_jenkins:9199554f08c2a4667d3bff7cf6a8999e@ussd-prd-jnks03.illumina.com:8080/job/Bcl2Fastq/job/develop/job/0-DeployBcl2Fastq/buildWithParameters?token=fIaAkrwzddOBOfUMm2jt&VersionString=2.19.1.402"
diff --git a/build7.sh b/build7.sh
new file mode 100755
index 0000000..e45acf1
--- /dev/null
+++ b/build7.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+version=`grep "\<BCL2FASTQ_VERSION\>" build/cxx/common/config.h | cut -d " " -f 3 | tr -d "\042"`
diff --git a/data/InterOp/parseIndexMetrics.py b/data/InterOp/parseIndexMetrics.py
new file mode 100755
index 0000000..36b1734
--- /dev/null
+++ b/data/InterOp/parseIndexMetrics.py
@@ -0,0 +1,101 @@
+# BCL to FASTQ file converter
+# Copyright (c) 2007-2017 Illumina, Inc.
+#
+# This software is covered by the accompanying EULA
+# and certain third party copyright/licenses, and any user of this
+# source file is bound by the terms therein.
+#
+# \file parseIndexMetrics.py
+#
+# \brief Parsing of binary SAV-compatible metrics.
+#
+# \author Peter Saffrey
+# \author Mauricio Varea
+#
+
+import sys
+import struct
+import csv
+
+row_order = [ "sample_name", "project_name", "index_name", "cluster_count", "cluster_proportion" ]
+
+class Lane(object):
+ def __init__(self, lanenum):
+ self.name = lanenum
+ self._indexdata = {}
+
+ def addClusterCount(self, index, value):
+ self._indexdata[index]["cluster_count"] += value
+
+ def findIndex(self, index):
+ return index in self._indexdata
+
+ def createIndex(self, index):
+ self._indexdata[index] = {}
+ self._indexdata[index]["index_name"] = index
+ self._indexdata[index]["cluster_count"] = 0
+
+ def addIndexValue(self, index, key, value):
+ self._indexdata[index][key] = value
+
+ def computeClusterPercentage(self):
+ clustercounts = [ self._indexdata[index]["cluster_count"] for index in self._indexdata ]
+ totalclusters = float(sum(clustercounts))
+ for index in self._indexdata:
+ cluster_proportion = float(self._indexdata[index]["cluster_count"]) / totalclusters
+ self.addIndexValue(index, "cluster_proportion", cluster_proportion)
+
+ def dump(self, outfh):
+ writer = csv.writer(outfh, delimiter="\t")
+ for index in self._indexdata:
+ row = [ self.name ]
+ for entry in row_order:
+ row.append(self._indexdata[index][entry])
+ writer.writerow(row)
+
+def dumpHeader(outfh):
+ writer = csv.writer(outfh, delimiter="\t")
+ writer.writerow([ "lane_number" ] + row_order)
+
+def getInt(fin, format, length):
+ return struct.unpack(format, f.read(length))[0]
+
+def getStr(fin, length):
+ fmt = "{}s".format(length)
+ return struct.unpack(fmt, f.read(length))[0]
+
+if __name__ == "__main__":
+ if len(sys.argv) != 2:
+ print "need an IndexMetricsOut.bin file"
+ sys.exit(1)
+ lanes = {}
+ binfile = sys.argv[1]
+ with open(binfile, "rb") as f:
+ version = f.read(1)
+ #print "version: %s" % ord(version)
+ while f:
+ # kind of hacky way of dropping out of the loop
+ try:
+ lanenum = getInt(f, 'H', 2)
+ if lanenum not in lanes:
+ lanes[lanenum] = Lane(lanenum)
+ except:
+ break
+ thislane = lanes[lanenum]
+ tilenum = getInt(f, 'H', 2)
+ readnum = getInt(f, 'H', 2)
+ #print "...processing: lane %d, tile %d, read %d" % (lanenum,tilenum,readnum)
+ index_length = getInt(f, 'H', 2)
+ index_name = getStr(f, index_length)
+ if not thislane.findIndex(index_name):
+ thislane.createIndex(index_name)
+ thislane.addClusterCount(index_name, getInt(f, 'I', 4))
+ name_length = getInt(f, 'H', 2)
+ thislane.addIndexValue(index_name, "sample_name", getStr(f, name_length))
+ project_length = getInt(f, 'H', 2)
+ thislane.addIndexValue(index_name, "project_name", getStr(f, project_length))
+ dumpHeader(sys.stdout)
+ for lanenum in lanes:
+ lane = lanes[lanenum]
+ lane.computeClusterPercentage()
+ lane.dump(sys.stdout)
diff --git a/data/generator/mkdata.sh b/data/generator/mkdata.sh
new file mode 100755
index 0000000..ac6a797
--- /dev/null
+++ b/data/generator/mkdata.sh
@@ -0,0 +1,192 @@
+#!/bin/bash
+
+#set -e
+#set -x
+
+# number of cycles
+cycles_count=200
+# first index cycle
+cycle_index_start=92
+# last index cycle
+cycle_index_end=108
+# number of clusters
+clusters_count=10000
+# tiles aggregation (1: nova, 0: hiseq/miseq)
+aggregate_tiles=1
+
+# generates predictable data with following properties:
+# - clusters are assigned to samples A,C,G,T periodically and in this order
+# - all clusters for samples C and G have filter flag set to true
+# - all clusters for sample T have filter flag set to false
+# - filter flag for clusters of sample A is alternating between true and false periodically
+# - there are 10000 clusters on 9 tiles
+# - tiles contain following number of clusters: 3,9,27,81,etc. (since 10000 is not divisible by 3, 9th tile contains only 160 clusters)
+# - sample sheet, runinfo XML and config XML (last one only for hiseq/miseq) need to be created by hand (examples below are compatible with above described data)
+
+# example sample sheet (<runfolder>/SampleSheet.csv)
+#[Header],,,,
+#Investigator Name,Isabelle,,,
+#Project Name,Nova,,,
+#Experiment Name,Orbital death ray research volume LXXIV,,,
+#Date,5/27/2025,,,
+#Workflow,GenerateFASTQ,,,
+#,,,,
+#[Settings],,,,
+#MaskAdapter,CGCGTATACGCGTATA,,,
+#TrimAdapter,GCGCATATGCGCATAT,,,
+#,,,,
+#[Data],,,,
+#SampleID,SampleName,index,index2
+#AA,AA,AAAAAAAA,AAAAAAAA
+#CC,CC,CCCCCCCC,CCCCCCCC
+#GG,GG,GGGGGGGG,GGGGGGGG
+#TT,TT,TTTTTTTT,TTTTTTTT
+#XX,XX,ACGTACGT,ACGTACGT
+
+# example runinfo XML (<runfolder>/RunInfo.xml)
+#<?xml version="1.0"?>
+#<RunInfo xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" Version="2">
+# <Run Id="Test" Number="42">
+# <Flowcell>Test</Flowcell>
+# <Instrument>CSSIM</Instrument>
+# <Date>8/20/2013 3:29:17 PM</Date>
+# <Reads>
+# <Read FirstCycle="1" LastCycle="91" IsIndexedRead="N" />
+# <Read FirstCycle="92" LastCycle="99" IsIndexedRead="Y" />
+# <Read FirstCycle="100" LastCycle="107" IsIndexedRead="Y" />
+# <Read FirstCycle="108" LastCycle="200" IsIndexedRead="N" />
+# </Reads>
+# <FlowcellLayout LaneCount="1" SurfaceCount="2" SwathCount="3" TileCount="6" SectionPerLane="3" LanePerSection="2" />
+# </Run>
+#</RunInfo>
+
+# example config XML (<runfolder>/Data/Intensities/BaseCalls/config.xml); hiseq/miseq only
+#<?xml version="1.0" encoding="utf-8"?>
+#<BaseCallAnalysis xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+# <Run Name="BaseCalls">
+# <TileSelection>
+# <Lane Index="1">
+# <Tile>1</Tile>
+# <Tile>2</Tile>
+# <Tile>3</Tile>
+# <Tile>4</Tile>
+# <Tile>5</Tile>
+# <Tile>6</Tile>
+# <Tile>7</Tile>
+# <Tile>8</Tile>
+# <Tile>9</Tile>
+# </Lane>
+# </TileSelection>
+# </Run>
+#</BaseCallAnalysis>
+
+
+mkdir -p ./Data/Intensities/BaseCalls/L001/
+mkdir -p ./Data/Intensities/L001/
+
+tmp_bcl_filename='./tmp_bcl'
+bci_filename='./Data/Intensities/BaseCalls/L001/s_1.bci'
+rm -f "$bci_filename"
+locs_filename='./Data/Intensities/L001/s_1.locs'
+filter_filename='./Data/Intensities/BaseCalls/L001/s_1.filter'
+
+for cycle_no in $(seq 1 $cycles_count)
+do
+ echo "Generating cycle: $cycle_no"
+
+ if [ $aggregate_tiles -eq 1 ]
+ then
+ bcl_filename=`printf './Data/Intensities/BaseCalls/L001/%04u.bcl.bgzf' $cycle_no`
+ printf '0: %.8x' $clusters_count | sed -E 's/0: (..)(..)(..)(..)/0: \4\3\2\1/' | xxd -r -g0 > "$tmp_bcl_filename"
+ gzip "$tmp_bcl_filename"
+ cat "$tmp_bcl_filename.gz" > "$bcl_filename"
+ rm "$tmp_bcl_filename.gz"
+
+ if [ $cycle_no -eq 1 ]
+ then
+ printf '0: 010000000000803f' | xxd -r -g0 > "$locs_filename"
+ printf '0: %.8x' $clusters_count | sed -E 's/0: (..)(..)(..)(..)/0: \4\3\2\1/' | xxd -r -g0 >> "$locs_filename"
+
+ printf '0: 0000000003000000' | xxd -r -g0 > "$filter_filename"
+ printf '0: %.8x' $clusters_count | sed -E 's/0: (..)(..)(..)(..)/0: \4\3\2\1/' | xxd -r -g0 >> "$filter_filename"
+ fi
+ else
+ bcl_dir=`printf './Data/Intensities/BaseCalls/L001/C%d.1/' $cycle_no`
+ mkdir -p "$bcl_dir"
+ fi
+
+
+ tile_idx=0
+ cluster_idx=0
+ clusters_on_current_tile=3
+ while [ $cluster_idx -lt $clusters_count ]
+ do
+ tile_cluster_idx=0
+ while [[ $cluster_idx -lt $clusters_count ]] && [[ $tile_cluster_idx -lt $clusters_on_current_tile ]]
+ do
+ if [[ $cycle_no -ge $cycle_index_start ]] && [[ $cycle_no -lt $cycle_index_end ]]
+ then
+ base=$(($cluster_idx % 4))
+ quality=0x3F
+ else
+ base=$(($(($cluster_idx + $cycle_no - 1)) % 0x04))
+ quality=$(($(($cluster_idx + $cycle_no - 1)) % 0x3F))
+ fi
+ printf '0: %.2x' $(($(($quality << 2)) | $base)) | xxd -r -g0 >> "$tmp_bcl_filename"
+
+ if [ $cycle_no -eq 1 ]
+ then
+ printf '0: cdcc8c3f9a99993f' | xxd -r -g0 >> "$locs_filename"
+ filter=1;
+ if [[ $(($cluster_idx % 8)) -eq 0 ]] || [[ $(($cluster_idx % 4)) -eq 3 ]]
+ then
+ filter=0;
+ fi
+ printf '0: %.2x' $filter | xxd -r -g0 >> "$filter_filename"
+ #printf '0: 01' | xxd -r -g0 >> "$filter_filename"
+ fi
+
+ tile_cluster_idx=$(($tile_cluster_idx + 1))
+ cluster_idx=$(($cluster_idx + 1))
+ done
+ if [ $aggregate_tiles -eq 1 ]
+ then
+ gzip "$tmp_bcl_filename"
+ cat "$tmp_bcl_filename.gz" >> "$bcl_filename"
+ rm "$tmp_bcl_filename.gz"
+ else
+ tile_number=$(($tile_idx + 1))
+ bcl_filename=`printf '%s/s_1_%d.bcl.gz' $bcl_dir $tile_number`
+ printf '0: %.8x' $tile_cluster_idx | sed -E 's/0: (..)(..)(..)(..)/0: \4\3\2\1/' | xxd -r -g0 > "$tmp_bcl_filename.h"
+ cat "$tmp_bcl_filename" >> "$tmp_bcl_filename.h"
+ rm "$tmp_bcl_filename"
+ gzip "$tmp_bcl_filename.h"
+ cat "$tmp_bcl_filename.h.gz" > "$bcl_filename"
+ rm "$tmp_bcl_filename.h.gz"
+
+ if [ $cycle_no -eq 1 ]
+ then
+ correct_filter_filename=`printf "./Data/Intensities/BaseCalls/L001/s_1_%d.filter" $tile_number`
+ printf '0: %.8x' $tile_cluster_idx | sed -E 's/0: (..)(..)(..)(..)/0: \4\3\2\1/' | xxd -r -g0 > "$correct_filter_filename"
+ cat "$filter_filename" >> "$correct_filter_filename"
+ rm "$filter_filename"
+
+ correct_locs_filename=`printf "./Data/Intensities/L001/s_1_%d.locs" $tile_number`
+ printf '0: %.8x' $tile_cluster_idx | sed -E 's/0: (..)(..)(..)(..)/0: \4\3\2\1/' | xxd -r -g0 > "$correct_locs_filename"
+ cat "$locs_filename" >> "$correct_locs_filename"
+ rm "$locs_filename"
+ fi
+ fi
+
+ tile_idx=$(($tile_idx + 1))
+ if [ $cycle_no -eq 1 ]
+ then
+ printf '0: %.8x' $tile_idx | sed -E 's/0: (..)(..)(..)(..)/0: \4\3\2\1/' | xxd -r -g0 >> "$bci_filename"
+ printf '0: %.8x' $tile_cluster_idx | sed -E 's/0: (..)(..)(..)(..)/0: \4\3\2\1/' | xxd -r -g0 >> "$bci_filename"
+ fi
+
+ clusters_on_current_tile=$(($clusters_on_current_tile * 3))
+ done
+done
+
+
diff --git a/redist/zlib-ng-1.9.9.tar.gz b/redist/zlib-ng-1.9.9.tar.gz
new file mode 100644
index 0000000..b21918b
Binary files /dev/null and b/redist/zlib-ng-1.9.9.tar.gz differ
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
new file mode 100644
index 0000000..1498f9c
--- /dev/null
+++ b/src/CMakeLists.txt
@@ -0,0 +1,208 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Top level cmake configuration file.
+##
+## author Come Raczy
+##
+################################################################################
+
+
+cmake_minimum_required(VERSION 2.8.4)
+enable_testing()
+
+if (DEFINED CMAKE_BUILD_TYPE)
+ set(CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE} CACHE STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel.")
+else()
+ set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel.")
+endif()
+
+project (bcl2fastq2)
+set(CPACK_RPM_SPEC_MORE_DEFINE "Prefix: ${BCL2FASTQ_PREFIX}")
+
+# Version information
+# these are just some defaults, real values should come from configure script via command line arguments
+if (NOT DEFINED BCL2FASTQ_NAME_SHORT)
+ set (BCL2FASTQ_NAME_SHORT "${PROJECT_NAME}")
+endif()
+if (NOT DEFINED BCL2FASTQ_NAME_LONG)
+ set (BCL2FASTQ_NAME_LONG "${PROJECT_NAME}")
+endif()
+if (NOT DEFINED BCL2FASTQ_COPYRIGHT)
+ set (BCL2FASTQ_COPYRIGHT "Copyright (c) Illumina, Inc.")
+endif()
+if (NOT DEFINED BCL2FASTQ_VERSION_MAJOR)
+ set (BCL2FASTQ_VERSION_MAJOR "2")
+endif()
+if (NOT DEFINED BCL2FASTQ_VERSION_MINOR)
+ set (BCL2FASTQ_VERSION_MINOR "0")
+endif()
+if (NOT DEFINED BCL2FASTQ_VERSION_PATCH)
+ set (BCL2FASTQ_VERSION_PATCH "0")
+endif()
+if (NOT DEFINED BCL2FASTQ_VERSION_BUILD)
+ set (BCL2FASTQ_VERSION_BUILD "0")
+endif()
+
+if (CPACK_GENERATOR)
+ message (STATUS "Configuring to produce the following package types: ${CPACK_GENERATOR}")
+ SET(CPACK_PACKAGE_VENDOR "Illumina")
+ SET(CPACK_PACKAGE_VERSION_MAJOR "${BCL2FASTQ_VERSION_MAJOR}")
+ SET(CPACK_PACKAGE_VERSION_MINOR "${BCL2FASTQ_VERSION_MINOR}")
+ SET(CPACK_PACKAGE_VERSION_PATCH "${BCL2FASTQ_VERSION_PATCH}.${BCL2FASTQ_VERSION_BUILD}")
+ SET(CPACK_PACKAGE_VERSION "v${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}")
+ SET(CPACK_SET_DESTDIR ON)
+ SET(CPACK_PACKAGE_RELOCATABLE OFF)
+ SET(CPACK_DEBIAN_PACKAGE_OWNER root)
+ SET(CPACK_DEBIAN_PACKAGE_GROUP root)
+ INCLUDE(CPack)
+endif (CPACK_GENERATOR)
+
+if (NOT CMAKE_PARALLEL)
+ set (CMAKE_PARALLEL "1")
+endif (NOT CMAKE_PARALLEL)
+
+# Installation directories
+if (NOT BCL2FASTQ_PREFIX)
+ set(BCL2FASTQ_PREFIX "/usr/local")
+endif (NOT BCL2FASTQ_PREFIX)
+
+if (NOT CMAKE_INSTALL_PREFIX)
+ set(CMAKE_INSTALL_PREFIX "${BCL2FASTQ_PREFIX}")
+endif (NOT CMAKE_INSTALL_PREFIX)
+
+if (CMAKE_INSTALL_PREFIX)
+ string(SUBSTRING "${CMAKE_INSTALL_PREFIX}" 0 1 slash)
+ string(COMPARE NOTEQUAL "/" "${slash}" isslash)
+ if (isslash)
+ install(CODE "
+ string(LENGTH \"\$ENV{DESTDIR}\" ddlen)
+ if (ddlen)
+ message (FATAL_ERROR \"Installation prefix must begin with '/' if DESTDIR is set.\")
+ endif (ddlen)
+ ")
+ endif (isslash)
+endif (CMAKE_INSTALL_PREFIX)
+
+if (NOT BCL2FASTQ_EXEC_PREFIX)
+ set(BCL2FASTQ_EXEC_PREFIX "." CACHE PATH "Install bin directory" FORCE)
+endif (NOT BCL2FASTQ_EXEC_PREFIX)
+
+if (NOT BCL2FASTQ_BINDIR)
+ set(BCL2FASTQ_BINDIR "${BCL2FASTQ_EXEC_PREFIX}/bin"
+ CACHE PATH "Install bin directory" FORCE)
+endif (NOT BCL2FASTQ_BINDIR)
+
+if (NOT BCL2FASTQ_LIBDIR)
+ set(BCL2FASTQ_LIBDIR
+ "${BCL2FASTQ_EXEC_PREFIX}/lib/${BCL2FASTQ_NAME_SHORT}-${BCL2FASTQ_VERSION}"
+ CACHE PATH "Install lib directory" FORCE)
+endif (NOT BCL2FASTQ_LIBDIR)
+
+if (NOT BCL2FASTQ_LIBEXECDIR)
+ set(BCL2FASTQ_LIBEXECDIR "${BCL2FASTQ_EXEC_PREFIX}/libexec/${BCL2FASTQ_NAME_SHORT}-${BCL2FASTQ_VERSION}"
+ CACHE PATH "Install libexec directory" FORCE)
+endif (NOT BCL2FASTQ_LIBEXECDIR)
+
+if (NOT BCL2FASTQ_INCLUDEDIR)
+ set(BCL2FASTQ_INCLUDEDIR "include/${BCL2FASTQ_NAME_SHORT}-${BCL2FASTQ_VERSION}"
+ CACHE PATH "Install include directory" FORCE)
+endif (NOT BCL2FASTQ_INCLUDEDIR)
+
+if (NOT BCL2FASTQ_DATADIR)
+ set(BCL2FASTQ_DATADIR "share"
+ CACHE PATH "Install data directory" FORCE)
+endif (NOT BCL2FASTQ_DATADIR)
+
+if (NOT BCL2FASTQ_ETCDIR)
+ set(BCL2FASTQ_ETCDIR
+ "${BCL2FASTQ_EXEC_PREFIX}/etc/${BCL2FASTQ_NAME_SHORT}-${BCL2FASTQ_VERSION}"
+ CACHE PATH "Install sysconfdir directory" FORCE)
+endif (NOT BCL2FASTQ_ETCDIR)
+
+if (NOT BCL2FASTQ_DOCDIR)
+ set(BCL2FASTQ_DOCDIR "doc/${BCL2FASTQ_NAME_SHORT}-${BCL2FASTQ_VERSION}"
+ CACHE PATH "Install doc directory" FORCE)
+endif (NOT BCL2FASTQ_DOCDIR)
+
+if (NOT BCL2FASTQ_MANDIR)
+ set(BCL2FASTQ_MANDIR "man"
+ CACHE PATH "Install man directory" FORCE)
+endif (NOT BCL2FASTQ_MANDIR)
+
+message (STATUS "install prefix: ${BCL2FASTQ_PREFIX}")
+message (STATUS "install exec prefix: ${BCL2FASTQ_EXEC_PREFIX}")
+message (STATUS "install bin dir: ${BCL2FASTQ_BINDIR}")
+message (STATUS "install lib dir: ${BCL2FASTQ_LIBDIR}")
+message (STATUS "install libexec dir: ${BCL2FASTQ_LIBEXECDIR}")
+message (STATUS "install include dir: ${BCL2FASTQ_INCLUDEDIR}")
+message (STATUS "install data dir: ${BCL2FASTQ_DATADIR}")
+message (STATUS "install doc dir: ${BCL2FASTQ_DOCDIR}")
+message (STATUS "install man dir: ${BCL2FASTQ_MANDIR}")
+
+add_custom_target(BCL2FASTQ_CXX)
+add_custom_target(BCL2FASTQ_OPT)
+
+# Get around broken BOOST cmake macros if BOOST_ROOT set
+set(CMAKEBOOSTROOT $ENV{BOOST_ROOT})
+if( NOT ${CMAKEBOOSTROOT} STREQUAL "" )
+ set(Boost_NO_SYSTEM_PATHS ON)
+ message("BOOST_ROOT is set, Boost_NO_SYSTEM_PATHS is ${Boost_NO_SYSTEM_PATHS}")
+endif( NOT ${CMAKEBOOSTROOT} STREQUAL "" )
+
+set(BOOST_REDIST_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../redist")
+
+# required boost libraries
+set (BCL2FASTQ_BOOST_VERSION 1.54.0)
+set (BCL2FASTQ_BOOST_COMPONENTS
+ chrono
+ date_time
+ filesystem
+ iostreams
+ program_options
+ regex
+ serialization
+ system
+ timer
+ thread
+)
+set (Boost_USE_MULTITHREAD ON)
+
+# required libxml2 and libxslt libraries
+set (BCL2FASTQ_LIBXML2_VERSION 2.7.8)
+set (LIBXML2_REDIST_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../redist")
+set (LIBXSLT_REDIST_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../redist")
+
+set (LIBXML2_INSTAL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../redist")
+set (LIBXSLT_INSTAL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../redist")
+
+set (BCL2FASTQ_LIBXSLT_VERSION 1.1.26)
+set (LIBXSLT_REDIST_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../redist")
+
+# globals and macros
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules")
+set (BCL2FASTQ_GLOBALS_CMAKE "${CMAKE_SOURCE_DIR}/cmake/globals.cmake")
+set (BCL2FASTQ_MACROS_CMAKE "${CMAKE_SOURCE_DIR}/cmake/macros.cmake")
+
+# redist includes
+include ("${CMAKE_SOURCE_DIR}/cmake/boost.cmake")
+include ("${CMAKE_SOURCE_DIR}/cmake/bcl2fastq_redist_macros.cmake")
+
+# subdirs
+add_subdirectory (cmake/preInstall)
+add_subdirectory (css)
+add_subdirectory (cxx)
+add_subdirectory (xsl)
+add_subdirectory (cmake/postInstall)
+
+
diff --git a/src/CODING b/src/CODING
new file mode 100644
index 0000000..d9d6bd7
--- /dev/null
+++ b/src/CODING
@@ -0,0 +1,6 @@
+Everything under this folder gets the cmake-configured. For example if a
+configuration or a source code file has @BCL2FASTQ_FULL_LIBDIR@ string, it gets
+converted into /<installation root>/lib/bcl2fastq_v<major>.<minor>.<development>/ in
+the installed or compiled copy of the file.
+
+
diff --git a/src/Changes b/src/Changes
new file mode 100644
index 0000000..a917fee
--- /dev/null
+++ b/src/Changes
@@ -0,0 +1,380 @@
+* BTOF-679: bcl2fastq should fully utilize the cpus
+
+bcl2fastq2-v2.18.0.12
+* BTOF-663: Revert to disallowing --use-bases-mask with UMI trimming
+
+bcl2fastq2-v2.18.0.11
+* BTOF-674: ConversionResults does not have an array of objects
+
+bcl2fastq2-v2.18.0.10
+* BTOF-663: Checked in missing files
+* BTOF-673: Update ConversionResults property in Stats.json file
+* BTOF-672: IndexMetricsOut.bin should concatenate indexes with a '-'
+* BTOF-663: Implement the TrimUMI sample sheet setting
+* BTOF-646: Read[1,2]StartFromCycle does not trim reads
+* BTOF-645: Unexpected error when index is missing from one lane
+
+bcl2fastq2-v2.18.0.9
+* BTOF-635: Requesting better fail message for incorrectly specified tiles include (--tiles)
+* BTOF-642: Missing PF Clusters information in reports file - NextSeq Regression run
+* BTOF-633: Fatal Internal Error when masking single index and outputting index files for HiSeqX Smoke
+* BTOF-638: Mask Second Index not working for NextSeq Smoke run
+* BTOF-639: Empty Index 1 behavior in v2.18 different Than v2.17
+* BTOF-447: Sample numbers are assigned by lane instead of order in samplesheet
+* BTOF-631: TrimmedBases for ReadNumber 1 is always 33 or 49 in Stats.json
+* BTOF-433: --create-fastq-for-index-reads does not produce index fastq files when sample sheet does not exist in the run folder
+
+bcl2fastq2-v2.18.0.8
+* BTOF-463: bcl2fastq2 errors out on data with more than 8 lanes
+
+bcl2fastq2-v2.18.0.7
+* BTOF-460: bcl2fastq does not properly set the number of allowed mismatches
+* BTOF-462: bcl2fastq throws an exception when the runfolder path contains a space
+
+bcl2fastq2-v2.18.0.6
+* BTOF-459: Stats files are malformed
+
+bcl2fastq2-v2.18.0.5
+* BTOF-453: Report all samples to IndexMetricsOut.bin, even ones that have 0 clusters demultiplexed.
+
+bcl2fastq2-v2.18.0.4
+* BTOF-455: "Identical barcode" error for samples loaded on different lanes
+
+bcl2fastq2-v2.18.0.3
+* BTOF-456: bcl2fastq throwing exception on smallRNA build
+
+bcl2fastq2-v2.18.0.2
+* BTOF-455: "Identical barcode" error for samples loaded on different lanes
+
+bcl2fastq2-v2.18.0.1
+* BTOF-454: bcl2fastq2 v2.18.0.0 fails with generating fastq on a smoke dataset that worked with older bcl2fastq2
+
+bcl2fastq2-v2.18.0.0
+*BTOF-452: Support multiple indexes for the same sample ID
+*BTOF-451: Consolidated stats format
+
+bcl2fastq2-v2.17.2.1
+* BTOF-450: Improve error handling when ignoring missing/corrupt files
+* BTOF-449: Read number in ConversionStats.xml can be nonsense
+* BTOF-271: Add Visual Studio project files for Windows build
+* BTOF-436: Need more file handles on Windows
+* BTOF-435: bcl2fastq runs slowly when tiles are small (NextSeq) and there are many samples
+* BTOF-250: Don't throw an exception if the Interop folder is read only. (Windows)
+* BTOF-374: Fix to adapter trimming algorithm
+* BTOF-410: Add log message for bgzf compression configuration
+* BTOF-431: UMI sequence should not be removed from reads by default
+* BTOF-404: When using "-v" options, bcl2fastq output more information than expected
+
+bcl2fastq2-v2.17.1.14
+* BTOF-368: The location coordinates are slightly different for MiSeq, NextSeq, and HiSeqX data
+
+bcl2fastq2-v2.17.1.13
+* BTOF-392: Error message does not change when barcode collision occurs for mismatch set to > 1
+* BTOF-405: Add the --no-bgzf-compression back in
+* BTOF-387: --tiles in command line option does not overwrite the samplesheet option.
+* BTOF-390: --use-bases-mask failed to parse 7bp index using "Y151, I7n, Y151" for a 8bp index run
+
+bcl2fastq2-v2.17.1.12
+* BTOF-137: Improve barcode collision error message
+
+bcl2fastq2-v2.17.1.11
+* BTOF-327: usebases mask '*' should match on 0 or more
+
+bcl2fastq2-v2.17.1.10
+* BTOF-336: IndexMetricsOut.bin should use PF clusters
+* BTOF-335: HTML report improvements
+
+bcl2fastq2-v2.17.1.9
+* BTOF-329: More bugs with masking index reads
+
+bcl2fastq2-v2.17.1.8
+* BTOF-329: UMIs are not supported with --use-base-mask
+
+bcl2fastq2-v2.17.1.7
+* BTOF-329: Handle masking an entire index
+
+bcl2fastq2-v2.17.1.6
+* BTOF-329: Throw exception when index length in --use-bases-mask doesn't match sample sheet
+* BTOF-334: Empty index in dual index dataset makes all reads undetermined
+
+bcl2fastq2-v2.17.1.5
+* BTOF-330: Masking a whole read with "--use-bases-mask" fails
+* BTOF-331: Fastq contains reads with duplicated names
+
+bcl2fastq2-v2.17.1.4
+* BTOF-328: Move files out of share/bcl2fastq2-v* directory
+* BTOF-327: usebases mask '*' should match on 0 or more
+
+bcl2fastq2-v2.17.1.3
+* BTOF-320: Fixed lane parsing logic
+
+bcl2fastq2-v2.17.1.2
+* BTOF-322: Disallow special characters in sample sheet settings
+* BTOF-320: Fixed unit tests
+* BTOF-320: Sample ID must be a unique identifier
+
+bcl2fastq2-v2.17.1.1
+* BTOF-303: Fixed unit tests
+* BTOF-303: Add option to automatically set allowed mismatches to 0 on barcode collision
+* BTOF-302: Trim white space in the settings section of the sample sheet
+* BTOF-192: Record barcode mismatches 1-5 in DemultiplexingStats.xml
+* BTOF-270: Delete --no-bgzf-compression option
+* BTOF-47: Use sample name and number for html report
+* BTOF-273: Warning message if --sample-sheet option is specified but not found
+* BTOF-272: Fixed empty space in sample sheet columns
+
+bcl2fastq2-v2.16.1.11
+* BTOF-271: Update bcl2fastq to run on Windows
+* BTOF-249: DemuxSummary files' header says SampleName but lists SampleID
+
+bcl2fastq2-v2.16.1.10
+* BTOF-250: Updates for accidental check in
+* BTOF-268: Bcl2fastq fails to read clocs files
+* BTOF-250: Don't throw an exception if the Interop folder is read only.
+* BTOF-191: Delete FASTQ files of 0 size if they exist - Fixed bug
+* BTOF-152: UMI support - Add '+' as delimiter, fix stats
+
+bcl2fastq2-v2.16.1.9
+* BTOF-191: Delete FASTQ files of 0 size if they exist
+
+bcl2fastq2-v2.16.1.8
+* BTOF-152: UMI support
+
+bcl2fastq2-v2.16.1.7
+* BTOF-246: When output reverse complement reads, the quality score does not reverse
+* BTOF-237: Isis support - stats and barcode fix
+* BTOF-245: All the index reads are masked with N/#
+* BTOF-237: Isis support - stats fix
+
+bcl2fastq2-v2.16.1.6
+*BTOF-241: Allow index2 to be used when index1 is missing in the sample sheet
+*BTOF-240: Compress complete fastq records for base space uploader compatibility
+*BTOF-239: Accept barcodes in SampleSheet.csv that are shorter than indicated by RunInfo.xml
+
+bcl2fastq2-v2.16.1.5
+* BTOF-237: Isis support - Adapter trimming stats
+
+bcl2fastq2-v2.16.1.4
+* BTOF-237: Isis support
+* BTOF-236: Support Opus+ program for HiSeq X
+
+bcl2fastq2-v2.16.1.3
+* BTOF-216: Port adapter locator enhancements from Isis
+* BTOF-214: Avoid memory allocations on multiple threads
+* BTOF-215: Use c++11
+* BTOF-211: Allow underscores in FASTQ file names
+
+bcl2fastq2-v2.16.1.2
+* merged with bcl2fastq2-v2.16.0.10
+* removed code accidentally checked in
+
+bcl2fastq2-v2.16.1.1
+* Isis support - stats and sample sheet parsing
+
+bcl2fastq2-v2.16.0.10
+* BTOF-198: Throw an exception for non-ACGTN characters in barcodes
+* BTOF-193: Separate adapters in sample sheet with '+'
+
+bcl2fastq2-v2.16.0.9
+* BTOF-201: Update default values for --minimum-trimmed-read-length and --mask-short-adapter-reads
+* BTOF-200: bcl2fastq failed when some lanes are specified but others are not
+
+bcl2fastq2-v2.16.0.8
+* BTOF-194: Fixed core dump when using --tiles
+
+bcl2fastq2-v2.16.0.7
+* BTOF-190: Throw an error if a sample is missing a barcode in the sample sheet
+* BTOF-191: Don't generate FASTQ files of 0 size
+* JAS-120: Modifications to support basespace fastq uploader
+
+bcl2fastq2-v2.16.0.6
+* BTOF-170: create stats directory using --stats-dir
+* BTOF-48: FASTQ naming convention for non-multiplexed run
+* BTOF-160: --ignore-missing-bcls option fixed
+
+bcl2fastq2-v2.16.0.5
+* BTOF-154: Updated help text of --ignore-missing-positions
+* BTOF-160: --ignore-missing-bcls option is not working
+* BTOF-48: FASTQ naming convention is different from GenerateFASTQ
+* BTOF-147: Seg fault when selecting tiles
+* BTOF-45: Seg fault when number of lanes in RunInfo.xml doesn't match config.xml
+* JAS-120: Allow bgzf compression to be turned off with: --no-bgzf-compression
+
+bcl2fastq2-v2.16.0.4
+* BTOF-156: Naming should fall back to SampleId if SampleName is absent
+* BTOF-150: Verify existence of necessary files at start of execution
+
+bcl2fastq2-v2.16.0.3
+* BTOF-136: Support for differing index lengths per lane
+* BTOF-137: Improve barcode collisions validation
+* BTOF-154: Fixed RPM installation with "--prefix"
+* BTOF-151: Adapter trimming stats categorized by sample
+* BTOF-155: Make coordinates in sequence identifiers unique
+* BTOF-151: Adapter trimming stats
+
+bcl2fastq2-v2.16.0.2
+* BTOF-146: Support for 0-padded tile numbers in filter file names
+* BTOF-153: Comply with bgzf standards
+
+bcl2fastq2-v2.16.0.1
+* BTOF-153: Write size of compressed blocks in gzip header
+* BTOF-152: Improve adapter trimming logic
+* BTOF-149: Add table with unknown barcodes to stats
+* BTOF-125: Explicit mention of version in log output
+* BTOF-137: Improve barcode collisions validation
+* BTOF-135: Option for output of fastq files without lane splitting
+* BTOF-130: Improved automatic thread allocation logic
+* BTOF-148: Improved handling of corrupt files
+* BTOF-134: Support for --sample-sheet
+* BTOF-147: Speed enhancements for the FASTQ conversion stage
+* BTOF-144: --use-bases-mask now accepts masking all bases of a read
+* BTOF-126: Adjust --minimum-trimmed-read-length to size of shortest non-index read
+* BTOF-140: Changed config param name from --ignore-missing-locs to --ignore-missing-positions
+* BTOF-129: Added support for various file formats: clocs, controls, filter, pos
+* BTOF-142: bcl2fastq does not compile with intel compilers
+* BTOF-141: comma separated list now accepted for tiles option
+* BTOF-138: Add config parameter for fastq compression level
+* BTOF-131: Support for uncompressed BCLs
+* BTOF-139: --ignore-missing-locs does not work
+
+bcl2fastq2-v2.15.0.4
+* BTOF-127: Full Hiseq X datatsets are stopping about ~75% into processing
+
+bcl2fastq2-v2.15.0.3
+* BTOF-128: Sample naming convention does not follow user guide
+
+bcl2fastq2-v2.15.0.2
+* BTOF-123: Lane Summary Cluster Counts in HTML report the same in Raw and Filtered sections for some datasets
+* BTOF-121: Boost installation is still showing a superfluous dependency on BZIP libs
+
+bcl2fastq2-v2.15.0.1
+* BTOF-120: Change versioning convention for upcoming releases
+* BTOF-118: Bcl2fastq is masking adapters when it should be trimming them
+* BTOF-119: Failure to generate reports for non-multiplexed sample
+
+bcl2fastq-02.14.05.29
+* BTOF-117: Internal Program Error occurring for datasets with more than 1 tile per lane
+* BTOF-115: logic error when dataset has no indices specified in the sample sheet
+
+bcl2fastq-02.14.05.28
+* BTOF-106: Implement table with most popular index sequences
+* BTOF-111: Downgrade reporting failures to a warning
+* BTOF-116: Error message if Reports or Stats directories clash with Project
+
+bcl2fastq-02.14.05.21
+* BTOF-109 %Reads identify in IndexMetrics InterOp are over 100% for certain samples/indexes
+* BTOF-112 'Sample_Project' or 'Project' can be used in SampleSheet.csv
+* BTOF-108 Add “--use-bases-mask" functionality
+* BTOF-105 Gather missing (PF) data in the report
+* BTOF-75 Generate DemultiplexSummary file
+
+bcl2fastq-02.14.05.13
+* BTOF-79 Add "Processing Complete" message to all log levels
+* BTOF-104 HiseqX Dataset only demultiplexes lane 1
+* BTOF-80 Avoid overwriting previous (especially 1.8.4) RPM installation
+* BTOF-82 Masking and trimming logic needs to be disabled when generating fastqs for index reads
+* BTOF-81 Add observed index sequence to FASTQ reference
+* BTOF-77 Demux by lane for hiseq datasets
+* BTOF-84 incorrect order of reads in fastq output files
+
+bcl2fastq-02.14.03.13
+* BTOF-74 the latest build does not work on NextSeq data
+
+bcl2fastq-02.14.03.07
+* BTOF-73 bcl2fastq crashes when reading bcl files with no data (registration failures/swath dropouts)
+
+bcl2fastq-02.14.02.25
+* BTOF-72 avoid -static when linking cppunittest binaries
+* BTOF-71 --ignore-missing-filter does not work
+
+bcl2fastq-02.14.02.21
+* BTOF-69 Capability to deal with tiles > 4M clusters
+* BTOF-66 Add support for bcl files that are in the same gzipped format as previous HiSeq models
+* BTOF-68 Improve auto-detection mechanism
+* BTOF-65 bcl2fastq should work, even when there is no config.xml file in the BaseCalls directory
+* BTOF-67 Patterned flowcells have only 1 locs file per run for all tiles
+
+bcl2fastq-02.14.01.17
+* BTOF-59 Provide reasonable defaults for thread counts of individual stages.
+* BTOF-56 Default value of mask-short-adapter-reads changed to 10 (was 32).
+* BTOF-61 Make adapter stringency configurable.
+* BTOF-60 Use FastIo instead of Boost lexical casts in FASTQ creation stage.
+* BTOF-59 Provide reasonable defaults for thread counts of individual stages.
+* BTOF-58 Write FASTQ complement.
+* BTOF-57 Convert only selected lanes/tiles.
+
+bcl2fastq-02.14.01.07
+* BTOF-56 Short read trimming special handling.
+* BTOF-55 Set default prefix.
+
+bcl2fastq-02.14.01.06
+
+bcl2fastq-02.13.12.27
+* BTOF-54 Include observed index in FASTQ header for Undetermined sample.
+
+bcl2fastq-02.13.12.20
+* BTOF-53 Corrected FASTQ header.
+
+bcl2fastq-02.13.12.15
+* BTOF-53 Corrected FASTQ header.
+
+bcl2fastq-02.13.12.14
+* BTOF-50 Added linking against librt.
+* BTOF-48 Added support for read-specific adapters.
+* BTOF-48 Added options for ignoring missing positions and filter files.
+* BTOF-52 Corrected handling of missing BCLs.
+* BTOF-51 Corrected BCL file name for HiSeq.
+* BTOF-42 Escape underscore in FASTQ file names.
+
+bcl2fastq-02.13.12.13
+* BTOF-42 Corrected default location of InterOp directory.
+
+bcl2fastq-02.13.12.12
+* BTOF-42 Escape non-alphanumeric characters in FASTQ file names.
+* BTOF-51 Corrected BCL file name for HiSeq.
+
+bcl2fastq-02.13.12.11
+* BTOF-36 HiSeq/MiSeq support.
+* BTOF-33 Correct positions data.
+* BTOF-42 ISIS-like naming scheme for output files.
+* BTOF-46 Corrected mismatching tiles in InterOp stats.
+* BOTF-49 Changed default number of allowed barcode mismatches to 1.
+
+bcl2fastq-02.13.11.15
+* BTOF-45 Use gcc/g++ by default.
+* BTOF-44 Added legacy sample sheet option 'Adapter' for adapter trimming.
+* BTOF-43 Corrected value of adapter stringency.
+* BTOF-39 Dependency on Boost.Chrono made conditional.
+* BTOF-38 Include non-PF clusters.
+* BTOF-37 Configurable number of mismatches.
+
+bcl2fastq-02.13.11.08
+* BTOF-34 Package name.
+* BTOF-35 Remove superfluous dependency on BZIP.
+* BTOF-19 Allow single sample without index in sample sheet.
+* BTOF-32 Do not demultiplex in case there are no index reads.
+* BTOF-30 Keep BCL and FASTQ files open (hacked, needs refactoring).
+* BTOF-30 Added fake BCL producer and FASTQ consumer.
+* BOTF-30 Added timing of individual stages.
+* BOTF-28 Solved deadlock after assertion.
+* BTOF-24 Merge data- and index- reads metadata into single container.
+* BTOF-22 Do not produce empty GZIPs.
+* BOTF-18 Sample sheet alternative column names "Sample_ID" and "Sample_Name".
+* BTOF-20 Skip only InterOp for filtered out clusters.
+* BTOF-20 Do not include filtered-out clusters in InterOp file.
+* BTOF-20 Do not include sample #0 in InterOp file.
+* BTOF-21 Cluster filtering.
+* BTOF-16 Demultiplexing statistics.
+* BTOF-15 Missng BCLs as no-calls.
+* BTOF-14 Create FASTQs for index reads.
+* BTOF-13 Adapter masking/trimming.
+* BTOF-12 Reading adapters from sample sheet.
+* BTOF-09 Corrected bug in initialization of lookup barcode table.
+* BTOF-09 Demultiplexing.
+* BTOF-8 Multiple barcodes per sample.
+* BTOF-10 Removed superfluous dependency on libgcrypt.
+* BTOF-8 Sample sheet parsing.
+* BTOF-4 Added positions handling.
+* BTOF-4 Added tile handling.
+* BTOF-4 Review amendments.
+* BTOF-4 Initial commit.
diff --git a/src/INSTALL b/src/INSTALL
new file mode 100644
index 0000000..fb4ceb0
--- /dev/null
+++ b/src/INSTALL
@@ -0,0 +1,113 @@
+
+Part I - Requirements
+=====================
+
+Network Infrastructure
+----------------------
+
+The large data volumes generated and moved when running bcl2fastq2 mean that you must have a
+high-throughput ethernet connection (at least 1 Gigabit recommended) or other data transfer
+mechanism. You can use a single multi-processor or a multi-core computer running Linux.
+
+Analysis Computer
+-----------------
+
+Illumina supports running bcl2fastq2 Conversion Software v2.17 only on Linux operating systems.
+If all of the prerequisites described in this section are met, it might be possible to run the
+software on other 64-bit Unix variants.
+
+Memory Requirements
+-------------------
+
+bcl2fastq2 requires a minimum of 32 GB RAM.
+
+Software Requirements
+---------------------
+
+bcl2fastq2 has been primarily developed and tested on CentOS 6 and RedHat Enterprise Linux 5,
+the recommended and supported platform. If all of the prerequisites described in this section
+are met, it may be possible to install and run bcl2fastq2 on other 64-bit Linux distributions
+or on other Unix variants. Particularly a similar distribution such as Fedora.
+
+The following software is required to run bcl2fastq2:
+
+ - zlib
+ - librt
+ - libpthread
+
+To build bcl2fastq2, you need the following software. Versions listed are tested and supported;
+newer versions are untested.
+
+ - gcc 4.7 (with support for c++11)
+ - boost 1.54 (with its dependencies)
+ - CMake 2.8.9
+ - zlib
+ - librt
+ - libpthread
+
+
+Part II - Installing bcl2fastq2
+===============================
+
+Installing from RPM Package
+---------------------------
+
+Root access to the system is a prerequisite for this installation procedure. The command line
+for installing the RPM file is as follows:
+
+ yum install -y <rpm-package-name>
+
+The starting point for the bcl2fastq converter is the binary executable /usr/local/bin/bcl2fastq.
+
+To install the rpm package in a user specified location, use the following command line:
+
+ rpm --install --prefix <user-specified-directory> <rpm-package-name>
+
+Installing from Source
+----------------------
+
+The installation uses the directory locations specified by the following environment variables:
+ * SOURCE: Location of the bcl2fastq2 source code
+ * BUILD: Location of the build directory
+ * INSTALL_DIR: Location where the executable is installed
+
+For example, these environment variables could be set as:
+
+ export TMP=/tmp
+ export SOURCE=${TMP}/bcl2fastq
+ export BUILD=${TMP}/bcl2fastq2-v2.17.1.x-build
+ export INSTALL_DIR=/usr/local/bcl2fastq2-v2.17.1.x
+
+NOTE: The build directory must be different from the source directory.
+
+The install procedure follows the usual steps: decompressing and extracting the source,
+configuring the build, building the package, and installing:
+
+1) Decompress and extract the source code:
+
+ cd ${TMP}
+ tar -xvzf path-to-tarball/bcl2fastq2-v2.17.10.x.tar.gz
+
+This command creates a bcl2fastq subdirectory in the ${TMP} directory.
+
+2) Configure the build:
+
+ mkdir ${BUILD}
+ cd ${BUILD}
+ ${SOURCE}/src/configure --prefix=${INSTALL_DIR}
+
+These commands create a build directory, move to that directory, and then run configure
+in that directory. The parameter --prefix provides the absolute path to the install
+directory. The command creates a subdirectory in the ${TMP} directory.
+
+3) Build the package:
+
+ make
+
+4) Install:
+
+ make install
+
+NOTE: This step can require root privilege, depending on the ${INSTALL_DIR} directory.
+
+
diff --git a/src/cmake/bcl2fastq_redist_macros.cmake b/src/cmake/bcl2fastq_redist_macros.cmake
new file mode 100644
index 0000000..b86326b
--- /dev/null
+++ b/src/cmake/bcl2fastq_redist_macros.cmake
@@ -0,0 +1,106 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file bcl2fastq_redist_macrocs.cmake
+##
+## Configuration file for libxml2, libxslt library search, and redist.
+##
+## author David Kimmel
+##
+################################################################################
+
+#
+# Find a library with specific version, search on system defaults
+#
+macro(find_package_version libname version)
+ string(TOUPPER ${libname} ${libname}_UPPER)
+
+ find_package("${libname}" ${version})
+
+ if(${${libname}_UPPER}_FOUND)
+ if("${${${libname}_UPPER}_VERSION_STRING}" STREQUAL "${version}")
+ message(" Found: ${libname}, correct version ${version}")
+ message(" ${${libname}_UPPER}_INCLUDE_DIR = ${${${libname}_UPPER}_INCLUDE_DIR}")
+ message(" ${${libname}_UPPER}_LIBRARIES = ${${${libname}_UPPER}_LIBRARIES}")
+ else("${${${libname}_UPPER}_VERSION_STRING}" STREQUAL "${version}")
+ message(" Not found: ${libname}, incorrect version ( ${${${libname}_UPPER}_VERSION} )")
+ set(${${libname}_UPPER}_FOUND "FALSE")
+ endif("${${${libname}_UPPER}_VERSION_STRING}" STREQUAL "${version}")
+ endif(${${libname}_UPPER}_FOUND)
+
+endmacro(find_package_version libname version)
+
+#
+# Redist if not found (untar, configure, make install)
+#
+macro(redist_package name version args)
+
+ if (NOT CMAKE_VERBOSE_MAKEFILE)
+ list(APPEND REDIST_QUIET "OUTPUT_QUIET")
+ list(APPEND REDIST_QUIET "ERROR_QUIET")
+ endif (NOT CMAKE_VERBOSE_MAKEFILE)
+
+ unset(HAVE_${name} CACHE)
+ set(${name}_VERREQ "${name}-${version}")
+ string(TOLOWER "${${name}_VERREQ}" nameb)
+ set(tgzname "${${name}_REDIST_DIR}/${nameb}.tar.gz")
+ message(" Redist ${name} ${nameb}")
+ message(" tar xzf (${tgzname})")
+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${tgzname} RESULT_VARIABLE TMP_RESULT ${REDIST_QUIET})
+ if (NOT TMP_RESULT)
+ message(STATUS "Successfuly unpacked ${tgzname} from the distribution package...")
+ else (NOT TMP_RESULT)
+ message (FATAL_ERROR "Failed to unpack ${tgzname}")
+ endif (NOT TMP_RESULT)
+
+ message(" configure ${args} in ${nameb}")
+ execute_process(COMMAND ./configure ${args} WORKING_DIRECTORY ${nameb} RESULT_VARIABLE TMP_RESULT ${REDIST_QUIET})
+ if (NOT TMP_RESULT)
+ message(STATUS "Successfuly configured ${nameb} from the distribution package...")
+ else (NOT TMP_RESULT)
+ message (FATAL_ERROR "Failed to configure ${nameb}")
+ endif (NOT TMP_RESULT)
+
+ message(" configure result ${res}")
+ message(" make install ${nameb}")
+ execute_process(COMMAND make -j ${CMAKE_PARALLEL} install WORKING_DIRECTORY ${nameb} RESULT_VARIABLE TMP_RESULT ${REDIST_QUIET})
+ if (NOT TMP_RESULT)
+ message(STATUS "Successfuly built ${nameb} from the distribution package...")
+ else (NOT TMP_RESULT)
+ message (FATAL_ERROR "Failed to build ${nameb}")
+ endif (NOT TMP_RESULT)
+endmacro(redist_package name args)
+
+#
+# Find a library given a path hint, assume version will be correct
+#
+macro(find_library_redist name pathhint header library)
+ unset(${name}_LIBRARIES CACHE)
+ # Search for library
+ unset(${name}_LIBRARIES CACHE)
+ find_library(${name}_LIBRARIES NAMES ${CMAKE_STATIC_LIBRARY_PREFIX}${library}${CMAKE_STATIC_LIBRARY_SUFFIX} HINTS ${pathhint}/lib NO_DEFAULT_PATH)
+
+ message(STATUS "Find library redist ${namenolib}, HINTS ${pathhint}/lib: ${${name}_LIBRARIES}")
+ # Search for include path
+ unset(${name}_INCLUDE_DIR CACHE)
+ string(TOLOWER ${name} namel)
+ find_path(${name}_INCLUDE_DIR ${header} HINTS ${pathhint}/include PATH_SUFFIXES ${namel} NO_DEFAULT_PATH)
+ set(${name}_INCLUDE_DIR ${${name}_INCLUDE_DIR} CACHE STRING "lib BOOL" FORCE)
+
+ if(${name}_INCLUDE_DIR AND ${name}_LIBRARIES)
+ set (HAVE_${name} true CACHE BOOL "lib bool" FORCE)
+ message (STATUS "Found redist ${name} header: ${${name}_INCLUDE_DIR}/${header}")
+ message (STATUS "Found redist ${name} library: ${${name}_LIBRARY}")
+ endif(${name}_INCLUDE_DIR AND ${name}_LIBRARIES)
+
+endmacro(find_library_redist name pathhint header)
+
+
diff --git a/src/cmake/boost.cmake b/src/cmake/boost.cmake
new file mode 100644
index 0000000..c48f4c7
--- /dev/null
+++ b/src/cmake/boost.cmake
@@ -0,0 +1,143 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for boost installation
+##
+## author Come Raczy
+##
+################################################################################
+
+macro (resetFindBoost)
+ unset (Boost_FOUND CACHE)
+ unset (Boost_INCLUDE_DIRS CACHE)
+ unset (Boost_INCLUDE_DIR CACHE)
+ unset (Boost_LIBRARIES CACHE)
+ unset (Boost_LIBRARY_DIRS CACHE)
+ unset (Boost_VERSION CACHE)
+ unset (Boost_LIB_VERSION CACHE)
+ unset (Boost_MAJOR_VERSION CACHE)
+ unset (Boost_MINOR_VERSION CACHE)
+ unset (Boost_SUBMINOR_VERSION CACHE)
+ unset (Boost_USE_STATIC_LIBS CACHE)
+
+ unset (ENV{BOOST_LIBRARYDIR})
+ unset (Boost_USE_MULTITHREADED CACHE)
+
+ foreach (COMPONENT ${BCL2FASTQ_BOOST_COMPONENTS})
+ STRING(TOUPPER ${COMPONENT} UPPERCOMPONENT)
+ unset (Boost_${UPPERCOMPONENT}_FOUND CACHE)
+ unset (Boost_${UPPERCOMPONENT}_LIBRARY CACHE)
+ unset (Boost_${UPPERCOMPONENT}_LIBRARY_RELEASE CACHE)
+ unset (Boost_${UPPERCOMPONENT}_LIBRARY_DEBUG CACHE)
+ endforeach (COMPONENT ${BCL2FASTQ_BOOST_COMPONENTS})
+
+
+ unset (Boost_FOUND)
+ unset (Boost_INCLUDE_DIRS)
+ unset (Boost_INCLUDE_DIR)
+ unset (Boost_LIBRARIES)
+ unset (Boost_LIBRARY_DIRS)
+ unset (Boost_VERSION)
+ unset (Boost_LIB_VERSION)
+ unset (Boost_MAJOR_VERSION)
+ unset (Boost_MINOR_VERSION)
+ unset (Boost_SUBMINOR_VERSION)
+ unset (Boost_USE_STATIC_LIBS)
+
+ foreach (COMPONENT ${BCL2FASTQ_BOOST_COMPONENTS})
+ STRING(TOUPPER ${COMPONENT} UPPERCOMPONENT)
+ unset (Boost_${UPPERCOMPONENT}_FOUND)
+ unset (Boost_${UPPERCOMPONENT}_LIBRARY)
+ unset (Boost_${UPPERCOMPONENT}_LIBRARY_RELEASE)
+ unset (Boost_${UPPERCOMPONENT}_LIBRARY_DEBUG)
+ endforeach (COMPONENT ${BCL2FASTQ_BOOST_COMPONENTS})
+
+endmacro (resetFindBoost)
+
+#
+# Not only finds boost but also sets the variables so that
+# it is being used for include and linking
+# Also makes sure pthread is available for boost
+#
+macro(bcl2fastq_find_boost boost_version boost_components)
+
+ # pthread library required by boost
+ bcl2fastq_find_library(PTHREAD "pthread.h" pthread)
+ if (HAVE_PTHREAD)
+ set (BCL2FASTQ_ADDITIONAL_LIB ${BCL2FASTQ_ADDITIONAL_LIB} pthread)
+ message(STATUS "pthread supported")
+ else (HAVE_PTHREAD)
+ message(STATUS "pthread headers: ${PTHREAD_INCLUDE_DIR}")
+ message(STATUS "pthread library: ${PTHREAD_LIBRARY}")
+ message(FATAL_ERROR "pthread library is required to build the BCL2FASTQ")
+ endif (HAVE_PTHREAD)
+
+ find_package(Boost ${boost_version} REQUIRED ${boost_components})
+
+ include_directories(BEFORE SYSTEM ${Boost_INCLUDE_DIRS})
+
+ set (HAVE_LIBBOOST_DATE_TIME ${Boost_DATE_TIME_FOUND})
+ set (HAVE_LIBBOOST_FILESYSTEM ${Boost_FILESYSTEM_FOUND})
+ set (HAVE_LIBBOOST_IOSTREAMS ${Boost_IOSTREAMS_FOUND})
+ set (HAVE_LIBBOOST_PROGRAM_OPTIONS ${Boost_PROGRAM_OPTIONS_FOUND})
+ set (HAVE_LIBBOOST_PYTHON ${Boost_PYTHON_FOUND})
+ set (HAVE_LIBBOOST_REGEX ${Boost_REGEX_FOUND})
+ set (HAVE_LIBBOOST_SERIALIZATION ${Boost_SERIALIZATION_FOUND})
+ set (HAVE_LIBBOOST_SYSTEM ${Boost_SYSTEM_FOUND})
+endmacro(bcl2fastq_find_boost)
+
+
+if (BCL2FASTQ_FORCE_STATIC_LINK)
+ set(Boost_USE_STATIC_LIBS ON)
+endif (BCL2FASTQ_FORCE_STATIC_LINK)
+
+find_package(Boost ${BCL2FASTQ_BOOST_VERSION} COMPONENTS ${BCL2FASTQ_BOOST_COMPONENTS})
+#
+# If the right version of boost is not found, it will be built from the distribution
+#
+if (NOT Boost_FOUND)
+ if (BOOSTROOT)
+ message (STATUS "BOOSTROOT is set to ${BOOSTROOT} but boost ${BCL2FASTQ_BOOST_VERSION} was not found.")
+ message (FATAL_ERROR "Unset BOOSTROOT or set it to the root location of boost ${BCL2FASTQ_BOOST_VERSION}.")
+ endif(BOOSTROOT)
+ if (BOOST_ROOT)
+ message (STATUS "BOOST_ROOT is set to ${BOOST_ROOT} but boost ${BCL2FASTQ_BOOST_VERSION} was not found.")
+ message (FATAL_ERROR "Unset BOOST_ROOT or set it to the root location of boost ${BCL2FASTQ_BOOST_VERSION}.")
+ endif(BOOST_ROOT)
+
+ # Try to find it in target installation location
+ resetFindBoost()
+ message(STATUS "Boost ${BCL2FASTQ_BOOST_VERSION} not found. Boost will be built from the distribution...")
+
+ set(ENV{BCL2FASTQ_BOOST_COMPONENTS} "${BCL2FASTQ_BOOST_COMPONENTS}")
+ set(ENV{BCL2FASTQ_BOOST_VERSION} "${BCL2FASTQ_BOOST_VERSION}")
+
+ execute_process(COMMAND "/bin/bash"
+"${CMAKE_SOURCE_DIR}/cmake/bootstrap/installBoost.sh" "${BOOST_REDIST_DIR}"
+"${CMAKE_CURRENT_BINARY_DIR}/bootstrap" "${CMAKE_PARALLEL}" RESULT_VARIABLE TMP_RESULT )
+
+ if (NOT TMP_RESULT)
+ message(STATUS "Successfuly built boost ${BCL2FASTQ_BOOST_VERSION} from the distribution package...")
+ else (NOT TMP_RESULT)
+ message (FATAL_ERROR "Failed to build boost ${BCL2FASTQ_BOOST_VERSION}")
+ endif (NOT TMP_RESULT)
+
+ set (BOOST_ROOT "${CMAKE_CURRENT_BINARY_DIR}/bootstrap")
+ #the re-distributed boost uses system layout which means no -mt in file names.
+ set (Boost_USE_MULTITHREADED OFF)
+ #force static linking with redistributed boost.
+ set (Boost_USE_STATIC_LIBS ON)
+
+endif (NOT Boost_FOUND)
+
+
diff --git a/src/cmake/bootstrap/common.sh b/src/cmake/bootstrap/common.sh
new file mode 100644
index 0000000..64d8db5
--- /dev/null
+++ b/src/cmake/bootstrap/common.sh
@@ -0,0 +1,51 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file common.sh
+##
+## Definition of functions and variables common to all bootstrap scripts.
+##
+## author Come Raczy
+##
+################################################################################
+
+function common_options () {
+ TEMP=`getopt -n $SCRIPT -o fc -- "$@"`
+ if [ $? != 0 ] ; then echo $SCRIPT: invalid option >&2; echo "Terminating..." >&2 ; exit 2 ; fi
+ eval set -- "$TEMP"
+ FORCE=
+ CLEAN=
+ while true ; do
+ case "$1" in
+ -f) FORCE=true ; shift ;;
+ -c) CLEAN=true ; shift ;;
+ --) shift ; break ;;
+ *) echo "Internal error!" >&2; exit 2 ;;
+ esac
+ done
+}
+
+function common_create_source () {
+ if [[ ! -e $SOURCE_TARBALL ]] ; then
+ echo $SCRIPT: source tarball $SOURCE_TARBALL not found >&2
+ exit 2
+ fi
+ echo Decompressing $SOURCE_TARBALL >&2
+ mkdir -p ${BUILD_DIR}
+ tar -C ${BUILD_DIR} -${TARBALL_COMPRESSION}xf $SOURCE_TARBALL
+
+ if [[ ! -d $SOURCE_DIR ]] ; then
+ echo $SOURCE_DIR does not exist >&2
+ exit 2
+ fi
+}
+
+
diff --git a/src/cmake/bootstrap/installBoost.sh b/src/cmake/bootstrap/installBoost.sh
new file mode 100644
index 0000000..8f25079
--- /dev/null
+++ b/src/cmake/bootstrap/installBoost.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file installBoost.sh
+##
+## Script to install Boost.
+##
+## author Come Raczy
+##
+################################################################################
+
+REDIST_DIR=$1
+INSTALL_DIR=$2
+if [[ $# -ge 3 ]] ; then PARALLEL=$3 ; else PARALLEL=1 ; fi
+
+. `dirname "$0"`/common.sh
+
+BUILD_DIR=${INSTALL_DIR}/build
+BIN_DIR=${INSTALL_DIR}/bin
+LIB_DIR=${INSTALL_DIR}/lib
+INCLUDE_DIR=${INSTALL_DIR}/include
+
+SCRIPT=`basename "$0"`
+VERSION=`echo ${BCL2FASTQ_BOOST_VERSION} | sed "s/\./_/g"`
+SOURCE_TARBALL=${REDIST_DIR}/boost_${VERSION}.tar.bz2
+TARBALL_COMPRESSION=j
+SOURCE_DIR=${BUILD_DIR}/boost_${VERSION}
+
+common_options $@
+
+if [[ $CLEAN ]] ; then
+ echo removing $SOURCE_DIR
+ rm -rf $SOURCE_DIR ${INCLUDE_DIR}/boost ${LIB_DIR}/libboost_*.{a,so}
+ exit 0
+fi
+
+common_create_source
+cd ${SOURCE_DIR} \
+ && ./bootstrap.sh ${BOOTSTRAP_OPTIONS} --prefix=${INSTALL_DIR} --with-libraries=`echo ${BCL2FASTQ_BOOST_COMPONENTS} | sed "s/;/,/g"` \
+ && echo "using gcc : : ${CXX} ;" >${SOURCE_DIR}/tools/build/v2/user-config.jam \
+ && echo "modules.poke : NO_BZIP2 : 1 ;" >>${SOURCE_DIR}/tools/build/v2/user-config.jam \
+ && ./bjam -j$PARALLEL ${BJAM_OPTIONS} --libdir=${INSTALL_DIR}/lib --layout=system link=static threading=multi install
+
+if [ $? != 0 ] ; then echo "$SCRIPT: build failed: Terminating..." >&2 ; exit 1 ; fi
+
+#echo "Cleaning up ${SOURCE_DIR}" >&2
+#rm -rf ${SOURCE_DIR}
+
+echo "boost-$VERSION installed successfully" >&2
+
+
diff --git a/src/cmake/bootstrap/installCmake.sh b/src/cmake/bootstrap/installCmake.sh
new file mode 100755
index 0000000..3d4ceb3
--- /dev/null
+++ b/src/cmake/bootstrap/installCmake.sh
@@ -0,0 +1,98 @@
+#!/bin/bash
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file installCmake.sh
+##
+## Installation script for cmake.
+##
+## author Come Raczy
+##
+################################################################################
+REDIST_DIR=$1
+INSTALL_DIR=$2
+if [[ $# -ge 3 ]] ; then PARALLEL=$3 ; else PARALLEL=1 ; fi
+
+. `dirname "$0"`/common.sh
+
+BUILD_DIR=${INSTALL_DIR}/build
+BIN_DIR=${INSTALL_DIR}/bin
+LIB_DIR=${INSTALL_DIR}/lib
+INCLUDE_DIR=${INSTALL_DIR}/include
+
+CMAKE_MAJOR=2
+CMAKE_MINOR=8
+CMAKE_PATCH=9
+CMAKE_REQUIRED="$CMAKE_MAJOR.$CMAKE_MINOR.$CMAKE_PATCH"
+TARBALL_VERSION="$CMAKE_MAJOR.$CMAKE_MINOR.$CMAKE_PATCH"
+SCRIPT=`basename "$0"`
+SOURCE_TARBALL=${REDIST_DIR}/cmake-$TARBALL_VERSION.tar.gz
+TARBALL_COMPRESSION=z
+SOURCE_DIR=${BUILD_DIR}/cmake-$TARBALL_VERSION
+CMAKE_DIR=cmake-$CMAKE_MAJOR.$CMAKE_MINOR
+
+common_options $@
+
+if [[ $CLEAN ]] ; then
+ echo removing $SOURCE_DIR >&2
+ rm -rf $SOURCE_DIR
+ rm -rf ${INSTALL_DIR}/{doc,share}/$CMAKE_DIR
+ rm -f ${BIN_DIR}/{ccmake,cmake,cpack,ctest}
+ rm -f ${INSTALL_DIR}/man/man1/{ccmake,cmake,cmakecommands,cmakecompat,cmakemodules,cmakeprops,cmakevars,cpack,ctest}.1
+ exit 0
+fi
+
+AVAILABLE_CMAKE_VERSION=`cmake --version 2> /dev/null`
+if [[ "${AVAILABLE_CMAKE_VERSION}" =~ ^cmake\ version\ ([0-9]+)\.([0-9]+)\.([0-9]+) && ! $FORCE ]] ; then
+ MAJOR=${BASH_REMATCH[1]}
+ MINOR=${BASH_REMATCH[2]}
+ PATCH=${BASH_REMATCH[3]}
+ if [[ "$MAJOR" -eq "$CMAKE_MAJOR" && ( "$MINOR" -gt "$CMAKE_MINOR" || "$MINOR" -eq "$CMAKE_MINOR" && "$PATCH" -ge "$CMAKE_PATCH" ) ]] ; then
+ echo "${BASH_REMATCH[0]} (>= $CMAKE_REQUIRED) is already installed" >&2
+ echo nothing to be done >&2
+ exit 1
+ fi
+fi
+
+OLD_CMAKE_VERSION=`${BIN_DIR}/cmake --version 2> /dev/null`;
+if [[ $OLD_CMAKE_VERSION == "cmake version $TARBALL_VERSION" && ! $FORCE ]] ; then
+ echo cmake version \"$TARBALL_VERSION\" is already installed at ${BIN_DIR}/cmake >&2
+ echo nothing to be done >&2
+ exit 0
+elif [[ $OLD_CMAKE_VERSION != "" ]] ; then
+ echo unable to install cmake version \"$TARBALL_VERSION\" in ${BIN_DIR} >&2
+ echo cmake version \"$OLD_CMAKE_VERSION\" is in the way. >&2
+ echo Please use an empty location to build the product. >&2
+ exit 2
+fi
+
+
+##
+## cleanup all existing source directory before proceeding
+##
+rm -rf $SOURCE_DIR
+
+common_create_source
+
+echo "Extracted cmake version $TARBALL_VERSION source code into $SOURCE_DIR" >&2
+echo "Installing cmake using: './bootstrap --prefix=\"${INSTALL_DIR}\" --parallel=\"$PARALLEL\" && make -j \"$PARALLEL\" && make install'" >&2
+cd $SOURCE_DIR && ./bootstrap --prefix="${INSTALL_DIR}" --parallel="$PARALLEL" && make -j "$PARALLEL" && make install
+
+if [ $? != 0 ] ; then echo "cmake: build failed: Terminating..." >&2 ; exit 2 ; fi
+
+echo "Cleaning up ${SOURCE_DIR}" >&2
+rm -rf ${SOURCE_DIR}
+
+echo CMake installed successfully >&2
+
+exit 0
+
+
diff --git a/src/cmake/cppunit.cmake b/src/cmake/cppunit.cmake
new file mode 100644
index 0000000..fce9594
--- /dev/null
+++ b/src/cmake/cppunit.cmake
@@ -0,0 +1,101 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file cppunit.cmake
+##
+## Configuration file for the cppunit subfolders
+##
+## author Come Raczy
+##
+################################################################################
+
+##
+## the location of the cppunit shared libraries will be needed for
+## LD_LIBRARY_PATH
+##
+
+if (HAVE_CPPUNIT)
+ get_filename_component(CPPUNIT_LOCATION ${HAVE_CPPUNIT} PATH)
+ include_directories(${CPPUNIT_INCLUDE_DIR})
+else (HAVE_CPPUNIT)
+ message(FATAL_ERROR "cppunit not found")
+endif (HAVE_CPPUNIT)
+
+##
+## find all the source files
+##
+
+file (GLOB BCL2FASTQ_TEST_SOURCE_LIST "*.cpp")
+
+##
+## create the targets to build the tests
+##
+
+set(BCL2FASTQ_CPPUNIT_TEST_NAME cppunitTest)
+add_executable(${BCL2FASTQ_UNIQUE_PREFIX}${BCL2FASTQ_CPPUNIT_TEST_NAME} ${BCL2FASTQ_TEST_SOURCE_LIST})
+set_target_properties(${BCL2FASTQ_UNIQUE_PREFIX}${BCL2FASTQ_CPPUNIT_TEST_NAME} PROPERTIES OUTPUT_NAME ${BCL2FASTQ_CPPUNIT_TEST_NAME})
+add_test(cppunit_${BCL2FASTQ_UNIQUE_PREFIX}${BCL2FASTQ_CPPUNIT_TEST_NAME} "${CMAKE_CURRENT_BINARY_DIR}/${BCL2FASTQ_CPPUNIT_TEST_NAME}")
+
+include_directories (${BCL2FASTQ_CXX_ALL_INCLUDES} ${BCL2FASTQ_OPT_INC} "${CMAKE_SOURCE_DIR}/cxx/unittest")
+
+set(BCL2FASTQ_LINK_LIBRARIES "-lpthread")
+if (HAVE_ZLIB)
+ set(BCL2FASTQ_LINK_LIBRARIES "${BCL2FASTQ_LINK_LIBRARIES} -lz")
+endif (HAVE_ZLIB)
+#if (HAVE_BZLIB)
+# set(BCL2FASTQ_LINK_LIBRARIES "${BCL2FASTQ_LINK_LIBRARIES} -lbz2")
+#endif (HAVE_BZLIB)
+if (NOT BCL2FASTQ_FORCE_STATIC_LINK)
+ set(BCL2FASTQ_LINK_LIBRARIES "${BCL2FASTQ_LINK_LIBRARIES} -ldl")
+endif (NOT BCL2FASTQ_FORCE_STATIC_LINK)
+
+target_link_libraries (${BCL2FASTQ_UNIQUE_PREFIX}${BCL2FASTQ_CPPUNIT_TEST_NAME}
+ bcl2fastq_${BCL2FASTQ_LIB_DIR} ${BCL2FASTQ_AVAILABLE_LIBRARIES}
+ bcl2fastq_cppunit ${Boost_LIBRARIES}
+ ${BCL2FASTQ_LINK_LIBRARIES} ${BCL2FASTQ_DEP_LIB} ${CPPUNIT_LIBRARY})
+
+##
+## Run some sanity check on the source file
+##
+foreach(BCL2FASTQ_CPPUNIT_SOURCE_FILE ${BCL2FASTQ_TEST_SOURCE_LIST})
+ get_filename_component(FILE_NAME ${BCL2FASTQ_CPPUNIT_SOURCE_FILE} NAME)
+ set(BCL2FASTQ_CPPUNIT_BINARY_FILE "${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}")
+ add_custom_command(TARGET ${BCL2FASTQ_UNIQUE_PREFIX}${BCL2FASTQ_CPPUNIT_TEST_NAME}
+ PRE_BUILD
+ COMMAND ${CMAKE_SOURCE_DIR}/cxx/unittest/check-source.sh ARGS ${BCL2FASTQ_CPPUNIT_BINARY_FILE}.checked ${BCL2FASTQ_CPPUNIT_SOURCE_FILE}
+ COMMENT "Sanity check on ${BCL2FASTQ_CPPUNIT_SOURCE_FILE}")
+endforeach(BCL2FASTQ_CPPUNIT_SOURCE_FILE)
+
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/RegistryNames.txt
+ COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/RegistryNames.txt ${CMAKE_CURRENT_BINARY_DIR}
+ DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/RegistryNames.txt
+ COMMENT "Copying RegistryNames.txt for ${BCL2FASTQ_UNIQUE_PREFIX}${BCL2FASTQ_CPPUNIT_TEST_NAME}")
+
+##
+## create the targets to run the tests
+##
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${BCL2FASTQ_CPPUNIT_TEST_NAME}.passed
+ COMMAND export LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:${CPPUNIT_LOCATION} && ./${BCL2FASTQ_CPPUNIT_TEST_NAME}
+ COMMAND touch ${BCL2FASTQ_CPPUNIT_TEST_NAME}.passed
+ DEPENDS ${BCL2FASTQ_UNIQUE_PREFIX}${BCL2FASTQ_CPPUNIT_TEST_NAME} ${CMAKE_CURRENT_BINARY_DIR}/RegistryNames.txt
+ COMMENT "Running unit tests ${BCL2FASTQ_UNIQUE_PREFIX}${BCL2FASTQ_CPPUNIT_TEST_NAME}")
+add_custom_target(${BCL2FASTQ_UNIQUE_PREFIX}passed ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${BCL2FASTQ_CPPUNIT_TEST_NAME}.passed)
+
+##
+## Copy the data directory from the source tree if available
+##
+
+find_path(${CMAKE_CURRENT_SOURCE_DIR}_DATA_DIR data PATHS ${CMAKE_CURRENT_SOURCE_DIR} NO_DEFAULT_PATH)
+if (${CMAKE_CURRENT_SOURCE_DIR}_DATA_DIR)
+message (STATUS "Adding the data subdirectory for the cppunits under ${BCL2FASTQ_LIB_DIR}")
+ add_subdirectory (data)
+ add_dependencies(${BCL2FASTQ_UNIQUE_PREFIX}passed ${BCL2FASTQ_UNIQUE_PREFIX}data)
+endif (${CMAKE_CURRENT_SOURCE_DIR}_DATA_DIR)
diff --git a/src/cmake/cxxConfigure.cmake b/src/cmake/cxxConfigure.cmake
new file mode 100644
index 0000000..872de24
--- /dev/null
+++ b/src/cmake/cxxConfigure.cmake
@@ -0,0 +1,181 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file cxxConfigure.cmake
+##
+## CMake configuration file for c++ executables.
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include ("${BCL2FASTQ_MACROS_CMAKE}")
+
+
+include(TestBigEndian)
+TEST_BIG_ENDIAN(BCL2FASTQ_IS_BIG_ENDIAN)
+
+include(CheckFunctionExists)
+
+bcl2fastq_find_header_or_die(HAVE_INTTYPES_H inttypes.h)
+bcl2fastq_find_header_or_die(HAVE_MALLOC_H malloc.h)
+bcl2fastq_find_header_or_die(HAVE_MCHECK_H mcheck.h)
+bcl2fastq_find_header_or_die(HAVE_MEMORY_H memory.h)
+bcl2fastq_find_header_or_die(HAVE_SIGNAL_H signal.h)
+bcl2fastq_find_header_or_die(HAVE_STDINT_H stdint.h)
+bcl2fastq_find_header_or_die(HAVE_STDLIB_H stdlib.h)
+bcl2fastq_find_header_or_die(HAVE_STRING_H string.h)
+bcl2fastq_find_header_or_die(HAVE_STRINGS_H strings.h)
+bcl2fastq_find_header_or_die(HAVE_TIME_H time.h)
+bcl2fastq_find_header_or_die(HAVE_UNISTD_H unistd.h)
+bcl2fastq_find_header_or_die(HAVE_SYS_STAT_H sys/stat.h)
+bcl2fastq_find_header_or_die(HAVE_SYS_TYPES_H sys/types.h)
+
+# Math functions that might be missing in some flavors of c++
+set (CMAKE_REQUIRED_LIBRARIES m)
+check_function_exists(floorf HAVE_FLOORF)
+check_function_exists(round HAVE_ROUND)
+check_function_exists(roundf HAVE_ROUNDF)
+check_function_exists(powf HAVE_POWF)
+check_function_exists(erf HAVE_ERF)
+check_function_exists(erf HAVE_ERFF)
+check_function_exists(erfc HAVE_ERFC)
+check_function_exists(erfc HAVE_ERFCF)
+
+# Systems calls
+check_function_exists(sysconf HAVE_SYSCONF)
+check_function_exists(clock HAVE_CLOCK)
+
+
+# optional support for numa
+bcl2fastq_find_library(NUMA numa.h numa)
+if (HAVE_NUMA)
+ message(STATUS "NUMA supported")
+ include_directories(BEFORE SYSTEM ${NUMA_INCLUDE_DIR})
+ set(BCL2FASTQ_ADDITIONAL_LIB ${BCL2FASTQ_ADDITIONAL_LIB} "${NUMA_LIBRARY}")
+else (HAVE_NUMA)
+ message(STATUS "No support for NUMA")
+endif (HAVE_NUMA)
+
+# optional support for gzip compression
+bcl2fastq_find_library(ZLIB zlib.h z)
+if (HAVE_ZLIB)
+ set (BCL2FASTQ_ADDITIONAL_LIB ${BCL2FASTQ_ADDITIONAL_LIB} z)
+ message(STATUS "gzip compression supported")
+else (HAVE_ZLIB)
+ message(FATAL_ERROR "No support for gzip compression")
+endif (HAVE_ZLIB)
+
+# optional support for librt
+bcl2fastq_find_library(LIBRT time.h rt)
+if (HAVE_LIBRT)
+ set (BCL2FASTQ_ADDITIONAL_LIB ${BCL2FASTQ_ADDITIONAL_LIB} rt)
+ message(STATUS "librt supported")
+else (HAVE_LIBRT)
+ message(FATAL_ERROR "No support for librt")
+endif (HAVE_LIBRT)
+
+
+bcl2fastq_find_boost(${BCL2FASTQ_BOOST_VERSION} "${BCL2FASTQ_BOOST_COMPONENTS}")
+
+bcl2fastq_find_library(CPGPLOT cpgplot.h cpgplot)
+bcl2fastq_find_library(PGPLOT cpgplot.h pgplot)
+bcl2fastq_find_library(X11 X.h X11)
+
+set(REINSTDIR ${CMAKE_BINARY_DIR}/bootstrap)
+
+# CPPUNIT
+bcl2fastq_find_library(CPPUNIT "cppunit/Test.h" cppunit${CPPUNIT_DEBUG})
+
+# XML2 - bootstrap first (if necessary) so xslt can build against it
+# XSLT and EXSLT
+if((NOT HAVE_LIBXML2) OR (NOT HAVE_LIBXSLT))
+ find_package_version(LibXml2 ${BCL2FASTQ_LIBXML2_VERSION})
+ find_package_version(LibXslt ${BCL2FASTQ_LIBXSLT_VERSION})
+endif((NOT HAVE_LIBXML2) OR (NOT HAVE_LIBXSLT))
+
+if((NOT HAVE_LIBXML2) OR (NOT HAVE_LIBXSLT))
+ redist_package(LIBXML2 ${BCL2FASTQ_LIBXML2_VERSION}
+ "--prefix=${REINSTDIR};--without-modules;--without-http;--without-ftp;--without-python;--without-threads;--without-schematron;--without-debug;--without-iconv")
+ find_library_redist(LIBXML2 ${REINSTDIR} libxml/xpath.h xml2)
+ redist_package(LIBXSLT ${BCL2FASTQ_LIBXSLT_VERSION} "--prefix=${REINSTDIR};--with-libxml-prefix=${REINSTDIR};--without-plugins;--without-crypto")
+ find_library_redist(LIBEXSLT ${REINSTDIR} libexslt/exslt.h exslt)
+ find_library_redist(LIBXSLT ${REINSTDIR} libxslt/xsltconfig.h xslt)
+endif((NOT HAVE_LIBXML2) OR (NOT HAVE_LIBXSLT))
+
+include_directories(BEFORE SYSTEM ${LIBXML2_INCLUDE_DIR})
+include_directories(BEFORE SYSTEM ${LIBXSLT_INCLUDE_DIR})
+include_directories(BEFORE SYSTEM ${LIBEXSLT_INCLUDE_DIR})
+set(BCL2FASTQ_DEP_LIB ${BCL2FASTQ_DEP_LIB} "${LIBEXSLT_LIBRARIES}" "${LIBXSLT_LIBRARIES}" "${LIBXML2_LIBRARIES}")
+
+#set (CMAKE_CXX_FLAGS "$ENV{CXX_FLAGS} $ENV{CXXFLAGS} -fopenmp -msse2 -Werror -Wall -Wextra -Wunused -Wno-long-long -Wsign-compare -Wpointer-arith" CACHE STRING "g++ flags" FORCE)
+set (CMAKE_CXX_FLAGS "$ENV{CXX_FLAGS} $ENV{CXXFLAGS} -std=c++11 -fopenmp -msse2 -Wall -Wextra -Wunused -Wno-long-long -Wsign-compare -Wpointer-arith" CACHE STRING "g++ flags" FORCE)
+#set (CMAKE_CXX_FLAGS_DEBUG "-O0 -g -pg -fprofile-arcs -ftest-coverage -D_GLIBCXX_DEBUG" CACHE STRING "g++ flags" FORCE)
+set (CMAKE_CXX_FLAGS_DEBUG "-O0 -std=c++11 -g -pg -fprofile-arcs -ftest-coverage" CACHE STRING "g++ flags" FORCE)
+#set (CMAKE_CXX_FLAGS_DEBUG "-std=c++11 -O0 -g -pg -fprofile-arcs -ftest-coverage" CACHE STRING "g++ flags" FORCE)
+set (CMAKE_CXX_FLAGS_RELEASE "-O3 -std=c++11 -DNDEBUG" CACHE STRING "g++ flags" FORCE)
+#set (CMAKE_CXX_FLAGS_RELEASE "-std=c++11 -O3 -DNDEBUG" CACHE STRING "g++ flags" FORCE)
+set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -std=c++11 -g" CACHE STRING "g++ flags" FORCE)
+#set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g -pg" CACHE STRING "g++ flags" FORCE)
+#set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "-std=c++11 -O3 -g" CACHE STRING "g++ flags" FORCE)
+set (CMAKE_CXX_FLAGS_MINSIZEREL "-Os -std=c++11 -DNDEBUG" CACHE STRING "g++ flags" FORCE)
+#set (CMAKE_CXX_FLAGS_MINSIZEREL "-std=c++11 -Os -DNDEBUG" CACHE STRING "g++ flags" FORCE)
+
+# Force static linking
+set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "")
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+ execute_process(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE version)
+ string(STRIP ${version} version)
+
+ string(REGEX REPLACE "^([0-9])\\.[0-9]\\.[0-9]" "\\1" major_version ${version})
+ string(REGEX REPLACE "^[0-9]\\.([0-9])\\.[0-9]" "\\1" minor_version ${version})
+ string(REGEX REPLACE "^[0-9]\\.[0-9]\\.([0-9])" "\\1" patch_version ${version})
+ if (major_version LESS 4 OR (major_version EQUAL 4 AND (minor_version LESS 1 OR (minor_version EQUAL 1 AND patch_version LESS 2) ) ) )
+ message (FATAL_ERROR "Unsupported GNU C++ compiler: g++ version ${version}: "
+ "only g++ versions >= 4.1.2 are supported")
+ endif (major_version LESS 4 OR (major_version EQUAL 4 AND (minor_version LESS 1 OR (minor_version EQUAL 1 AND patch_version LESS 2) ) ) )
+
+ set("${CMAKE_CXX_COMPILER_ID}${major_version}" true)
+ set("${CMAKE_CXX_COMPILER_ID}${major_version}${minor_version}" true)
+ set("${CMAKE_CXX_COMPILER_ID}${major_version}${minor_version}${patch_version}" true)
+ message (STATUS "using compiler: gcc version ${version}")
+
+endif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+
+##
+## Suppress spurious warnings in less recent compilers
+##
+if (NOT GNU42)
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter ")
+endif (NOT GNU42)
+
+if (GNU412 OR GNU42 OR GNU43)
+ ## Before 4.1.2, pedantic breaks on boost lambda expressions
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic ")
+endif (GNU412 OR GNU42 OR GNU43)
+
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "^i[67]86$")
+ ##
+ ## Use scalar floating point instructions from the SSE instruction set.
+ ## Note: Pentium3 SSE supports only single precision arithmetics
+ ##
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse -mfpmath=sse")
+endif (CMAKE_SYSTEM_PROCESSOR MATCHES "^i[67]86$")
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "^i[345]86$")
+ ##
+ ## Prevent using 80bits registers (more consistent rounding)
+ ##
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffloat-store")
+endif (CMAKE_SYSTEM_PROCESSOR MATCHES "^i[345]86$")
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/lib/common/config.h.in ${BCL2FASTQ_CXX_CONFIG_H_DIR}/config.h)
diff --git a/src/cmake/cxxExecutable.cmake b/src/cmake/cxxExecutable.cmake
new file mode 100644
index 0000000..cfd2ecf
--- /dev/null
+++ b/src/cmake/cxxExecutable.cmake
@@ -0,0 +1,48 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file cxxExecutable.cmake
+##
+## CMake configuration file for all the c++ executables.
+##
+## author Roman Petrovski
+##
+################################################################################
+
+
+include (${BCL2FASTQ_GLOBALS_CMAKE})
+
+# Support for static linking. Notice this is done here and not in cxxConfigure to
+# allow dynamic linking for cppunit tests as some platforms lack static libcppunit.
+# Note that this implies that all libraries must be found with the
+# exact file name (libXXX.a or libXXX.so)
+if (BCL2FASTQ_FORCE_STATIC_LINK)
+ message(STATUS "All libraries will be statically linked")
+ set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "-static")
+ set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "-static")
+ # ensure that even if cmake decides to allow for dynamic libs resolution,
+ # this gets overriden into static...
+ set(CMAKE_EXE_LINK_DYNAMIC_CXX_FLAGS ${CMAKE_EXE_LINK_STATIC_CXX_FLAGS})
+ set(BCL2FASTQ_LIBRARY_PREFIX ${CMAKE_STATIC_LIBRARY_PREFIX})
+ set(BCL2FASTQ_LIBRARY_SUFFIX ${CMAKE_STATIC_LIBRARY_SUFFIX})
+else (BCL2FASTQ_FORCE_STATIC_LINK)
+ set(BCL2FASTQ_LIBRARY_PREFIX "")
+ set(BCL2FASTQ_LIBRARY_SUFFIX "")
+endif (BCL2FASTQ_FORCE_STATIC_LINK)
+
+get_filename_component(BCL2FASTQ_CURRENT_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+message (STATUS "Adding the cpp program subdirectory: ${BCL2FASTQ_CURRENT_DIR_NAME}")
+include_directories (${BCL2FASTQ_CXX_ALL_INCLUDES})
+include_directories (${CMAKE_CURRENT_BINARY_DIR})
+include_directories (${CMAKE_CURRENT_SOURCE_DIR})
+include_directories (${BCL2FASTQ_CXX_CONFIG_H_DIR})
+
+
diff --git a/src/cmake/cxxLibrary.cmake b/src/cmake/cxxLibrary.cmake
new file mode 100644
index 0000000..d377a63
--- /dev/null
+++ b/src/cmake/cxxLibrary.cmake
@@ -0,0 +1,73 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file cxxLibrary.cmake
+##
+## CMake configuration file for all the c++ libraries
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include_directories (${BCL2FASTQ_CXX_ALL_INCLUDES})
+include_directories (${CMAKE_CURRENT_BINARY_DIR})
+include_directories (${CMAKE_CURRENT_SOURCE_DIR})
+include_directories (${BCL2FASTQ_CXX_CONFIG_H_DIR})
+
+get_filename_component(BCL2FASTQ_CURRENT_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+message (STATUS "Adding the cpp library subdirectory: ${BCL2FASTQ_CURRENT_DIR_NAME}")
+
+##
+## Some generators (VS) require all targets to be unique across the project.
+## Therefore, a unique prefix is needed to create the target names which are
+## shared across libraries
+##
+
+string(REGEX REPLACE ${CMAKE_SOURCE_DIR}/cpp/ "" TMP1 ${CMAKE_CURRENT_SOURCE_DIR}/)
+string(REGEX REPLACE "/" "_" BCL2FASTQ_UNIQUE_PREFIX ${TMP1})
+
+##
+## build the library
+##
+file(GLOB_RECURSE BCL2FASTQ_LIBRARY_SOURCES_WITH_CPPUNIT *.cpp *.c)
+
+foreach (SOURCE_FILE ${BCL2FASTQ_LIBRARY_SOURCES_WITH_CPPUNIT})
+ string(REGEX MATCH "cppunit" CPPUNIT_MATCH ${SOURCE_FILE} )
+ if (NOT CPPUNIT_MATCH)
+ set(BCL2FASTQ_LIBRARY_SOURCES ${BCL2FASTQ_LIBRARY_SOURCES} ${SOURCE_FILE})
+ endif (NOT CPPUNIT_MATCH)
+endforeach (SOURCE_FILE)
+
+foreach (SOURCE_FILE ${BCL2FASTQ_LIBRARY_SOURCES})
+ get_filename_component(SOURCE_NAME ${SOURCE_FILE} NAME_WE)
+ if (${SOURCE_NAME}_COMPILE_FLAGS)
+ set_source_files_properties(${SOURCE_FILE} PROPERTIES COMPILE_FLAGS ${${SOURCE_NAME}_COMPILE_FLAGS})
+ endif (${SOURCE_NAME}_COMPILE_FLAGS)
+endforeach (SOURCE_FILE)
+
+#include_directories (${BCL2FASTQ_COMMON_INCLUDE} )
+add_library (bcl2fastq_${BCL2FASTQ_LIB_DIR} STATIC ${BCL2FASTQ_LIBRARY_SOURCES})
+add_dependencies(bcl2fastq_${BCL2FASTQ_LIB_DIR} BCL2FASTQ_OPT)
+
+##
+## build the unit tests if any (this should be mandatory really)
+##
+
+if (HAVE_CPPUNIT AND BCL2FASTQ_UNIT_TESTS)
+ find_path(${CMAKE_CURRENT_SOURCE_DIR}_CPPUNIT_DIR cppunit PATHS ${CMAKE_CURRENT_SOURCE_DIR} NO_DEFAULT_PATH)
+ if (${CMAKE_CURRENT_SOURCE_DIR}_CPPUNIT_DIR)
+ message (STATUS "Adding the cppunit subdirectory for ${BCL2FASTQ_LIB_DIR}")
+ add_subdirectory (cppunit)
+ endif (${CMAKE_CURRENT_SOURCE_DIR}_CPPUNIT_DIR)
+endif(HAVE_CPPUNIT AND BCL2FASTQ_UNIT_TESTS)
+
+
diff --git a/src/cmake/globals.cmake b/src/cmake/globals.cmake
new file mode 100644
index 0000000..3c85206
--- /dev/null
+++ b/src/cmake/globals.cmake
@@ -0,0 +1,64 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file globals.cmake
+##
+## CMake configuration file to identify the configuration of the system.
+##
+## author Roman Petrovski
+##
+################################################################################
+
+set(BCL2FASTQ_ORIG_ETCDIR "${CMAKE_INSTALL_PREFIX}/${BCL2FASTQ_ETCDIR}")
+set(BCL2FASTQ_ORIG_DATADIR "${CMAKE_INSTALL_PREFIX}/${BCL2FASTQ_DATADIR}")
+set(BCL2FASTQ_ORIG_BINDIR "${CMAKE_INSTALL_PREFIX}/${BCL2FASTQ_BINDIR}")
+set(BCL2FASTQ_ORIG_LIBDIR "${CMAKE_INSTALL_PREFIX}/${BCL2FASTQ_LIBDIR}")
+set(BCL2FASTQ_ORIG_LIBEXECDIR "${CMAKE_INSTALL_PREFIX}/${BCL2FASTQ_LIBEXECDIR}")
+
+get_filename_component(BCL2FASTQ_FULL_ETCDIR "${BCL2FASTQ_ORIG_ETCDIR}" ABSOLUTE)
+get_filename_component(BCL2FASTQ_FULL_DATADIR "${BCL2FASTQ_ORIG_DATADIR}" ABSOLUTE)
+get_filename_component(BCL2FASTQ_FULL_BINDIR "${BCL2FASTQ_ORIG_BINDIR}" ABSOLUTE)
+get_filename_component(BCL2FASTQ_FULL_LIBDIR "${BCL2FASTQ_ORIG_LIBDIR}" ABSOLUTE)
+get_filename_component(BCL2FASTQ_FULL_LIBEXECDIR "${BCL2FASTQ_ORIG_LIBEXECDIR}" ABSOLUTE)
+
+set(BCL2FASTQ_PARTIAL_ETCDIR "${BCL2FASTQ_ETCDIR}")
+set(BCL2FASTQ_PARTIAL_DATADIR "${BCL2FASTQ_DATADIR}")
+set(BCL2FASTQ_PARTIAL_BINDIR "${BCL2FASTQ_BINDIR}")
+set(BCL2FASTQ_PARTIAL_LIBDIR "${BCL2FASTQ_LIBDIR}")
+set(BCL2FASTQ_PARTIAL_LIBEXECDIR "${BCL2FASTQ_LIBEXECDIR}")
+
+install(CODE "
+
+ # _DEST_ variables always point to location where files are copied
+ get_filename_component(BCL2FASTQ_DEST_ETCDIR \"\$ENV{DESTDIR}${BCL2FASTQ_ORIG_ETCDIR}\" ABSOLUTE)
+ get_filename_component(BCL2FASTQ_DEST_DATADIR \"\$ENV{DESTDIR}${BCL2FASTQ_ORIG_DATADIR}\" ABSOLUTE)
+ get_filename_component(BCL2FASTQ_DEST_BINDIR \"\$ENV{DESTDIR}${BCL2FASTQ_ORIG_BINDIR}\" ABSOLUTE)
+ get_filename_component(BCL2FASTQ_DEST_LIBDIR \"\$ENV{DESTDIR}${BCL2FASTQ_ORIG_LIBDIR}\" ABSOLUTE)
+ get_filename_component(BCL2FASTQ_DEST_LIBEXECDIR \"\$ENV{DESTDIR}${BCL2FASTQ_ORIG_LIBEXECDIR}\" ABSOLUTE)
+
+ set(BCL2FASTQ_FULL_ETCDIR \"${BCL2FASTQ_FULL_ETCDIR}\")
+ set(BCL2FASTQ_FULL_DATADIR \"${BCL2FASTQ_FULL_DATADIR}\")
+ set(BCL2FASTQ_FULL_BINDIR \"${BCL2FASTQ_FULL_BINDIR}\")
+ set(BCL2FASTQ_FULL_LIBDIR \"${BCL2FASTQ_FULL_LIBDIR}\")
+ set(BCL2FASTQ_FULL_LIBEXECDIR \"${BCL2FASTQ_FULL_LIBEXECDIR}\")
+
+ set(BCL2FASTQ_PARTIAL_ETCDIR \"${BCL2FASTQ_PARTIAL_ETCDIR}\")
+ set(BCL2FASTQ_PARTIAL_DATADIR \"${BCL2FASTQ_PARTIAL_DATADIR}\")
+ set(BCL2FASTQ_PARTIAL_BINDIR \"${BCL2FASTQ_PARTIAL_BINDIR}\")
+ set(BCL2FASTQ_PARTIAL_LIBDIR \"${BCL2FASTQ_PARTIAL_LIBDIR}\")
+ set(BCL2FASTQ_PARTIAL_LIBEXECDIR \"${BCL2FASTQ_PARTIAL_LIBEXECDIR}\")
+
+ set(BCL2FASTQ_VERSION_FULL \"${BCL2FASTQ_VERSION_FULL}\")
+ set(BCL2FASTQ_EXECUTABLE_PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE)
+ set(BCL2FASTQ_LIBRARY_PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ)
+ ")
+
+
diff --git a/src/cmake/macros.cmake b/src/cmake/macros.cmake
new file mode 100644
index 0000000..a6f656b
--- /dev/null
+++ b/src/cmake/macros.cmake
@@ -0,0 +1,84 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file macros.cmake
+##
+## CMake configuration file for common installation macros.
+##
+## author Roman Petrovski
+## author Mauricio Varea
+##
+################################################################################
+
+macro(configure_files srcDir destDir pattern)
+ file(GLOB templateFiles RELATIVE ${srcDir} ${srcDir}/${pattern})
+ foreach(templateFile ${templateFiles})
+ message(STATUS "Configuring file ${srcDir}/${templateFile}")
+ configure_file(${srcDir}/${templateFile} ${destDir}/${templateFile} @ONLY IMMEDIATE)
+ endforeach(templateFile)
+endmacro(configure_files)
+
+macro(install_files srcDir destDir pattern perm)
+ file(GLOB templateFiles ${srcDir}/${pattern})
+ file(INSTALL DESTINATION ${destDir} TYPE FILE
+ FILES ${templateFiles} PERMISSIONS ${perm})
+endmacro(install_files)
+
+macro(configure_files_recursively srcDir destDir pattern)
+ file(GLOB_RECURSE templateFiles RELATIVE ${srcDir} ${srcDir}/${pattern})
+ foreach(templateFile ${templateFiles})
+ message(STATUS "Configuring file ${srcDir}/${templateFile}")
+ configure_file(${srcDir}/${templateFile} ${destDir}/${templateFile} @ONLY IMMEDIATE)
+ endforeach(templateFile)
+endmacro(configure_files_recursively)
+
+macro(install_files_recursively srcDir destDir pattern perm)
+ file(GLOB_RECURSE templateFiles RELATIVE ${srcDir} ${srcDir}/${pattern})
+ foreach(templateFile ${templateFiles})
+ get_filename_component(DIRNAME "${templateFile}" PATH)
+ file(INSTALL DESTINATION ${destDir}/${DIRNAME} TYPE FILE
+ FILES ${srcDir}/${templateFile} PERMISSIONS ${perm})
+ endforeach(templateFile)
+endmacro(install_files_recursively)
+
+#
+# Macro to find libraries, with support for static-only search
+#
+macro(bcl2fastq_find_library name header library)
+if (NOT ${name}_INCLUDE_DIR)
+ find_path(${name}_INCLUDE_DIR ${header}
+ HINTS ENV C_INCLUDE_PATH ENV CPATH ENV CPLUS_INCLUDE_PATH)
+endif (NOT ${name}_INCLUDE_DIR)
+if (${name}_INCLUDE_DIR AND NOT ${name}_LIBRARY)
+ find_library(${name}_LIBRARY
+ NAMES "${BCL2FASTQ_LIBRARY_PREFIX}${library}${BCL2FASTQ_LIBRARY_SUFFIX}"
+ HINTS ENV LIBRARY_PATH)
+endif (${name}_INCLUDE_DIR AND NOT ${name}_LIBRARY)
+if(${name}_INCLUDE_DIR AND ${name}_LIBRARY)
+ set (HAVE_${name} ${${name}_LIBRARY})
+ message (STATUS "Found ${name} header: ${${name}_INCLUDE_DIR}/${header}")
+ message (STATUS "Found ${name} library: ${${name}_LIBRARY}")
+endif(${name}_INCLUDE_DIR AND ${name}_LIBRARY)
+endmacro(bcl2fastq_find_library)
+
+#
+# Macro to find libraries, with support for static-only search
+#
+macro(bcl2fastq_find_header_or_die variable file)
+find_file(${variable} ${file} HINTS ENV C_INCLUDE_PATH ENV CPATH ENV CPLUS_INCLUDE_PATH)
+if (${variable})
+ message(STATUS "${file} found as ${${variable}}")
+else (${variable})
+ message(FATAL_ERROR "Required header ${file} not found.")
+endif (${variable})
+endmacro(bcl2fastq_find_header_or_die)
+
+
diff --git a/src/cmake/modules/FindMarkdown.cmake b/src/cmake/modules/FindMarkdown.cmake
new file mode 100644
index 0000000..b982236
--- /dev/null
+++ b/src/cmake/modules/FindMarkdown.cmake
@@ -0,0 +1,58 @@
+# - try to find Markdown tool
+#
+# Cache Variables:
+# MARKDOWN_EXECUTABLE
+#
+# Non-cache variables you might use in your CMakeLists.txt:
+# MARKDOWN_FOUND
+#
+# Requires these CMake modules:
+# FindPackageHandleStandardArgs (known included with CMake >=2.6.2)
+#
+# Original Author:
+# 2011 Ryan Pavlik <rpavlik at iastate.edu> <abiryan at ryand.net>
+# http://academic.cleardefinition.com
+# Iowa State University HCI Graduate Program/VRAC
+#
+# Copyright Iowa State University 2011.
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# http://www.boost.org/LICENSE_1_0.txt)
+
+file(TO_CMAKE_PATH "${MARKDOWN_ROOT_DIR}" MARKDOWN_ROOT_DIR)
+set(MARKDOWN_ROOT_DIR
+"${MARKDOWN_ROOT_DIR}"
+CACHE
+PATH
+"Path to search for Markdown")
+
+if(MARKDOWN_EXECUTABLE AND NOT EXISTS "${MARKDOWN_EXECUTABLE}")
+set(MARKDOWN_EXECUTABLE "notfound" CACHE PATH FORCE "")
+endif()
+
+# If we have a custom path, look there first.
+if(MARKDOWN_ROOT_DIR)
+find_program(MARKDOWN_EXECUTABLE
+NAMES
+markdown
+PATHS
+"${MARKDOWN_ROOT_DIR}"
+PATH_SUFFIXES
+bin
+NO_DEFAULT_PATH)
+endif()
+
+find_program(MARKDOWN_EXECUTABLE NAMES markdown)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Markdown
+DEFAULT_MSG
+MARKDOWN_EXECUTABLE)
+
+if(MARKDOWN_FOUND)
+mark_as_advanced(MARKDOWN_ROOT_DIR)
+endif()
+
+mark_as_advanced(MARKDOWN_EXECUTABLE)
+
+
diff --git a/src/cmake/modules/UseMarkdown.cmake b/src/cmake/modules/UseMarkdown.cmake
new file mode 100644
index 0000000..b92d386
--- /dev/null
+++ b/src/cmake/modules/UseMarkdown.cmake
@@ -0,0 +1,118 @@
+# - Convert markdown source files to HTML as a custom target
+#
+# include(UseMarkdown)
+# add_markdown_target(<target_name> <directory to copy to> <markdownfile> [<markdownfile>...] [RENAME <newname>])
+# Relative paths for the destination directory are considered with
+# with respect to CMAKE_CURRENT_BINARY_DIR. The RENAME argument is only
+# valid with a single markdown file as input.
+#
+#
+# install_markdown_target(<target_name> [extra arguments to INSTALL(FILES ...) ])
+#
+#
+# Requires CMake 2.6 or newer (uses the 'function' command)
+#
+# Original Author:
+# 2011 Ryan Pavlik <rpavlik at iastate.edu> <abiryan at ryand.net>
+# http://academic.cleardefinition.com
+# Iowa State University HCI Graduate Program/VRAC
+#
+# Copyright Iowa State University 2011-2012.
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# http://www.boost.org/LICENSE_1_0.txt)
+
+if(__add_markdown_target)
+return()
+endif()
+set(__add_markdown_target YES)
+
+define_property(TARGET
+PROPERTY
+MARKDOWN_TARGET_OUTPUTS
+BRIEF_DOCS
+"Markdown target outputs"
+FULL_DOCS
+"Output files of a target created by add_markdown_target")
+
+function(add_markdown_target _target _dest)
+
+if(NOT ARGN)
+message(WARNING
+"In add_markdown_target call for target ${_target}, no source files were specified!")
+return()
+endif()
+
+find_package(Markdown QUIET)
+if(NOT MARKDOWN_EXECUTABLE)
+message(FATAL_ERROR "Can't find a markdown conversion tool!")
+endif()
+
+set(NEW_NAME)
+list(FIND ARGN "RENAME" _renameloc)
+if(_renameloc GREATER -1)
+list(LENGTH ARGN _len)
+if(NOT _len EQUAL 3)
+message(FATAL_ERROR
+"Specifying RENAME requires 1 input file and 1 output name!")
+endif()
+list(GET ARGN 2 NEW_NAME)
+list(GET ARGN 0 ARGN)
+endif()
+
+set(ALLFILES)
+set(SOURCES)
+foreach(fn ${ARGN})
+# Produce an absolute path to the input file
+if(IS_ABSOLUTE "${fn}")
+get_filename_component(fullpath "${fn}" ABSOLUTE)
+get_filename_component(fn "${fn}" NAME)
+else()
+get_filename_component(fullpath
+"${CMAKE_CURRENT_SOURCE_DIR}/${fn}"
+ABSOLUTE)
+endif()
+get_filename_component(fn_noext "${fn}" NAME_WE)
+
+# Clean up output file name
+if(NEW_NAME)
+get_filename_component(absout "${_dest}/${NEW_NAME}" ABSOLUTE)
+else()
+get_filename_component(absout "${_dest}/${fn_noext}.html" ABSOLUTE)
+endif()
+
+add_custom_command(OUTPUT "${absout}"
+COMMAND
+${CMAKE_COMMAND}
+ARGS -E make_directory "${_dest}"
+COMMAND
+${MARKDOWN_EXECUTABLE}
+ARGS ${MARKDOWN_PARAMS} "${fullpath}" > "${absout}"
+MAIN_DEPENDENCY "${fullpath}"
+VERBATIM
+COMMENT "Converting Markdown ${fn} to HTML in ${absout}...")
+list(APPEND SOURCES "${fullpath}")
+list(APPEND ALLFILES "${absout}")
+endforeach()
+
+# Custom target depending on all the file copy commands
+add_custom_target(${_target}
+ALL
+SOURCES ${SOURCES}
+DEPENDS ${ALLFILES})
+set_property(TARGET ${_target} PROPERTY MARKDOWN_TARGET_OUTPUTS "${ALLFILES}")
+endfunction()
+
+function(install_markdown_target _target)
+get_target_property(_mtoutputs ${_target} MARKDOWN_TARGET_OUTPUTS)
+if(NOT _mtoutputs)
+message(WARNING
+"install_markdown_target called on a target not created with add_markdown_target!")
+return()
+endif()
+
+# Forward the call to install
+install(FILES ${_mtoutputs} ${ARGN})
+endfunction()
+
+
diff --git a/src/cmake/postInstall/CMakeLists.txt b/src/cmake/postInstall/CMakeLists.txt
new file mode 100644
index 0000000..5ec7ba5
--- /dev/null
+++ b/src/cmake/postInstall/CMakeLists.txt
@@ -0,0 +1,20 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the postInstall subdirectory.
+##
+## author Roman Petrovski
+##
+################################################################################
+
+
diff --git a/src/cmake/preInstall/CMakeLists.txt b/src/cmake/preInstall/CMakeLists.txt
new file mode 100644
index 0000000..a7cb91d
--- /dev/null
+++ b/src/cmake/preInstall/CMakeLists.txt
@@ -0,0 +1,23 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the preInstall subdirectory.
+##
+## author Roman Petrovski
+##
+################################################################################
+
+add_subdirectory (checkTargetPathsWritable)
+add_subdirectory (copyrightAndChanges)
+
+
diff --git a/src/cmake/preInstall/checkTargetPathsWritable/CMakeLists.txt b/src/cmake/preInstall/checkTargetPathsWritable/CMakeLists.txt
new file mode 100644
index 0000000..5816bc7
--- /dev/null
+++ b/src/cmake/preInstall/checkTargetPathsWritable/CMakeLists.txt
@@ -0,0 +1,31 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the checkTargetPathsWritable subdirectory.
+##
+## author Roman Petrovski
+##
+################################################################################
+
+
+message (STATUS "Verifying target directories access")
+
+include ("${BCL2FASTQ_GLOBALS_CMAKE}")
+install(
+ CODE "set(BCL2FASTQ_DEST_DIRS \"\${BCL2FASTQ_DEST_ETCDIR}\" \"\${BCL2FASTQ_DEST_DATADIR}\"
+ \"\${BCL2FASTQ_DEST_BINDIR}\" \"\${BCL2FASTQ_DEST_LIBDIR}\"
+ \"\${BCL2FASTQ_DEST_LIBEXECDIR}\")"
+ SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/checkTargetPathWritable.cmake"
+)
+
+
diff --git a/src/cmake/preInstall/checkTargetPathsWritable/checkTargetPathWritable.cmake b/src/cmake/preInstall/checkTargetPathsWritable/checkTargetPathWritable.cmake
new file mode 100644
index 0000000..72e0a75
--- /dev/null
+++ b/src/cmake/preInstall/checkTargetPathsWritable/checkTargetPathWritable.cmake
@@ -0,0 +1,37 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file checkTargetPathWritable.cmake
+##
+## Script checking writable state of installation path.
+##
+## author Roman Petrovski
+##
+################################################################################
+
+
+foreach (BCL2FASTQ_DEST_DIR ${BCL2FASTQ_DEST_DIRS})
+ message (STATUS "Testing access to ${BCL2FASTQ_DEST_DIR}...")
+ execute_process(COMMAND "${CMAKE_COMMAND}" "-E" "make_directory" "${BCL2FASTQ_DEST_DIR}/test" RESULT_VARIABLE TMP_RESULT )
+ if (TMP_RESULT)
+ message (STATUS "ERROR: Directory is not writeable: ${BCL2FASTQ_DEST_DIR}")
+ message (STATUS "If you don't have administrator access to the "
+ "target installation location, please use --prefix "
+ "command-line option when configuring BCL2FASTQ. "
+ "Please use configure --help for all installer "
+ "command-line options details.")
+ message (FATAL_ERROR "ERROR: BCL2FASTQ installation cannot continue")
+ else (TMP_RESULT)
+ message (STATUS "Directory is writeable: ${BCL2FASTQ_DEST_DIR}")
+ endif (TMP_RESULT)
+endforeach (BCL2FASTQ_DEST_DIR ${BCL2FASTQ_DEST_DIRS})
+
+
diff --git a/src/cmake/preInstall/copyrightAndChanges/CMakeLists.txt b/src/cmake/preInstall/copyrightAndChanges/CMakeLists.txt
new file mode 100644
index 0000000..9f8e0a2
--- /dev/null
+++ b/src/cmake/preInstall/copyrightAndChanges/CMakeLists.txt
@@ -0,0 +1,24 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the COPYRIGHT and Changes file installation.
+##
+## author Roman Petrovski
+##
+################################################################################
+
+
+# Installing top level components
+install(FILES "${BCL2FASTQ_SOURCE_DIR}/Changes" "${CMAKE_SOURCE_DIR}/../COPYRIGHT" DESTINATION "${BCL2FASTQ_DATADIR}")
+
+
diff --git a/src/cmake/zlib-ng.cmake b/src/cmake/zlib-ng.cmake
new file mode 100644
index 0000000..feeb6c6
--- /dev/null
+++ b/src/cmake/zlib-ng.cmake
@@ -0,0 +1,30 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2016 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for zlib-ng installation
+##
+## author Aaron Day
+##
+################################################################################
+
+set (ENV{CFLAGS} "-mavx -O3 -I${CMAKE_BINARY_DIR}/bootstrap/include")
+set(REINSTDIR ${CMAKE_BINARY_DIR}/bootstrap)
+
+set(CMAKE_C_FLAGS_CPY "${CMAKE_C_FLAGS}")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx -O3")
+redist_package(ZLIB-NG ${BCL2FASTQ_ZLIB-NG_VERSION}
+ "--zlib-compat;--prefix=${REINSTDIR}")
+set(CMAKE_C_FLAGS ${CMAKE_C_FLAGS_CPY})
+
+find_library_redist(ZLIB-NG ${REINSTDIR} zlib.h z)
+include_directories(BEFORE SYSTEM ${ZLIB_INCLUDE_DIR})
diff --git a/src/configure b/src/configure
new file mode 100755
index 0000000..6f5da7e
--- /dev/null
+++ b/src/configure
@@ -0,0 +1,431 @@
+#!/bin/bash
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file configure
+##
+## Top level configuration file.
+##
+## author Come Raczy
+##
+################################################################################
+
+
+# set -x
+# set -e
+set -o pipefail # bash only
+shopt -s compat31 2>/dev/null
+
+
+# Version information
+bcl2fastq_name_short="bcl2fastq"
+bcl2fastq_name_long="BCL to FASTQ file converter"
+bcl2fastq_copyright="Copyright (c) 2007-2017 Illumina, Inc."
+bcl2fastq_version_major="2"
+bcl2fastq_version_minor="19"
+bcl2fastq_version_patch="1"
+bcl2fastq_version_build="403"
+bcl2fastq_version="${bcl2fastq_version_major}.${bcl2fastq_version_minor}.${bcl2fastq_version_patch}.${bcl2fastq_version_build}"
+bcl2fastq_version_full="${bcl2fastq_name_long} (${bcl2fastq_name_short}-${bcl2fastq_version})"
+
+# Display BCL2FASTQ configure usage
+bcl2fastq_usage()
+{
+ cat <<EOF
+Usage: $0 [options]
+Options: [defaults in brackets after descriptions]
+
+Configuration:
+
+ --terse display less information
+ (disables CMAKE_VERBOSE_MAKEFILE)
+
+ --verbose display more information
+ (enables CMAKE_VERBOSE_MAKEFILE)
+
+ --version only print version information
+
+ --help print this message
+
+ --build-type specify the build type for CMake (affects
+ compiler options). Allowed values are "",
+ "Debug", "Release", "RelWithDebInfo",
+ and "MinSizeRel"
+ [RelWithDebInfo]
+
+ --static forces library static linking
+
+ --package-type=type enables generation of deployment package target
+ (make package). Valid types are: rpm, deb, tgz
+
+ --parallel=n build cmake and boost in parallel if needed,
+ where n is the number of nodes
+ [1]
+
+ --with-cmake=CMAKE specify the cmake executable
+ [cmake]
+
+ --force-builddir allows build directory to be the same as source
+ directory
+
+ --with-unit-tests allow unit testing during the build
+
+ --without-unit-tests prevent unit testing during the build (default)
+
+Directory and file names:
+
+ --prefix=PREFIX install files in tree rooted at PREFIX
+ [${bcl2fastq_default_prefix}]
+
+ --exec-prefix=EPREFIX install binary files in tree rooted at EPREFIX
+ [PREFIX]
+
+ --bindir=DIR install executable in DIR
+ [EPREFIX/bin]
+
+ --libdir=DIR install library files in DIR
+ [EPREFIX/lib/${bcl2fastq_name_short}-${bcl2fastq_version}]
+
+ --libexecdir=DIR install library programs in DIR
+ [EPREFIX/libexec/${bcl2fastq_name_short}-${bcl2fastq_version}]
+
+ --includedir=DIR install include files in DIR
+ [PREFIX/include/${bcl2fastq_name_short}-${bcl2fastq_version}]
+
+ --datadir=DATADIR install data files in DIR
+ [PREFIX/share/${bcl2fastq_name_short}-${bcl2fastq_version}]
+
+ --docdir=DIR install documentation in DIR
+ [DATADIR/doc]
+
+ --mandir=DIR install man pages files in DIR/manN
+ [PREFIX/man]
+
+ --builddir=DIR build bcl2fastq in DIR
+ [./]
+
+Some influential environment variables:
+
+ BOOST_ROOT location of the boost library
+
+ BOOST_INCLUDEDIR location of the include directory of boost
+
+ BOOST_LIBRARYDIR location of the lib directory of boost
+
+ CC C compiler command
+
+ CFLAGS C compiler flags
+
+ LDFLAGS linker flags, e.g. -L<lib dir> if you have
+ libraries in a nonstandard directory <lib dir>
+
+ CXX C++ compiler command
+
+ CXXFLAGS C++ compiler flags
+
+ CMAKE_OPTIONS CMake command line options
+ (for CMAKE_BUILD_TYPE, use --build-type)
+
+
+Use these variables to override the choices made by 'configure' or to help
+it to find libraries and programs with nonstandard names/locations.
+
+EOF
+ exit 10
+}
+
+# Helper function to fix windows paths.
+bcl2fastq_fix_slashes ()
+{
+ echo "$1" | sed 's/\\/\//g'
+}
+
+
+# Use gcc/g++ by default.
+export CC=${CC:="gcc"}
+export CXX=${CXX:="g++"}
+
+# Detect system and directory information.
+bcl2fastq_system="`uname`"
+bcl2fastq_processor="`uname -p`"
+bcl2fastq_source_dir="`echo $0 | sed -n '/\//{s/\/[^\/]*$//;p;}'`"
+bcl2fastq_source_dir="`(cd "${bcl2fastq_source_dir}";pwd)`"
+bcl2fastq_redist_dir="${bcl2fastq_source_dir}/../redist"
+bcl2fastq_bootstrap_dir="${bcl2fastq_source_dir}/cmake/bootstrap"
+bcl2fastq_build_dir="`pwd`"
+
+# Determine whether this is a MinGW environment.
+if echo "${bcl2fastq_system}" | grep MINGW >/dev/null 2>&1; then
+ bcl2fastq_system_mingw=true
+else
+ bcl2fastq_system_mingw=false
+fi
+
+# Determine whether this is OS X
+if echo "${bcl2fastq_system}" | grep Darwin >/dev/null 2>&1; then
+ bcl2fastq_system_darwin=true
+else
+ bcl2fastq_system_darwin=false
+fi
+
+# Choose the default install prefix.
+if ${bcl2fastq_system_mingw}; then
+ if [ "x${PROGRAMFILES}" != "x" ]; then
+ bcl2fastq_default_prefix=`bcl2fastq_fix_slashes "${PROGRAMFILES}/CMake"`
+ elif [ "x${ProgramFiles}" != "x" ]; then
+ bcl2fastq_default_prefix=`bcl2fastq_fix_slashes "${ProgramFiles}/CMake"`
+ elif [ "x${SYSTEMDRIVE}" != "x" ]; then
+ bcl2fastq_default_prefix=`bcl2fastq_fix_slashes "${SYSTEMDRIVE}/Program Files/CMake"`
+ elif [ "x${SystemDrive}" != "x" ]; then
+ bcl2fastq_default_prefix=`bcl2fastq_fix_slashes "${SystemDrive}/Program Files/CMake"`
+ else
+ bcl2fastq_default_prefix="c:/Program Files/CMake"
+ fi
+else
+ bcl2fastq_default_prefix="/usr/local"
+fi
+
+# Parse arguments
+bcl2fastq_build_type=RelWithDebInfo
+bcl2fastq_cmake_generator="Unix Makefiles"
+bcl2fastq_verbose=
+bcl2fastq_parallel=1
+for a in "$@"; do
+ if echo $a | grep "^--terse" > /dev/null 2> /dev/null; then
+ CMAKE_OPTIONS="$CMAKE_OPTIONS -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF -DBoost_DEBUG:BOOL=OFF"
+ bcl2fastq_verbose=
+ fi
+ if echo $a | grep "^--verbose" > /dev/null 2> /dev/null; then
+ CMAKE_OPTIONS="$CMAKE_OPTIONS -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DBoost_DEBUG:BOOL=ON"
+ bcl2fastq_verbose=true
+ fi
+ if echo $a | grep "^--version" > /dev/null 2> /dev/null; then
+ echo "${bcl2fastq_version_full}, ${bcl2fastq_copyright}"
+ exit 2
+ fi
+ if echo $a | grep "^--help" > /dev/null 2> /dev/null; then
+ bcl2fastq_usage
+ fi
+ if echo $a | grep "^--build-type" > /dev/null 2> /dev/null; then
+ bcl2fastq_build_type=`echo $a | sed "s/^--build-type=//"`
+ fi
+ if echo $a | grep "^--static" > /dev/null 2> /dev/null; then
+ bcl2fastq_static=true
+ fi
+ if echo $a | grep "^--package-type=rpm" > /dev/null 2> /dev/null; then
+ bcl2fastq_package=RPM
+ fi
+ if echo $a | grep "^--package-type=deb" > /dev/null 2> /dev/null; then
+ bcl2fastq_package=DEB
+ fi
+ if echo $a | grep "^--package-type=tgz" > /dev/null 2> /dev/null; then
+ bcl2fastq_package=TGZ
+ fi
+ if echo $a | grep "^--parallel" > /dev/null 2> /dev/null; then
+ bcl2fastq_parallel=`echo $a | sed "s/^--parallel=//"`
+ fi
+ if echo $a | grep "^--with-cmake" > /dev/null 2> /dev/null; then
+ bcl2fastq_cmake=`echo $a | sed "s/^--with-cmake=//"`
+ bcl2fastq_cmake=`bcl2fastq_fix_slashes "${bcl2fastq_cmake}"` || exit $?
+ fi
+ if echo $a | grep "^--force-builddir" > /dev/null 2> /dev/null; then
+ bcl2fastq_force_builddir=true
+ fi
+ if echo $a | grep "^--with-unit-tests" > /dev/null 2> /dev/null; then
+ bcl2fastq_unit_tests=true
+ fi
+ if echo $a | grep "^--without-unit-tests" > /dev/null 2> /dev/null; then
+ bcl2fastq_unit_tests=
+ fi
+ if echo $a | grep "^--prefix=" > /dev/null 2> /dev/null; then
+ bcl2fastq_prefix_dir=`echo $a | sed "s/^--prefix=//"`
+ bcl2fastq_prefix_dir=`bcl2fastq_fix_slashes "${bcl2fastq_prefix_dir}"`
+ fi
+ if echo $a | grep "^--exec-prefix=" > /dev/null 2> /dev/null; then
+ bcl2fastq_exec_prefix_dir=`echo $a | sed "s/^--exec-prefix=//"`
+ bcl2fastq_exec_prefix_dir=`bcl2fastq_fix_slashes "${bcl2fastq_exec_prefix_dir}"`
+ fi
+ if echo $a | grep "^--bindir=" > /dev/null 2> /dev/null; then
+ bcl2fastq_bin_dir=`echo $a | sed "s/^--bindir=//"`
+ bcl2fastq_bin_dir=`bcl2fastq_fix_slashes "${bcl2fastq_bin_dir}"`
+ fi
+ if echo $a | grep "^--libdir=" > /dev/null 2> /dev/null; then
+ bcl2fastq_lib_dir=`echo $a | sed "s/^--libdir=//"`
+ bcl2fastq_lib_dir=`bcl2fastq_fix_slashes "${bcl2fastq_lib_dir}"`
+ fi
+ if echo $a | grep "^--libexecdir=" > /dev/null 2> /dev/null; then
+ bcl2fastq_libexec_dir=`echo $a | sed "s/^--libexecdir=//"`
+ bcl2fastq_libexec_dir=`bcl2fastq_fix_slashes "${bcl2fastq_libexec_dir}"`
+ fi
+ if echo $a | grep "^--includedir=" > /dev/null 2> /dev/null; then
+ bcl2fastq_include_dir=`echo $a | sed "s/^--includedir=//"`
+ bcl2fastq_include_dir=`bcl2fastq_fix_slashes "${bcl2fastq_include_dir}"`
+ fi
+ if echo $a | grep "^--datadir=" > /dev/null 2> /dev/null; then
+ bcl2fastq_data_dir=`echo $a | sed "s/^--datadir=//"`
+ bcl2fastq_data_dir=`bcl2fastq_fix_slashes "${bcl2fastq_data_dir}"`
+ fi
+ if echo $a | grep "^--docdir=" > /dev/null 2> /dev/null; then
+ bcl2fastq_doc_dir=`echo $a | sed "s/^--docdir=//"`
+ bcl2fastq_doc_dir=`bcl2fastq_fix_slashes "${bcl2fastq_doc_dir}"`
+ fi
+ if echo $a | grep "^--mandir=" > /dev/null 2> /dev/null; then
+ bcl2fastq_man_dir=`echo $a | sed "s/^--mandir=//"`
+ bcl2fastq_man_dir=`bcl2fastq_fix_slashes "${bcl2fastq_man_dir}"`
+ fi
+ if echo $a | grep "^--builddir=" > /dev/null 2> /dev/null; then
+ bcl2fastq_build_dir=`echo $a | sed "s/^--builddir=//"`
+ bcl2fastq_build_dir=`bcl2fastq_fix_slashes "${bcl2fastq_build_dir}"`
+ fi
+done
+
+if [ "${bcl2fastq_build_dir}" == "${bcl2fastq_source_dir}" ]; then
+ echo "Warning: it is recommended to build BCL2FASTQ outside of the source directory:"
+ echo " mkdir ../bcl2fastq-build"
+ echo " cd ../bcl2fastq-build"
+ echo " ../$(basename $(pwd))/configure --prefix=/path/to/install/dir"
+ echo " make"
+ echo " make install"
+ if [ "x${bcl2fastq_force_builddir}" == "x" ]; then
+ echo "Use --force-builddir option to proceed with current build directory: ${bcl2fastq_build_dir}"
+ exit 6
+ else
+ echo "Build directory forced by --force-builddir: ${bcl2fastq_build_dir}"
+ fi
+fi
+
+# It is important to build cmake in a separate subtree from boost. Cmake 2.8.9 checks its installation folder
+# when searching for boost. If incompatible boost is present elsewhere in the system (/usr/include and such),
+# cmake caches the folder contents of its installation folder and fails to see iSAAC-built boost libraries
+# after they appear there.
+bcl2fastq_cmake_install_dir="${bcl2fastq_build_dir}/bootstrap_cmake"
+if [ "x${bcl2fastq_cmake}" == "x" ] ; then
+ ${bcl2fastq_bootstrap_dir}/installCmake.sh ${bcl2fastq_redist_dir} ${bcl2fastq_cmake_install_dir} "${bcl2fastq_parallel}"
+ cmake_install_exit_code="$?"
+ if [ $cmake_install_exit_code == "1" ]; then
+ bcl2fastq_cmake=cmake
+ echo "Using existing `which cmake`"
+ elif [ $cmake_install_exit_code == "0" ]; then
+ bcl2fastq_cmake="${bcl2fastq_cmake_install_dir}/bin/cmake"
+ echo "Using installed ${bcl2fastq_cmake}"
+ else
+ echo "Failed to verify or install cmake"
+ exit 3
+ fi
+fi
+
+if [ "x${bcl2fastq_prefix_dir}" == "x" ] ; then bcl2fastq_prefix_dir="${bcl2fastq_default_prefix}" ; fi
+
+# display information if required
+if [ "x${bcl2fastq_verbose}" != "x" ]; then
+ echo "Source directory: ${bcl2fastq_source_dir}"
+ echo "Prefix directory: ${bcl2fastq_prefix_dir}"
+ echo "Exec directory: ${bcl2fastq_exec_prefix_dir}"
+ echo "Binary directory: ${bcl2fastq_bin_dir}"
+ echo "Lib directory: ${bcl2fastq_lib_dir}"
+ echo "Libexec directory: ${bcl2fastq_libexec_dir}"
+ echo "Include directory: ${bcl2fastq_include_dir}"
+ echo "Data directory: ${bcl2fastq_data_dir}"
+ echo "Doc directory: ${bcl2fastq_doc_dir}"
+ echo "Man directory: ${bcl2fastq_man_dir}"
+ echo "Build directory: ${bcl2fastq_build_dir}"
+ echo "Cmake executable: ${bcl2fastq_cmake}"
+fi
+
+# create the build directory if necessary
+if [[ ! -d "${bcl2fastq_build_dir}" ]]; then
+ mkdir "${bcl2fastq_build_dir}"
+ if [ "$?" != 0 ]; then
+ echo "Couldn't create the build directory: ${bcl2fastq_build_dir}"
+ exit 4
+ fi
+fi
+
+# cmake options: general
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_NAME_SHORT:STRING=\"${bcl2fastq_name_short}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_NAME_LONG:STRING=\"${bcl2fastq_name_long}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_COPYRIGHT:STRING=\"${bcl2fastq_copyright}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_VERSION_MAJOR:STRING=\"${bcl2fastq_version_major}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_VERSION_MINOR:STRING=\"${bcl2fastq_version_minor}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_VERSION_PATCH:STRING=\"${bcl2fastq_version_patch}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_VERSION_BUILD:STRING=\"${bcl2fastq_version_build}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_VERSION:STRING=\"${bcl2fastq_version}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_SOURCE_DIR:STRING=\"${bcl2fastq_source_dir}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_PREFIX:PATH=\"${bcl2fastq_prefix_dir}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_EXEC_PREFIX:PATH=\"${bcl2fastq_exec_prefix_dir}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DCMAKE_INSTALL_PREFIX:PATH=\"${bcl2fastq_prefix_dir}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_BINDIR:PATH=\"${bcl2fastq_bin_dir}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_LIBDIR:PATH=\"${bcl2fastq_lib_dir}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_LIBEXECDIR:PATH=\"${bcl2fastq_libexec_dir}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_INCLUDEDIR:PATH=\"${bcl2fastq_include_dir}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_DATADIR:PATH=\"${bcl2fastq_data_dir}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_DOCDIR:PATH=\"${bcl2fastq_doc_dir}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_MANDIR:PATH=\"${bcl2fastq_man_dir}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=\"${bcl2fastq_build_type}\""
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DCMAKE_PARALLEL:STRING=\"${bcl2fastq_parallel}\""
+
+# cmake options: force static linking
+if [ "x${bcl2fastq_static}" != "x" ]; then
+ CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_FORCE_STATIC_LINK:BOOL=ON"
+ CMAKE_OPTIONS="$CMAKE_OPTIONS -DLINK_SEARCH_END_STATIC:BOOL=ON"
+fi
+
+# cmake options: cppunit
+if [ "x${bcl2fastq_unit_tests}" != "x" ]; then
+ CMAKE_OPTIONS="$CMAKE_OPTIONS -DBCL2FASTQ_UNIT_TESTS:BOOL=ON"
+fi
+
+# cmake options: packaging
+if [ "DEB" == "${bcl2fastq_package}" ]; then
+ CMAKE_OPTIONS="${CMAKE_OPTIONS} \
+ -DCPACK_GENERATOR:STRING=\"DEB\" \
+ -DCPACK_SYSTEM_NAME:STRING=\"${bcl2fastq_system}-${bcl2fastq_processor}\" \
+ -DCPACK_PACKAGE_CONTACT:STRING=\"'support at illumina.com\" \
+ -DCPACK_DEBIAN_PACKAGE_ARCHITECTURE:STRING=\"`dpkg --print-architecture`\""
+ CMAKE_OPTIONS="$CMAKE_OPTIONS -DCPACK_DEBIAN_PACKAGE_DEPENDS:STRING=\"\""
+
+elif [ "RPM" == "${bcl2fastq_package}" ]; then
+ CMAKE_OPTIONS="${CMAKE_OPTIONS} \
+ -DCPACK_GENERATOR:STRING=\"RPM\" \
+ -DCPACK_SYSTEM_NAME:STRING=\"${bcl2fastq_system}-${bcl2fastq_processor}\" \
+ -DCPACK_PACKAGE_CONTACT:STRING=\"support at illumina.com\""
+ CMAKE_OPTIONS="$CMAKE_OPTIONS -DCPACK_RPM_PACKAGE_REQUIRES:STRING=\"\""
+elif [ "TGZ" == "${bcl2fastq_package}" ]; then
+ CMAKE_OPTIONS="${CMAKE_OPTIONS} \
+ -DCPACK_GENERATOR:STRING=\"TGZ\" \
+ -DCPACK_SYSTEM_NAME:STRING=\"${bcl2fastq_system}-${bcl2fastq_processor}\""
+fi
+
+# invoke cmake
+bcl2fastq_cmake_command="${bcl2fastq_cmake} -H'${bcl2fastq_source_dir}' -B'${bcl2fastq_build_dir}' -G'${bcl2fastq_cmake_generator}' ${CMAKE_OPTIONS}"
+if [ "x${bcl2fastq_verbose}" != "x" ]; then
+ echo "Running on: `uname -a`"
+ echo "Configuring the build directory with:"
+ echo " ${bcl2fastq_cmake_command}"
+fi
+
+eval "${bcl2fastq_cmake_command}"
+
+if [ "$?" != 0 ]; then
+ echo "Couldn't configure the project:"
+ echo " ${bcl2fastq_cmake_command}"
+ if [ -f ${bcl2fastq_build_dir}/CMakeCache.txt ]; then
+ echo "Moving CMakeCache.txt to CMakeCache.txt.removed"
+ rm -f ${bcl2fastq_build_dir}/CMakeCache.txt.removed && mv ${bcl2fastq_build_dir}/CMakeCache.txt ${bcl2fastq_build_dir}/CMakeCache.txt.removed
+ fi
+ exit 5
+fi
+
+echo "The build directory ${bcl2fastq_build_dir} was configured successfully"
+echo ""
+echo Type "make" at the top level of the root directory to build BCL2FASTQ
+
+
diff --git a/src/css/CMakeLists.txt b/src/css/CMakeLists.txt
new file mode 100644
index 0000000..3381290
--- /dev/null
+++ b/src/css/CMakeLists.txt
@@ -0,0 +1,28 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the css subfolder
+##
+## author Mauricio Varea
+##
+################################################################################
+include ("${BCL2FASTQ_GLOBALS_CMAKE}")
+
+install(CODE "
+ include (\"${BCL2FASTQ_MACROS_CMAKE}\")
+ configure_files_recursively (\"${CMAKE_CURRENT_SOURCE_DIR}\" \"${CMAKE_CURRENT_BINARY_DIR}\" \"*.css\")
+ install_files_recursively (\"${CMAKE_CURRENT_BINARY_DIR}\" \"${BCL2FASTQ_ORIG_DATADIR}/css\" \"*.css\" \"\${BCL2FASTQ_LIBRARY_PERMISSIONS}\")
+ configure_files_recursively (\"${CMAKE_CURRENT_SOURCE_DIR}\" \"${CMAKE_CURRENT_BINARY_DIR}\" \"*.css.xml\")
+ install_files_recursively (\"${CMAKE_CURRENT_BINARY_DIR}\" \"${BCL2FASTQ_ORIG_DATADIR}/css\" \"*.css.xml\" \"\${BCL2FASTQ_LIBRARY_PERMISSIONS}\")
+ ")
+
diff --git a/src/css/Report.css.xml b/src/css/Report.css.xml
new file mode 100644
index 0000000..b644ce9
--- /dev/null
+++ b/src/css/Report.css.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<css>
+body {
+ font-size: 100%; /*The biggest table so far is the demultiplex summary. Verify if it is still viewable if you change this*/
+ font-family:monospace;
+}
+
+/* Firefox and probably others: force right scrollbar to make sure that percentage colun widths are calculated
+ same way as in data table */
+#ScrollableTableHeaderDiv {overflow: scroll; overflow-x: hidden; border-left: 1px gray solid; border-bottom: 0px gray solid;
+ padding:0px; margin: 0px;
+
+ border-width: 1px 1px 1px 1px;
+ border-style: inset inset inset inset;
+}
+
+/*IE 7 and 8 only. Disable the header table right scrollbar so that it does not get painted on top of the
+ last column heading*/
+*+html #ScrollableTableHeaderDiv {overflow: hidden; overflow-x: hidden; border-left: 1px gray solid; border-bottom: 0px gray solid;
+ padding:0px; margin: 0px;
+
+ border-width: 1px 1px 1px 1px;
+ border-style: inset inset inset inset;
+}
+
+/*IE 6 only (unverified). Disable the header table right scrollbar so that it does not get painted on top of the
+ last column heading*/
+* html #ScrollableTableHeaderDiv {overflow: hidden; overflow-x: hidden; border-left: 1px gray solid; border-bottom: 0px gray solid;
+ padding:0px; margin: 0px;
+
+ border-width: 1px 1px 1px 1px;
+ border-style: inset inset inset inset;
+}
+
+table#ReportTable {
+ border-width: 1px 1px 1px 1px;
+ border-collapse: collapse;
+
+}
+
+table#ReportTable th {
+}
+
+table#ReportTable td {
+ text-align:right;
+padding: 0.3em;
+
+}
+
+</css>
\ No newline at end of file
diff --git a/src/cxx/CMakeLists.txt b/src/cxx/CMakeLists.txt
new file mode 100644
index 0000000..2d23b50
--- /dev/null
+++ b/src/cxx/CMakeLists.txt
@@ -0,0 +1,76 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the cxx subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include (${BCL2FASTQ_GLOBALS_CMAKE})
+
+set(CMAKE_SKIP_BUILD_RPATH FALSE)
+set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
+set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+
+set (BCL2FASTQ_CXX_EXECUTABLE_CMAKE "${CMAKE_SOURCE_DIR}/cmake/cxxExecutable.cmake")
+set (BCL2FASTQ_CXX_LIBRARY_CMAKE "${CMAKE_SOURCE_DIR}/cmake/cxxLibrary.cmake")
+set (BCL2FASTQ_CXX_CONFIGURE_CMAKE "${CMAKE_SOURCE_DIR}/cmake/cxxConfigure.cmake")
+
+set(BCL2FASTQ_CXX_CONFIG_H_DIR ${CMAKE_CURRENT_BINARY_DIR}/common)
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake")
+include ("${BCL2FASTQ_CXX_CONFIGURE_CMAKE}")
+
+if (HAVE_CPPUNIT AND BCL2FASTQ_UNIT_TESTS)
+ set (BCL2FASTQ_CPPUNIT_CMAKE "${CMAKE_SOURCE_DIR}/cmake/cppunit.cmake")
+ add_subdirectory (unittest)
+endif (HAVE_CPPUNIT AND BCL2FASTQ_UNIT_TESTS)
+
+
+##
+## The include directories
+##
+set (BCL2FASTQ_CXX_ALL_INCLUDES "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}")
+
+##
+## Build all the libraries for the project
+##
+add_subdirectory (lib)
+
+##
+## build all the applications for the project
+##
+
+add_subdirectory (bin)
+add_subdirectory (libexec)
+
+##
+## build all the internal applications for the project
+##
+
+#add_subdirectory (libexec)
+
+##
+## build the documentation when available
+##
+include (FindDoxygen)
+message (STATUS "Doxygen: ${DOXYGEN_EXECUTABLE}. Dot: ${DOXYGEN_DOT_EXECUTABLE}.")
+if (DOXYGEN_FOUND)
+ set (DOXYFILE ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
+ message (STATUS "Creating Doxygen config file: ${DOXYFILE}")
+ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in ${DOXYFILE} @ONLY IMMEDIATE)
+ add_custom_target(doc ${DOXYGEN_EXECUTABLE} ${DOXYFILE})
+endif (DOXYGEN_FOUND)
+
+
diff --git a/src/cxx/CODING b/src/cxx/CODING
new file mode 100644
index 0000000..767e2f4
--- /dev/null
+++ b/src/cxx/CODING
@@ -0,0 +1,10 @@
+Directory structure:
+- bin: all user-facing programs
+- include: all headers
+- lib: all libraries
+
+The 'include' and 'lib' directories are subdivided into several libraries:
+- common: general-purpose components that are common to all programs
+- options: the program options
+
+
diff --git a/src/cxx/Doxyfile.in b/src/cxx/Doxyfile.in
new file mode 100644
index 0000000..3281d6a
--- /dev/null
+++ b/src/cxx/Doxyfile.in
@@ -0,0 +1,276 @@
+# Doxyfile 1.4.7
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+PROJECT_NAME = @PACKAGE_NAME@
+PROJECT_NUMBER = @VERSION@
+OUTPUT_DIRECTORY = @CMAKE_BINARY_DIR@/cxx/doxygen
+CREATE_SUBDIRS = NO
+OUTPUT_LANGUAGE = English
+USE_WINDOWS_ENCODING = NO
+BRIEF_MEMBER_DESC = YES
+REPEAT_BRIEF = YES
+ABBREVIATE_BRIEF = "The $name class" \
+ "The $name widget" \
+ "The $name file" \
+ is \
+ provides \
+ specifies \
+ contains \
+ represents \
+ a \
+ an \
+ the
+ALWAYS_DETAILED_SEC = NO
+INLINE_INHERITED_MEMB = NO
+FULL_PATH_NAMES = YES
+STRIP_FROM_PATH = @CMAKE_SOURCE_DIR@/cxx/ @CMAKE_BINARY_DIR@/cxx/
+STRIP_FROM_INC_PATH =
+SHORT_NAMES = NO
+JAVADOC_AUTOBRIEF = NO
+MULTILINE_CPP_IS_BRIEF = NO
+DETAILS_AT_TOP = NO
+INHERIT_DOCS = YES
+SEPARATE_MEMBER_PAGES = NO
+TAB_SIZE = 8
+ALIASES =
+OPTIMIZE_OUTPUT_FOR_C = NO
+OPTIMIZE_OUTPUT_JAVA = NO
+BUILTIN_STL_SUPPORT = NO
+DISTRIBUTE_GROUP_DOC = NO
+SUBGROUPING = YES
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+EXTRACT_ALL = YES
+EXTRACT_PRIVATE = YES
+EXTRACT_STATIC = YES
+EXTRACT_LOCAL_CLASSES = YES
+EXTRACT_LOCAL_METHODS = NO
+HIDE_UNDOC_MEMBERS = NO
+HIDE_UNDOC_CLASSES = NO
+HIDE_FRIEND_COMPOUNDS = NO
+HIDE_IN_BODY_DOCS = NO
+INTERNAL_DOCS = NO
+CASE_SENSE_NAMES = YES
+HIDE_SCOPE_NAMES = NO
+SHOW_INCLUDE_FILES = YES
+INLINE_INFO = YES
+SORT_MEMBER_DOCS = YES
+SORT_BRIEF_DOCS = NO
+SORT_BY_SCOPE_NAME = NO
+GENERATE_TODOLIST = YES
+GENERATE_TESTLIST = YES
+GENERATE_BUGLIST = YES
+GENERATE_DEPRECATEDLIST= YES
+ENABLED_SECTIONS =
+MAX_INITIALIZER_LINES = 30
+SHOW_USED_FILES = YES
+SHOW_DIRECTORIES = NO
+FILE_VERSION_FILTER =
+#---------------------------------------------------------------------------
+# configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+QUIET = NO
+WARNINGS = YES
+WARN_IF_UNDOCUMENTED = YES
+WARN_IF_DOC_ERROR = YES
+WARN_NO_PARAMDOC = NO
+WARN_FORMAT = "$file:$line: $text"
+WARN_LOGFILE =
+#---------------------------------------------------------------------------
+# configuration options related to the input files
+#---------------------------------------------------------------------------
+INPUT = @CMAKE_BINARY_DIR@/cxx/common @CMAKE_SOURCE_DIR@/cxx
+FILE_PATTERNS = *.c \
+ *.cc \
+ *.cxx \
+ *.cpp \
+ *.c++ \
+ *.d \
+ *.java \
+ *.ii \
+ *.ixx \
+ *.ipp \
+ *.i++ \
+ *.inl \
+ *.h \
+ *.hh \
+ *.hxx \
+ *.hpp \
+ *.h++ \
+ *.idl \
+ *.odl \
+ *.cs \
+ *.php \
+ *.php3 \
+ *.inc \
+ *.m \
+ *.mm \
+ *.dox \
+ *.py \
+ *.C \
+ *.CC \
+ *.C++ \
+ *.II \
+ *.I++ \
+ *.H \
+ *.HH \
+ *.H++ \
+ *.CS \
+ *.PHP \
+ *.PHP3 \
+ *.M \
+ *.MM \
+ *.PY
+RECURSIVE = YES
+EXCLUDE =
+EXCLUDE_SYMLINKS = NO
+EXCLUDE_PATTERNS =
+EXAMPLE_PATH =
+EXAMPLE_PATTERNS = *
+EXAMPLE_RECURSIVE = NO
+IMAGE_PATH =
+INPUT_FILTER =
+FILTER_PATTERNS =
+FILTER_SOURCE_FILES = NO
+#---------------------------------------------------------------------------
+# configuration options related to source browsing
+#---------------------------------------------------------------------------
+SOURCE_BROWSER = YES
+INLINE_SOURCES = NO
+STRIP_CODE_COMMENTS = YES
+REFERENCED_BY_RELATION = YES
+REFERENCES_RELATION = YES
+REFERENCES_LINK_SOURCE = YES
+USE_HTAGS = NO
+VERBATIM_HEADERS = YES
+#---------------------------------------------------------------------------
+# configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+ALPHABETICAL_INDEX = NO
+COLS_IN_ALPHA_INDEX = 5
+IGNORE_PREFIX =
+#---------------------------------------------------------------------------
+# configuration options related to the HTML output
+#---------------------------------------------------------------------------
+GENERATE_HTML = YES
+HTML_OUTPUT = html
+HTML_FILE_EXTENSION = .html
+HTML_HEADER =
+HTML_FOOTER =
+HTML_STYLESHEET =
+HTML_ALIGN_MEMBERS = YES
+GENERATE_HTMLHELP = NO
+CHM_FILE =
+HHC_LOCATION =
+GENERATE_CHI = NO
+BINARY_TOC = NO
+TOC_EXPAND = NO
+DISABLE_INDEX = NO
+ENUM_VALUES_PER_LINE = 4
+GENERATE_TREEVIEW = NO
+TREEVIEW_WIDTH = 250
+#---------------------------------------------------------------------------
+# configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+GENERATE_LATEX = NO
+LATEX_OUTPUT = latex
+LATEX_CMD_NAME = latex
+MAKEINDEX_CMD_NAME = makeindex
+COMPACT_LATEX = NO
+PAPER_TYPE = a4wide
+EXTRA_PACKAGES =
+LATEX_HEADER =
+PDF_HYPERLINKS = NO
+USE_PDFLATEX = NO
+LATEX_BATCHMODE = NO
+LATEX_HIDE_INDICES = NO
+#---------------------------------------------------------------------------
+# configuration options related to the RTF output
+#---------------------------------------------------------------------------
+GENERATE_RTF = NO
+RTF_OUTPUT = rtf
+COMPACT_RTF = NO
+RTF_HYPERLINKS = NO
+RTF_STYLESHEET_FILE =
+RTF_EXTENSIONS_FILE =
+#---------------------------------------------------------------------------
+# configuration options related to the man page output
+#---------------------------------------------------------------------------
+GENERATE_MAN = NO
+MAN_OUTPUT = man
+MAN_EXTENSION = .3
+MAN_LINKS = NO
+#---------------------------------------------------------------------------
+# configuration options related to the XML output
+#---------------------------------------------------------------------------
+GENERATE_XML = NO
+XML_OUTPUT = xml
+XML_SCHEMA =
+XML_DTD =
+XML_PROGRAMLISTING = YES
+#---------------------------------------------------------------------------
+# configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+GENERATE_AUTOGEN_DEF = NO
+#---------------------------------------------------------------------------
+# configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+GENERATE_PERLMOD = NO
+PERLMOD_LATEX = NO
+PERLMOD_PRETTY = YES
+PERLMOD_MAKEVAR_PREFIX =
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+ENABLE_PREPROCESSING = YES
+MACRO_EXPANSION = NO
+EXPAND_ONLY_PREDEF = NO
+SEARCH_INCLUDES = YES
+INCLUDE_PATH =
+INCLUDE_FILE_PATTERNS =
+PREDEFINED =
+EXPAND_AS_DEFINED =
+SKIP_FUNCTION_MACROS = YES
+#---------------------------------------------------------------------------
+# Configuration::additions related to external references
+#---------------------------------------------------------------------------
+TAGFILES =
+GENERATE_TAGFILE =
+ALLEXTERNALS = NO
+EXTERNAL_GROUPS = YES
+PERL_PATH = /usr/bin/perl
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+CLASS_DIAGRAMS = NO
+HIDE_UNDOC_RELATIONS = YES
+HAVE_DOT = YES
+CLASS_GRAPH = YES
+COLLABORATION_GRAPH = YES
+GROUP_GRAPHS = YES
+UML_LOOK = NO
+TEMPLATE_RELATIONS = YES
+INCLUDE_GRAPH = YES
+INCLUDED_BY_GRAPH = YES
+CALL_GRAPH = YES
+CALLER_GRAPH = YES
+GRAPHICAL_HIERARCHY = YES
+DIRECTORY_GRAPH = YES
+DOT_IMAGE_FORMAT = png
+DOT_PATH =
+DOTFILE_DIRS =
+MAX_DOT_GRAPH_WIDTH = 1024
+MAX_DOT_GRAPH_HEIGHT = 1024
+MAX_DOT_GRAPH_DEPTH = 1000
+DOT_TRANSPARENT = YES
+DOT_MULTI_TARGETS = YES
+GENERATE_LEGEND = YES
+DOT_CLEANUP = YES
+#---------------------------------------------------------------------------
+# Configuration::additions related to the search engine
+#---------------------------------------------------------------------------
+SEARCHENGINE = YES
+
diff --git a/src/cxx/bcl2fastq.sln b/src/cxx/bcl2fastq.sln
new file mode 100755
index 0000000..5fbbe7e
--- /dev/null
+++ b/src/cxx/bcl2fastq.sln
@@ -0,0 +1,32 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2013
+VisualStudioVersion = 12.0.31101.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bcl2fastq", "bcl2fastq.vcxproj", "{DE34C5BD-A131-49B0-AF6C-1D1B5282DA0C}"
+EndProject
+Global
+ GlobalSection(TeamFoundationVersionControl) = preSolution
+ SccNumberOfProjects = 1
+ SccEnterpriseProvider = {4CA58AB2-18FA-4F8D-95D4-32DDF27D184C}
+ SccTeamFoundationServer = http://ussd-prd-tfsa01:8080/tfs/defaultcollection
+ SccProjectUniqueName0 = bcl2fastq.vcxproj
+ SccProjectName0 = .
+ SccAuxPath0 = http://ussd-prd-tfsa01:8080/tfs/defaultcollection
+ SccLocalPath0 = .
+ SccProvider0 = {4CA58AB2-18FA-4F8D-95D4-32DDF27D184C}
+ EndGlobalSection
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|x64 = Debug|x64
+ Release|x64 = Release|x64
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {DE34C5BD-A131-49B0-AF6C-1D1B5282DA0C}.Debug|x64.ActiveCfg = Debug|x64
+ {DE34C5BD-A131-49B0-AF6C-1D1B5282DA0C}.Debug|x64.Build.0 = Debug|x64
+ {DE34C5BD-A131-49B0-AF6C-1D1B5282DA0C}.Release|x64.ActiveCfg = Release|x64
+ {DE34C5BD-A131-49B0-AF6C-1D1B5282DA0C}.Release|x64.Build.0 = Release|x64
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
diff --git a/src/cxx/bcl2fastq.vcxproj b/src/cxx/bcl2fastq.vcxproj
new file mode 100755
index 0000000..43f5096
--- /dev/null
+++ b/src/cxx/bcl2fastq.vcxproj
@@ -0,0 +1,326 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{DE34C5BD-A131-49B0-AF6C-1D1B5282DA0C}</ProjectGuid>
+ <SccProjectName>SAK</SccProjectName>
+ <SccAuxPath>SAK</SccAuxPath>
+ <SccLocalPath>SAK</SccLocalPath>
+ <SccProvider>SAK</SccProvider>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>bcl2fastq</RootNamespace>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v140</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v140</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v140</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v140</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <OutDir>$(Platform)\$(Configuration)\</OutDir>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level3</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level3</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <AdditionalIncludeDirectories>$(ProjectDir)..\Compression\zlib-1.2.5;$(SolutionDir)..\..\redist\boost-1.54\include\boost-1_54;include;$(SolutionDir)..\..\redist\libxslt-1.1.26\include;$(SolutionDir)..\..\redist\libxml2-2.7.8\include</AdditionalIncludeDirectories>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <AdditionalLibraryDirectories>$(ProjectDir)..\Compression\$(Platform)\$(Configuration);$(ProjectDir)..\..\redist\boost-1.54\lib64;$(ProjectDir)..\..\redist\libxslt-1.1.26\lib;$(ProjectDir)..\..\redist\libxml2-2.7.8\lib</AdditionalLibraryDirectories>
+ </Link>
+ <PostBuildEvent>
+ <Command>robocopy "$(SolutionDir)..\..\redist\libxml2-2.7.8\lib" $(OutputPath) libxml2.dll</Command>
+ </PostBuildEvent>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level3</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level3</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <AdditionalIncludeDirectories>$(SolutionDir)..\..\redist\Compression\zlib-1.2.5;$(SolutionDir)..\..\redist\boost-1.58\include;include;$(SolutionDir)..\..\redist\libxslt-1.1.26\include;$(SolutionDir)..\..\redist\libxml2-2.7.8\include</AdditionalIncludeDirectories>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ <AdditionalLibraryDirectories>$(ProjectDir)..\Compression\$(Platform)\$(Configuration);$(ProjectDir)..\..\redist\boost-1.58\lib64;$(ProjectDir)..\..\redist\libxslt-1.1.26\lib;$(ProjectDir)..\..\redist\libxml2-2.7.8\lib</AdditionalLibraryDirectories>
+ <AdditionalDependencies>libxml2.lib;libxslt.lib;libexslt.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ </Link>
+ <PostBuildEvent>
+ <Command>robocopy "$(SolutionDir)..\..\redist\libxml2-2.7.8\lib" $(OutputPath) libxml2.dll
+robocopy "$(SolutionDir)..\..\redist\libxslt-1.1.26\lib" $(OutputPath) libxslt.dll libexslt.dll
+robocopy "$(SolutionDir)..\xsl" $(OutputPath)share\xsl /E
+robocopy "$(SolutionDir)..\css" $(OutputPath)share\css /E
+
+if %ERRORLEVEL% EQU 1 exit 0</Command>
+ </PostBuildEvent>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClInclude Include="include\common\CsvGrammar.hh" />
+ <ClInclude Include="include\common\CsvGrammar.hpp" />
+ <ClInclude Include="include\common\Debug.hh" />
+ <ClInclude Include="include\common\DirectoryValidator.hh" />
+ <ClInclude Include="include\common\Endianness.hh" />
+ <ClInclude Include="include\common\Exceptions.hh" />
+ <ClInclude Include="include\common\FastIo.hh" />
+ <ClInclude Include="include\common\FastIo.hpp" />
+ <ClInclude Include="include\common\FileSystem.hh" />
+ <ClInclude Include="include\common\InstallationPath.hh" />
+ <ClInclude Include="include\common\Logger.hh" />
+ <ClInclude Include="include\common\Options.hh" />
+ <ClInclude Include="include\common\Program.hh" />
+ <ClInclude Include="include\common\Program.hpp" />
+ <ClInclude Include="include\common\ProgramInfo.hh" />
+ <ClInclude Include="include\common\SampleMetadata.hh" />
+ <ClInclude Include="include\common\StaticMemPool.hh" />
+ <ClInclude Include="include\common\StaticMemPool.hpp" />
+ <ClInclude Include="include\common\SystemCompatibility.hh" />
+ <ClInclude Include="include\common\SystemCompatibility.hpp" />
+ <ClInclude Include="include\common\Threads.hh" />
+ <ClInclude Include="include\common\Threads.hpp" />
+ <ClInclude Include="include\common\Types.hh" />
+ <ClInclude Include="include\config\Bcl2FastqOptions.hh" />
+ <ClInclude Include="include\config\SampleSheetCsv.hh" />
+ <ClInclude Include="include\conversion\AdapterLocator.hh" />
+ <ClInclude Include="include\conversion\AdapterLocator.hpp" />
+ <ClInclude Include="include\conversion\BclBaseConversion.hh" />
+ <ClInclude Include="include\conversion\BclLoader.hh" />
+ <ClInclude Include="include\conversion\BclReader.hh" />
+ <ClInclude Include="include\conversion\Converter.hh" />
+ <ClInclude Include="include\conversion\Demultiplexer.hh" />
+ <ClInclude Include="include\conversion\Demultiplexer.hpp" />
+ <ClInclude Include="include\conversion\FastqBuffer.hh" />
+ <ClInclude Include="include\conversion\FastqCreator.hh" />
+ <ClInclude Include="include\conversion\FastqCreator.hpp" />
+ <ClInclude Include="include\conversion\FastqIterator.hh" />
+ <ClInclude Include="include\conversion\FastqIterator.hpp" />
+ <ClInclude Include="include\conversion\FastqWriter.hh" />
+ <ClInclude Include="include\conversion\SampleIndex.hh" />
+ <ClInclude Include="include\conversion\Stage.hh" />
+ <ClInclude Include="include\conversion\Stage.hpp" />
+ <ClInclude Include="include\conversion\Task.hh" />
+ <ClInclude Include="include\conversion\TaskQueue.hh" />
+ <ClInclude Include="include\conversion\TaskQueue.hpp" />
+ <ClInclude Include="include\conversion\ThreadPool.hh" />
+ <ClInclude Include="include\conversion\ThreadSafeQueue.hh" />
+ <ClInclude Include="include\conversion\ThreadSafeQueue.hpp" />
+ <ClInclude Include="include\data\AggregatedBclFileReader.hh" />
+ <ClInclude Include="include\data\BclBuffer.hh" />
+ <ClInclude Include="include\data\BclFile.hh" />
+ <ClInclude Include="include\data\BclFileReader.hh" />
+ <ClInclude Include="include\data\ControlFile.hh" />
+ <ClInclude Include="include\data\CycleBCIFile.hh" />
+ <ClInclude Include="include\data\FastqFile.hh" />
+ <ClInclude Include="include\data\FileReader.hh" />
+ <ClInclude Include="include\data\FileReader.hpp" />
+ <ClInclude Include="include\data\FilterFile.hh" />
+ <ClInclude Include="include\data\InteropFile.hh" />
+ <ClInclude Include="include\data\PositionsFile.hh" />
+ <ClInclude Include="include\data\RawBclBuffer.hh" />
+ <ClInclude Include="include\data\TileBclFileReader.hh" />
+ <ClInclude Include="include\io\Bgzf.hh" />
+ <ClInclude Include="include\io\BgzfCompressor.hh" />
+ <ClInclude Include="include\io\FileBufWithReopen.hh" />
+ <ClInclude Include="include\io\FileBufWithReopen.hpp" />
+ <ClInclude Include="include\io\GzipCompressor.hh" />
+ <ClInclude Include="include\io\GzipDecompressor.hh" />
+ <ClInclude Include="include\io\GzipDecompressor.hpp" />
+ <ClInclude Include="include\io\GzipHeader.hh" />
+ <ClInclude Include="include\io\Utility.hh" />
+ <ClInclude Include="include\io\Xml.hh" />
+ <ClInclude Include="include\io\Xml.hpp" />
+ <ClInclude Include="include\layout\Barcode.hh" />
+ <ClInclude Include="include\layout\Barcode.hpp" />
+ <ClInclude Include="include\layout\BarcodeCollisionDetector.hh" />
+ <ClInclude Include="include\layout\BarcodeTranslationTable.hh" />
+ <ClInclude Include="include\layout\BCIndex.hh" />
+ <ClInclude Include="include\layout\ConfigXml.hh" />
+ <ClInclude Include="include\layout\CycleInfo.hh" />
+ <ClInclude Include="include\layout\FileExistenceVerifier.hh" />
+ <ClInclude Include="include\layout\FlowcellInfo.hh" />
+ <ClInclude Include="include\layout\LaneInfo.hh" />
+ <ClInclude Include="include\layout\Layout.hh" />
+ <ClInclude Include="include\layout\ReadInfo.hh" />
+ <ClInclude Include="include\layout\ReadMetadata.hh" />
+ <ClInclude Include="include\layout\RunInfoXml.hh" />
+ <ClInclude Include="include\layout\SampleInfo.hh" />
+ <ClInclude Include="include\layout\TileInfo.hh" />
+ <ClInclude Include="include\layout\UseBasesMask.hh" />
+ <ClInclude Include="include\stats\BarcodeHits.hh" />
+ <ClInclude Include="include\stats\BarcodeStats.hh" />
+ <ClInclude Include="include\stats\ConversionStatsXml.hh" />
+ <ClInclude Include="include\stats\DemultiplexingStatsXml.hh" />
+ <ClInclude Include="include\stats\DemuxReportGenerator.hh" />
+ <ClInclude Include="include\stats\Json.hh" />
+ <ClInclude Include="include\stats\TileStats.hpp" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="bin\bcl2fastq.cpp" />
+ <ClCompile Include="lib\common\CsvGrammar.cpp" />
+ <ClCompile Include="lib\common\DirectoryValidator.cpp" />
+ <ClCompile Include="lib\common\Exceptions.cpp" />
+ <ClCompile Include="lib\common\FileSystem.cpp" />
+ <ClCompile Include="lib\common\InstallationPath.cpp" />
+ <ClCompile Include="lib\common\Logger.cpp" />
+ <ClCompile Include="lib\common\Options.cpp" />
+ <ClCompile Include="lib\common\ProgramInfo.cpp" />
+ <ClCompile Include="lib\common\SystemCompatibility.cpp" />
+ <ClCompile Include="lib\config\Bcl2FastqOptions.cpp" />
+ <ClCompile Include="lib\config\SampleSheetCsv.cpp" />
+ <ClCompile Include="lib\conversion\BclLoader.cpp" />
+ <ClCompile Include="lib\conversion\BclReader.cpp" />
+ <ClCompile Include="lib\conversion\Converter.cpp" />
+ <ClCompile Include="lib\conversion\Demultiplexer.cpp" />
+ <ClCompile Include="lib\conversion\FastqBuffer.cpp" />
+ <ClCompile Include="lib\conversion\FastqCreator.cpp" />
+ <ClCompile Include="lib\conversion\FastqWriter.cpp" />
+ <ClCompile Include="lib\conversion\SampleIndex.cpp" />
+ <ClCompile Include="lib\conversion\Stage.cpp" />
+ <ClCompile Include="lib\conversion\Task.cpp" />
+ <ClCompile Include="lib\conversion\ThreadPool.cpp" />
+ <ClCompile Include="lib\data\AggregatedBclFileReader.cpp" />
+ <ClCompile Include="lib\data\BclFile.cpp" />
+ <ClCompile Include="lib\data\CbclFile.cpp" />
+ <ClCompile Include="lib\data\ControlFile.cpp" />
+ <ClCompile Include="lib\data\CycleBCIFile.cpp" />
+ <ClCompile Include="lib\data\FastqFile.cpp" />
+ <ClCompile Include="lib\data\FileReader.cpp" />
+ <ClCompile Include="lib\data\FilterFile.cpp" />
+ <ClCompile Include="lib\data\InteropFile.cpp" />
+ <ClCompile Include="lib\data\PositionsFile.cpp" />
+ <ClCompile Include="lib\data\TileBclFileReader.cpp" />
+ <ClCompile Include="lib\io\GzipCompressor.cpp" />
+ <ClCompile Include="lib\io\GzipDecompressor.cpp" />
+ <ClCompile Include="lib\io\SyncFile.cpp" />
+ <ClCompile Include="lib\io\Utility.cpp" />
+ <ClCompile Include="lib\io\Xml.cpp" />
+ <ClCompile Include="lib\layout\Barcode.cpp" />
+ <ClCompile Include="lib\layout\BarcodeCollisionDetector.cpp" />
+ <ClCompile Include="lib\layout\BarcodeTranslationTable.cpp" />
+ <ClCompile Include="lib\layout\BCIndex.cpp" />
+ <ClCompile Include="lib\layout\ConfigXml.cpp" />
+ <ClCompile Include="lib\layout\CycleInfo.cpp" />
+ <ClCompile Include="lib\layout\FileExistenceVerifier.cpp" />
+ <ClCompile Include="lib\layout\FlowcellInfo.cpp" />
+ <ClCompile Include="lib\layout\LaneInfo.cpp" />
+ <ClCompile Include="lib\layout\Layout.cpp" />
+ <ClCompile Include="lib\layout\ReadInfo.cpp" />
+ <ClCompile Include="lib\layout\RunInfoXml.cpp" />
+ <ClCompile Include="lib\layout\SampleInfo.cpp" />
+ <ClCompile Include="lib\layout\TileInfo.cpp" />
+ <ClCompile Include="lib\layout\UseBasesMask.cpp" />
+ <ClCompile Include="lib\stats\ConversionStatsXml.cpp" />
+ <ClCompile Include="lib\stats\DemultiplexingStatsXml.cpp" />
+ <ClCompile Include="lib\stats\DemuxReportGenerator.cpp" />
+ <ClCompile Include="lib\stats\Json.cpp" />
+ <ClCompile Include="lib\stats\TileStats.cpp" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/src/cxx/bcl2fastq.vcxproj.filters b/src/cxx/bcl2fastq.vcxproj.filters
new file mode 100755
index 0000000..4c09875
--- /dev/null
+++ b/src/cxx/bcl2fastq.vcxproj.filters
@@ -0,0 +1,489 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup>
+ <Filter Include="Source Files">
+ <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+ <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+ </Filter>
+ <Filter Include="Header Files">
+ <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+ <Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
+ </Filter>
+ <Filter Include="Resource Files">
+ <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+ <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+ </Filter>
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="include\data\BclFile.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\ControlFile.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\CycleBCIFile.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\FastqFile.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\FileReader.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\FileReader.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\FilterFile.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\InteropFile.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\PositionsFile.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\io\Bgzf.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\io\BgzfCompressor.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\io\FileBufWithReopen.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\io\FileBufWithReopen.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\io\GzipCompressor.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\io\GzipDecompressor.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\io\GzipDecompressor.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\io\GzipHeader.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\io\Utility.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\io\Xml.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\io\Xml.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\Barcode.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\Barcode.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\BarcodeCollisionDetector.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\BarcodeTranslationTable.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\BCIndex.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\ConfigXml.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\CycleInfo.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\FileExistenceVerifier.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\FlowcellInfo.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\LaneInfo.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\Layout.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\ReadInfo.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\ReadMetadata.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\RunInfoXml.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\SampleInfo.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\TileInfo.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\layout\UseBasesMask.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\stats\BarcodeHits.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\stats\BarcodeStats.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\stats\ConversionStatsXml.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\stats\DemultiplexingStatsXml.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\stats\DemuxReportGenerator.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\stats\TileStats.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\CsvGrammar.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\CsvGrammar.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\Debug.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\DirectoryValidator.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\Endianness.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\Exceptions.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\FastIo.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\FastIo.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\FileSystem.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\InstallationPath.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\Logger.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\Options.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\Program.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\Program.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\ProgramInfo.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\SampleMetadata.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\StaticMemPool.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\StaticMemPool.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\SystemCompatibility.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\SystemCompatibility.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\Threads.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\Threads.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\common\Types.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\config\Bcl2FastqOptions.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\config\SampleSheetCsv.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\AdapterLocator.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\BclBaseConversion.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\BclLoader.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\Converter.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\Demultiplexer.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\FastqBuffer.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\FastqCreator.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\FastqIterator.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\FastqIterator.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\FastqWriter.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\SampleIndex.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\Stage.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\Stage.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\Task.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\TaskQueue.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\stats\Json.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\TaskQueue.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\ThreadPool.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\ThreadSafeQueue.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\ThreadSafeQueue.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\BclReader.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\BclBuffer.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\BclFileReader.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\RawBclBuffer.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\AggregatedBclFileReader.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\data\TileBclFileReader.hh">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\AdapterLocator.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\Demultiplexer.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="include\conversion\FastqCreator.hpp">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="lib\common\CsvGrammar.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\common\DirectoryValidator.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\common\Exceptions.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\common\FileSystem.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\common\InstallationPath.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\common\Logger.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\common\Options.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\common\ProgramInfo.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\common\SystemCompatibility.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\config\Bcl2FastqOptions.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\config\SampleSheetCsv.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\conversion\BclLoader.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\conversion\Converter.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\conversion\Demultiplexer.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\conversion\FastqBuffer.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\conversion\FastqCreator.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\conversion\FastqWriter.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\conversion\SampleIndex.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\conversion\Stage.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\conversion\Task.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\data\BclFile.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\data\ControlFile.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\data\CycleBCIFile.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\data\FastqFile.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\data\FileReader.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\data\FilterFile.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\data\InteropFile.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\data\PositionsFile.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\io\GzipCompressor.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\io\GzipDecompressor.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\io\Utility.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\io\Xml.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\Barcode.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\BarcodeCollisionDetector.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\BarcodeTranslationTable.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\BCIndex.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\ConfigXml.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\CycleInfo.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\FileExistenceVerifier.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\FlowcellInfo.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\LaneInfo.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\Layout.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\ReadInfo.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\RunInfoXml.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\SampleInfo.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\TileInfo.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\layout\UseBasesMask.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\stats\ConversionStatsXml.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\stats\DemultiplexingStatsXml.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\stats\DemuxReportGenerator.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\stats\TileStats.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="bin\bcl2fastq.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\stats\Json.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\conversion\ThreadPool.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\conversion\BclReader.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\data\AggregatedBclFileReader.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\data\TileBclFileReader.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\data\CbclFile.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="lib\io\SyncFile.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ </ItemGroup>
+</Project>
\ No newline at end of file
diff --git a/src/cxx/bcl2fastq.vcxproj.vspscc b/src/cxx/bcl2fastq.vcxproj.vspscc
new file mode 100755
index 0000000..feffdec
--- /dev/null
+++ b/src/cxx/bcl2fastq.vcxproj.vspscc
@@ -0,0 +1,10 @@
+""
+{
+"FILE_VERSION" = "9237"
+"ENLISTMENT_CHOICE" = "NEVER"
+"PROJECT_FILE_RELATIVE_PATH" = ""
+"NUMBER_OF_EXCLUDED_FILES" = "0"
+"ORIGINAL_PROJECT_FILE_PATH" = ""
+"NUMBER_OF_NESTED_PROJECTS" = "0"
+"SOURCE_CONTROL_SETTINGS_PROVIDER" = "PROVIDER"
+}
diff --git a/src/cxx/bin/CMakeLists.txt b/src/cxx/bin/CMakeLists.txt
new file mode 100644
index 0000000..916d246
--- /dev/null
+++ b/src/cxx/bin/CMakeLists.txt
@@ -0,0 +1,38 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the cxx/bin subdirectory.
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CXX_EXECUTABLE_CMAKE})
+
+file (GLOB BCL2FASTQ_PROGRAM_SOURCE_LIST [a-zA-Z0-9]*.cpp)
+
+##
+## Generic rule for all the other programs
+##
+foreach(BCL2FASTQ_PROGRAM_SOURCE ${BCL2FASTQ_PROGRAM_SOURCE_LIST})
+ get_filename_component(BCL2FASTQ_PROGRAM ${BCL2FASTQ_PROGRAM_SOURCE} NAME_WE)
+ add_executable (${BCL2FASTQ_PROGRAM} ${BCL2FASTQ_PROGRAM_SOURCE})
+ target_link_libraries (${BCL2FASTQ_PROGRAM} ${BCL2FASTQ_AVAILABLE_LIBRARIES}
+ ${Boost_LIBRARIES}
+ ${BCL2FASTQ_DEP_LIB}
+ ${BCL2FASTQ_ADDITIONAL_LIB} )
+ install(TARGETS ${BCL2FASTQ_PROGRAM} RUNTIME DESTINATION ${BCL2FASTQ_BINDIR})
+endforeach(BCL2FASTQ_PROGRAM_SOURCE)
+
+
diff --git a/src/cxx/bin/bcl2fastq.cpp b/src/cxx/bin/bcl2fastq.cpp
new file mode 100644
index 0000000..476c28f
--- /dev/null
+++ b/src/cxx/bin/bcl2fastq.cpp
@@ -0,0 +1,155 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file bcl2fastq.cpp
+ *
+ * \brief Bcl2Fastq main.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+#include <boost/cstdlib.hpp>
+
+#include <algorithm>
+
+#include <boost/bind.hpp>
+#include <boost/cstdlib.hpp>
+
+#include "common/Program.hh"
+#include "common/ProgramInfo.hh"
+#include "config/Bcl2FastqOptions.hh"
+
+#include "layout/RunInfoXml.hh"
+#include "layout/Layout.hh"
+#include "layout/LaneInfo.hh"
+#include "conversion/Converter.hh"
+#include "stats/DemuxReportGenerator.hh"
+
+
+namespace detail {
+
+static const unsigned TOP_UNKNOWN_BARCODES_MAX = 1000;
+
+static const std::string interopStatsFileName("IndexMetricsOut.bin");
+static const std::string demuxStatsFileName("DemultiplexingStats.xml");
+static const std::string conversionStatsFileName("ConversionStats.xml");
+static const std::string statsJsonFileName("Stats.json");
+static const std::string adapterStatsFileName("AdapterTrimming.txt");
+static const std::string fastqSummaryPartialFileName("FastqSummaryF1L");
+static const std::string demuxSummaryPartialFileName("DemuxSummaryF1L");
+
+/// \brief Process single lane.
+/// \param layout Flowcell layout.
+/// \param options Program options.
+/// \param laneInfo Lane metadata.
+static void processLane(
+ const bcl2fastq::config::Bcl2FastqOptions &options,
+ const bcl2fastq::layout::Layout &layout,
+ const bcl2fastq::layout::LaneInfo &laneInfo,
+ bcl2fastq::conversion::ConverterStats &stats
+)
+{
+ bcl2fastq::conversion::Converter( options, layout, laneInfo, stats ).run();
+ stats.getUnknownBarcodeStat( laneInfo ).findPopular( TOP_UNKNOWN_BARCODES_MAX, laneInfo.getNumber() );
+}
+
+
+} // namespace detail
+
+
+static void program(
+ const bcl2fastq::common::ProgramInfo &info,
+ bcl2fastq::config::Bcl2FastqOptions &options
+)
+{
+ bcl2fastq::common::adjustMaxFileHandles();
+
+ const bcl2fastq::layout::RunInfoXml runInfoXml = bcl2fastq::layout::createRunInfoXml(options.getRunfolderDir());
+
+ bcl2fastq::layout::Layout layout(
+ options.getIntensitiesDir(),
+ options.getInputDir(),
+ options.getOutputDir(),
+ options.getReportsDir(),
+ options.getStatsDir(),
+ options.getSampleSheet(),
+ runInfoXml,
+ options.getTilesFilterList(),
+ options.getUseBasesMasks(),
+ options.getMinimumTrimmedReadLength(),
+ options.getAutoSetToZeroBarcodeMismatches(),
+ options.getBarcodeMismatches(),
+ options.getIgnoreMissingBcls(),
+ options.getIgnoreMissingFilters(),
+ options.getIgnoreMissingPositions(),
+ options.getSampleSheet().getRead1StartCycle(),
+ options.getSampleSheet().getRead2StartCycle(),
+ options.getSampleSheet().getRead1EndCycle(),
+ options.getSampleSheet().getRead2EndCycle(),
+ options.getSampleSheet().getRead1UmiLength(),
+ options.getSampleSheet().getRead2UmiLength(),
+ options.getSampleSheet().getRead1UmiStartFromCycle(),
+ options.getSampleSheet().getRead2UmiStartFromCycle(),
+ options.getSampleSheet().trimUmi() == bcl2fastq::config::SampleSheetCsv::TriState::TRUE,
+ options.getIncludeNonPfClusters()
+ );
+
+ bcl2fastq::conversion::ConverterStats allStats(
+ layout,
+ options.getStatsDir() / boost::filesystem::path( detail::fastqSummaryPartialFileName ),
+ options.getStatsDir() / boost::filesystem::path( detail::demuxSummaryPartialFileName ),
+ options.getStatsDir() / boost::filesystem::path( detail::adapterStatsFileName ),
+ options.getInteropDir() / boost::filesystem::path( detail::interopStatsFileName ),
+ options.getStatsDir() / boost::filesystem::path( detail::demuxStatsFileName ),
+ options.getStatsDir() / boost::filesystem::path( detail::conversionStatsFileName ),
+ options.getStatsDir() / boost::filesystem::path( detail::statsJsonFileName )
+ );
+
+ std::for_each(
+ layout.laneInfosBegin(),
+ layout.laneInfosEnd(),
+ boost::bind(
+ detail::processLane,
+ boost::ref(options),
+ boost::ref(layout),
+ _1,
+ boost::ref(allStats)
+ )
+ );
+
+ try {
+ auto start = std::chrono::steady_clock::now();
+
+ allStats.dumpInteropStats();
+ allStats.dumpDemuxStats();
+ allStats.dumpConversionStats();
+
+ bcl2fastq::stats::DemuxReportGenerator(
+ layout,
+ options.getReportsDir()
+ ).run(
+ allStats.getDemuxStatsXmlPath(),
+ allStats.getConversionStatsXmlPath()
+ );
+
+ BCL2FASTQ_LOG(bcl2fastq::common::LogLevel::DEBUG) << "Stats calculated in: " << std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - start).count() << "s" << std::endl;
+
+ } catch( bcl2fastq::common::RuntimeError &e ) {
+ BCL2FASTQ_LOG(bcl2fastq::common::LogLevel::WARNING) << "Could not compute and write statistics for this conversion: "
+ << e.what() << std::endl;
+ }
+}
+
+int main(int argc, char *argv[])
+{
+ bcl2fastq::common::run(program, argc, argv);
+ return boost::exit_success;
+}
+
+
diff --git a/src/cxx/include/common/CsvGrammar.hh b/src/cxx/include/common/CsvGrammar.hh
new file mode 100644
index 0000000..fec2334
--- /dev/null
+++ b/src/cxx/include/common/CsvGrammar.hh
@@ -0,0 +1,124 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file CsvGrammar.hh
+ *
+ * \brief Basic CSV grammar.
+ *
+ * \author Roman Petrovski
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_CSV_GRAMMAR_HH
+#define BCL2FASTQ_COMMON_CSV_GRAMMAR_HH
+
+
+#include <vector>
+#include <string>
+
+#include <boost/filesystem/path.hpp>
+
+#include <boost/spirit/include/qi_grammar.hpp>
+#include <boost/spirit/include/qi_rule.hpp>
+#include <boost/spirit/home/support/unused.hpp>
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+/// \brief CSV grammar attribute.
+typedef std::vector<std::vector<std::string> > CsvGrammarAttribute;
+
+/// \brief CSV grammar.
+template <typename Iterator>
+struct CsvGrammar : public boost::spirit::qi::grammar<Iterator, CsvGrammarAttribute()>
+{
+public:
+
+ /// \brief Default constructor.
+ CsvGrammar();
+
+public:
+
+ /// \brief CSV file.
+ boost::spirit::qi::rule<Iterator, CsvGrammarAttribute()> start_;
+
+ /// \brief CSV line.
+ boost::spirit::qi::rule<Iterator, std::vector<std::string>() > line_;
+
+ /// \brief Csv data line.
+ boost::spirit::qi::rule<Iterator, std::vector<std::string>() > dataLine_;
+
+ /// \brief CSV comment line.
+ boost::spirit::qi::rule<Iterator, boost::spirit::unused_type()> commentLine_;
+
+ /// \brief CSV field.
+ boost::spirit::qi::rule<Iterator, std::string()> field_;
+
+ /// \brief Escaped (quoted) field value.
+ boost::spirit::qi::rule<Iterator, std::string()> escaped_;
+
+ /// \brief Non-escaped (not quoted) field value.
+ boost::spirit::qi::rule<Iterator, std::string()> nonEscaped_;
+
+ /// \brief All printable characters plus space, except dquote_ and comma_.
+ boost::spirit::qi::rule<Iterator, char()> textData_;
+
+ /// \brief CR.
+ boost::spirit::qi::rule<Iterator, char()> cr_;
+
+ /// \brief LF.
+ boost::spirit::qi::rule<Iterator, char()> lf_;
+
+ /// \brief Line ending.
+ boost::spirit::qi::rule<Iterator, boost::spirit::unused_type()> crlf_;
+
+ /// \brief Comma.
+ boost::spirit::qi::rule<Iterator, char()> comma_;
+
+ /// \brief Double-quote.
+ boost::spirit::qi::rule<Iterator, char()> dquote_;
+};
+
+
+/// \brief Parse CSV data.
+/// \param csvDataBegin Begin iterator to CSV data to be parsed.
+/// \param csvDataEnd End iterator to CSV data to be parsed.
+/// \return Data strucutre containing data parsed out of the CSV data.
+template<typename Iterator>
+CsvGrammarAttribute parseCsvData(Iterator csvDataBegin, Iterator csvDataEnd);
+
+/// \brief Parse CSV file.
+/// \param csvFile Path to CSV file to be parsed.
+/// \return Data strucutre containing data parsed out of the CSV file.
+CsvGrammarAttribute parseCsvFile(const boost::filesystem::path& csvFile);
+
+/// \brief Insert missing columns into CSV data structure.
+/// \param csvData CSV data structure to operate on.
+/// \param defaultValue Default value for inserted fields.
+/// \return Number of columns per row.
+/// \post CSV data structure containing the same number of columns in each row.
+CsvGrammarAttribute::value_type::size_type correctCsvColumnsCount(CsvGrammarAttribute &csvData, const std::string &defaultValue = "");
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#include "common/CsvGrammar.hpp"
+
+
+#endif //BCL2FASTQ_COMMON_CSV_GRAMMAR_HH
+
+
diff --git a/src/cxx/include/common/CsvGrammar.hpp b/src/cxx/include/common/CsvGrammar.hpp
new file mode 100644
index 0000000..15b4e40
--- /dev/null
+++ b/src/cxx/include/common/CsvGrammar.hpp
@@ -0,0 +1,108 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file CsvGrammar.hpp
+ *
+ * \brief Basic CSV grammar.
+ *
+ * \author Roman Petrovski
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_CSVGRAMMAR_HPP
+#define BCL2FASTQ_COMMON_CSVGRAMMAR_HPP
+
+
+#include <boost/spirit/include/qi.hpp>
+#include <boost/spirit/include/phoenix_stl.hpp>
+
+#include "common/Exceptions.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+template <typename Iterator>
+CsvGrammar<Iterator>::CsvGrammar()
+: CsvGrammar::base_type(start_)
+{
+ using boost::spirit::omit;
+ using boost::spirit::qi::_val;
+ using boost::spirit::qi::_1;
+ using boost::spirit::ascii::char_;
+ using boost::phoenix::push_back;
+
+ // CSV ABNF mainly taken from http://www.ietf.org/rfc/rfc4180.txt
+ // with some modification to make sample sheet users life easier
+ cr_ = char_('\x0d');
+ lf_ = char_('\x0a');
+ crlf_ = +(cr_ | lf_); // unlike the real csv, sample sheets often get mixed line endings
+ comma_ = char_(',');
+ comma_.name(",");
+ dquote_ = char_('"');
+
+ //textData_ = char_('\x20', '\x21') | char_('\x23', '\x2b') | char_('\x2d', '\x7e');
+ textData_ = char_('\x20', '\x7e') - (char_(',') | char_('"'));
+
+ /// \todo Refactoring: Probably omit[dquote_] should be used.
+ escaped_ = dquote_
+ >> *(textData_[push_back(_val, _1)] |
+ comma_[push_back(_val, _1)] |
+ cr_[push_back(_val, _1)] |
+ lf_[push_back(_val, _1)] |
+ (dquote_ >> dquote_)[push_back(_val, '"')])
+ >> dquote_
+ ;
+
+ nonEscaped_ = *textData_;
+
+ field_ = escaped_ | nonEscaped_;
+
+ commentLine_ = char_('#') >> *(char_('\x00', '\x7e') - crlf_);
+
+ dataLine_ = field_ % comma_;
+
+ line_ = *(commentLine_ >> +crlf_) >> dataLine_ >> *(+crlf_ >> commentLine_);
+
+ start_ = line_ % crlf_ >> *crlf_;
+}
+
+
+template<typename Iterator>
+CsvGrammarAttribute parseCsvData(Iterator csvDataBegin, Iterator csvDataEnd)
+{
+ common::CsvGrammar<Iterator> parser;
+
+ common::CsvGrammarAttribute csvData;
+ if (!boost::spirit::qi::parse(csvDataBegin, csvDataEnd, parser, csvData) || csvDataEnd != csvDataBegin)
+ {
+ int errnum = errno;
+ const std::string message = "Could not parse the CSV stream text:\n"
+ + std::string(csvDataBegin, csvDataEnd)
+ ;
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, message));
+ }
+
+ correctCsvColumnsCount(csvData, "");
+ return csvData;
+}
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#endif //BCL2FASTQ_COMMON_CSVGRAMMAR_HPP
+
+
diff --git a/src/cxx/include/common/Debug.hh b/src/cxx/include/common/Debug.hh
new file mode 100644
index 0000000..ea12b91
--- /dev/null
+++ b/src/cxx/include/common/Debug.hh
@@ -0,0 +1,69 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Debug.hh
+ *
+ * \brief Various debugging-related helpers.
+ *
+ * \author Roman Petrovski
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_DEBUG_HH
+#define BCL2FASTQ_COMMON_DEBUG_HH
+
+
+#include "common/Logger.hh"
+#include "common/SystemCompatibility.hh"
+
+
+/// \brief Asserts condition (always, i.e. even if NDEBUG is set and so on).
+/// \param condition Condition to be asserted.
+/// \param message Error message.
+/// \note Also uses ostream serialization which, unlike the standard assert,
+/// has shown not to allocate the dynamic memory at the time when you least
+/// expect this to happen.
+#define BCL2FASTQ_ASSERT_MSG(condition, message) \
+ { \
+ if (condition) \
+ { \
+ /* nothing here */ \
+ } \
+ else \
+ { \
+ BCL2FASTQ_LOG(::bcl2fastq::common::LogLevel::FATAL) \
+ << "***** Internal Program Error - assertion (" \
+ << #condition \
+ << ") failed in " \
+ << __FILE__ << ":" \
+ << __LINE__ << ":" \
+ << (BOOST_CURRENT_FUNCTION) << ": " \
+ << message \
+ << std::endl \
+ ; \
+ ::bcl2fastq::common::terminateWithCoreDump(); \
+ } \
+ }
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_COMMON_DEBUG_HH
+
+
diff --git a/src/cxx/include/common/DirectoryValidator.hh b/src/cxx/include/common/DirectoryValidator.hh
new file mode 100644
index 0000000..96874b0
--- /dev/null
+++ b/src/cxx/include/common/DirectoryValidator.hh
@@ -0,0 +1,84 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file DirectoryValidator.hh
+ *
+ * \brief Declaration of directory validator.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_DIRECTORY_VALIDATOR_HH
+#define BCL2FASTQ_COMMON_DIRECTORY_VALIDATOR_HH
+
+#include "common/Exceptions.hh"
+
+#include <boost/noncopyable.hpp>
+#include <boost/filesystem/path.hpp>
+#include <vector>
+#include <utility>
+
+namespace bcl2fastq
+{
+namespace common
+{
+
+class DirectoryValidationError : public common::RuntimeError
+{
+public:
+ DirectoryValidationError(const boost::filesystem::path& uniquePath,
+ const std::string& uniquePathDescription,
+ const boost::filesystem::path& invalidPath,
+ const std::string& invalidPathDescription);
+
+ DirectoryValidationError(const DirectoryValidationError& other);
+
+ std::string createErrorMessage(const boost::filesystem::path& uniquePath,
+ const std::string& uniquePathDescription,
+ const boost::filesystem::path& invalidPath,
+ const std::string& invalidPathDescription);
+};
+
+/// \brief Directory validator.
+class DirectoryValidator : private boost::noncopyable
+{
+public:
+ static DirectoryValidator& getSingleton();
+
+ /// \brief Add a unique path.
+ /// \param Unique path
+ /// \param description of path (used in error message)
+ void addUniquePath(const boost::filesystem::path& path,
+ const std::string& description);
+
+ /// \brief Validate that there is no clash with the unique paths.
+ /// \param path to validate
+ /// \param description of path (used in error message)
+ void validateNoClash(const boost::filesystem::path& path,
+ const std::string& description) const;
+
+private:
+
+ /// \brief Constructor.
+ DirectoryValidator();
+
+ /// \brief uniquePaths_.
+ ///
+ /// Source: All paths which must be unique.
+ std::vector< std::pair<boost::filesystem::path, std::string> > uniquePaths_;
+};
+
+
+} // namespace common
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_COMMON_DIRECTORY_VALIDATOR_HH
+
+
diff --git a/src/cxx/include/common/Endianness.hh b/src/cxx/include/common/Endianness.hh
new file mode 100644
index 0000000..12b5914
--- /dev/null
+++ b/src/cxx/include/common/Endianness.hh
@@ -0,0 +1,52 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Endianness.hh
+ *
+ * \brief Fast IO routines for integers and fixed width floating points.
+ *
+ * \author Come Raczy
+ */
+
+#ifndef BCL2FASTQ_COMMON_ENDIANNESS_HH
+#define BCL2FASTQ_COMMON_ENDIANNESS_HH
+
+
+namespace bcl2fastq
+{
+namespace common
+{
+
+template <typename T, typename IterT> IterT extractLittleEndian(const IterT p, T &result)
+{
+#ifdef BCL2FASTQ_IS_BIG_ENDIAN
+ // TODO: untested
+ BOOST_STATIC_ASSERT(sizeof(T) / 8 == 4);
+ // this should work on any endian machine but if we're on little-endian already the alternative is probably faster
+ result = T(*p) + T(*(p + 1)) * 256 + T(*(p + 2)) * 256 * 256 + T(*(p + 3)) * 256 * 256 * 256;
+#else
+ result = *reinterpret_cast<const T*>(&*p);
+#endif
+ return p + sizeof(T);
+}
+
+template <typename T, typename IterT> T extractLittleEndian(const IterT p)
+{
+#ifdef BCL2FASTQ_IS_BIG_ENDIAN
+ // TODO: untested
+ BOOST_STATIC_ASSERT(sizeof(T) / 8 == 4);
+ // this should work on any endian machine but if we're on little-endian already the alternative is probably faster
+ return T(*p) + T(*(p + 1)) * 256 + T(*(p + 2)) * 256 * 256 + T(*(p + 3)) * 256 * 256 * 256;
+#endif
+ return *reinterpret_cast<const T*>(&*p);
+}
+
+} // namespace common
+} // namespace bcl2fastq
+
+#endif // #ifndef BCL2FASTQ_COMMON_ENDIANNESS_HH
diff --git a/src/cxx/include/common/Exceptions.hh b/src/cxx/include/common/Exceptions.hh
new file mode 100644
index 0000000..2f3a384
--- /dev/null
+++ b/src/cxx/include/common/Exceptions.hh
@@ -0,0 +1,222 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Exceptions.hh
+ *
+ * \brief Declaration of the common exception mechanism.
+ *
+ * \author Come Raczy
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_EXCEPTIONS_HH
+#define BCL2FASTQ_COMMON_EXCEPTIONS_HH
+
+
+#include <string>
+#include <stdexcept>
+#include <ios>
+
+#include <boost/exception/exception.hpp>
+// definition of BOOST_EXCEPTION_THROW macro
+// (not directly used by this file, included for user convenience)
+#include <boost/throw_exception.hpp>
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+/// \brief Virtual base class to all the exception classes in Bcl2FastQ.
+/// \note Use BOOST_THROW_EXCEPTION to get the contect info
+/// (file, function, line) at the throw site.
+class Exception : public boost::exception
+{
+public:
+
+ /// \brief Constructor.
+ /// \param errorNumber Error number.
+ /// \param message Error mesasge.
+ Exception(int errorNumber, const std::string &message);
+
+ /// \brief Copy constructor.
+ /// \param that Other exception object to copy from.
+ Exception(const Exception &that);
+
+ /// \brief Virtual destructor.
+ /// \par Exception guarantee:
+ /// no-throw
+ virtual ~Exception() throw();
+
+private:
+
+ /// \brief Restricted assignment operator.
+ Exception & operator=(const Exception &);
+
+public:
+
+ /// \brief Get error number.
+ /// \return Error number.
+ int getErrorNumber() const;
+
+ /// \brief Gete error message.
+ /// \return Error message.
+ const std::string &getMessage() const;
+
+ /// \brief Get error context.
+ /// \return Error context.
+ std::string getContext() const;
+
+private:
+
+ /// \brief Error number.
+ const int errorNumber_;
+
+ /// \brief Error message.
+ const std::string message_;
+};
+
+
+/// \brief Exception for errors presumably detectable only when the program executes.
+class RuntimeError : public Exception, public std::runtime_error
+{
+public:
+
+ /// \brief Constructor.
+ /// \param errorNumber Error number.
+ /// \param message Error message.
+ RuntimeError(int errorNumber, const std::string &message);
+
+ /// \brief Constructor.
+ /// \param message Error message.
+ explicit RuntimeError(const std::string &message);
+
+ /// \brief Copy constructor.
+ /// \param that Other exception object to copy from.
+ RuntimeError(const RuntimeError &that);
+
+public:
+
+ /// \brief Get error message.
+ /// \par Exception guarantee:
+ /// no-throw
+ virtual const char * what() const throw();
+};
+
+
+/// \brief Exception for errors presumably detectable before the program executes, such as violations of logical preconditions or class invariants.
+class LogicError : public Exception, public std::logic_error
+{
+public:
+
+ /// \brief Constructor.
+ /// \param errorNumber Error number.
+ /// \param message Error message.
+ LogicError(int errorNumber, const std::string &message);
+
+ /// \brief Constructor.
+ /// \param message Error message.
+ explicit LogicError(const std::string &message);
+
+ /// \brief Copy constructor.
+ /// \param that Other exception object to copy from.
+ LogicError(const LogicError &that);
+
+public:
+
+ /// \brief Get error message.
+ /// \par Exception guarantee:
+ /// no-throw
+ virtual const char * what() const throw();
+};
+
+
+/// \brief Exception thrown when there are problems with the IO operations.
+/// \todo Refactoring: In C++11, @c ios_base::failure is no longer derived
+/// directly from @c std::exception, but rather from std::system_error,
+/// which is it turn derived from std::runtime_error.
+class IoError: public Exception, public std::ios_base::failure
+{
+public:
+
+ /// \brief Constructor.
+ /// \param errorNumber Error number.
+ /// \param message Error message.
+ IoError(int errorNumber, const std::string &message);
+
+ /// \brief Constructor.
+ /// \param message Error message.
+ explicit IoError(const std::string &message);
+
+ /// \brief Copy constructor.
+ /// \param that Other exception object to copy from.
+ IoError(const IoError &that);
+
+public:
+
+ /// \brief Get error message.
+ /// \par Exception guarantee:
+ /// no-throw
+ virtual const char * what() const throw();
+};
+
+
+/// \brief Exception for input data errors.
+class InputDataError : public RuntimeError
+{
+public:
+
+ /// \brief Constructor.
+ /// \param errorNumber Error number.
+ /// \param message Error message.
+ InputDataError(int errorNumber, const std::string &message);
+
+ /// \brief Constructor.
+ /// \param message Error message.
+ explicit InputDataError(const std::string &message);
+
+ /// \brief Copy constructor.
+ /// \param that Other exception object to copy from.
+ InputDataError(const InputDataError &that);
+};
+
+
+/// \brief Exception for out-of-range errors.
+class OutOfRange : public LogicError
+{
+public:
+
+ /// \brief Constructor.
+ /// \param errorNumber Error number.
+ /// \param message Error message.
+ OutOfRange(int errorNumber, const std::string &message);
+
+ /// \brief Constructor.
+ /// \param message Error message.
+ explicit OutOfRange(const std::string &message);
+
+ /// \brief Copy constructor.
+ /// \param that Other exception object to copy from.
+ OutOfRange(const OutOfRange &that);
+};
+
+
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_COMMON_EXCEPTIONS_HH
+
+
diff --git a/src/cxx/include/common/FastIo.hh b/src/cxx/include/common/FastIo.hh
new file mode 100644
index 0000000..c538b1e
--- /dev/null
+++ b/src/cxx/include/common/FastIo.hh
@@ -0,0 +1,58 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FastIo.hh
+ *
+ * \brief Definition fast IO routines for integers and fixed width floating points.
+ *
+ * \author Come Raczy
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_FASTIO_HH
+#define BCL2FASTQ_COMMON_FASTIO_HH
+
+
+#include <algorithm>
+#include <limits>
+#include <vector>
+
+#include <boost/type_traits.hpp>
+#include <boost/mpl/assert.hpp>
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+/// \brief Fast and portable output of an unsigned integer into a stream.
+///
+/// Drops all the usual formatting options to the benefit of speed.
+///
+/// Can be instantiated on unsigned versions of char, short, int and
+/// long, and generally on any type supporting '<=', '%', '/=', '+'
+/// and defining 'digits10 in std::numeric_limits.
+template<typename T>
+void putUnsignedInteger(T value, std::vector<char>& copyBuffer);
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#include "common/FastIo.hpp"
+
+
+#endif // BCL2FASTQ_COMMON_FASTIO_HH
+
+
diff --git a/src/cxx/include/common/FastIo.hpp b/src/cxx/include/common/FastIo.hpp
new file mode 100644
index 0000000..67a0b54
--- /dev/null
+++ b/src/cxx/include/common/FastIo.hpp
@@ -0,0 +1,62 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FastIo.hpp
+ *
+ * \brief Implementation of fast IO routines for integers and fixed width floating points.
+ *
+ * \author Come Raczy
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_FASTIO_HPP
+#define BCL2FASTQ_COMMON_FASTIO_HPP
+
+
+#include "common/FastIo.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+template<typename T>
+void putUnsignedInteger(T value, std::vector<char>& copyBuffer)
+{
+ // generate a compilation error when instantiated on signed types
+ BOOST_MPL_ASSERT_MSG(boost::is_unsigned<T>::value,
+ SIGNED_TYPES_ARE_NOT_ALLOWED_FOR_putUnsignedInteger, (T));
+
+ char begin[1 + std::numeric_limits<T>::digits10];
+ char *buffer = begin;
+ while (value >= 10)
+ {
+ *buffer++ = '0' + (value % 10);
+ value /= 10;
+ }
+ *buffer++ = '0' + value;
+ std::reverse(begin, buffer);
+
+ size_t bufferSize = copyBuffer.size();
+ copyBuffer.resize(bufferSize + buffer-begin);
+ std::copy(begin, buffer, copyBuffer.begin() + bufferSize);
+}
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_COMMON_FASTIO_HPP
+
+
diff --git a/src/cxx/include/common/FileSystem.hh b/src/cxx/include/common/FileSystem.hh
new file mode 100644
index 0000000..8f180e1
--- /dev/null
+++ b/src/cxx/include/common/FileSystem.hh
@@ -0,0 +1,45 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FileSystem.hh
+ *
+ * file system helper utilities.
+ *
+ * \author Roman Petrovski
+ * \author Mauricio Varea
+ */
+
+#ifndef BCL2FASTQ_COMMON_FILE_SYSTEM_HH
+#define BCL2FASTQ_COMMON_FILE_SYSTEM_HH
+
+#include <vector>
+#include <boost/filesystem/path.hpp>
+
+#include "common/Debug.hh"
+
+namespace bcl2fastq
+{
+namespace common
+{
+
+void createDirectories(std::vector<boost::filesystem::path> createList);
+
+
+inline bool isDotGzPath(const boost::filesystem::path& path)
+{
+ static const char dotGz[] = {'.', 'g', 'z'};
+ return path.string().length() > sizeof(dotGz) &&
+ 0 == path.string().compare(path.string().size() - sizeof(dotGz), sizeof(dotGz), dotGz);
+}
+
+char getDirectorySeparatorChar();
+
+} //namespace common
+} //namespace bcl2fastq
+
+#endif // BCL2FASTQ_COMMON_FILE_SYSTEM_HH
diff --git a/src/cxx/include/common/InstallationPath.hh b/src/cxx/include/common/InstallationPath.hh
new file mode 100644
index 0000000..2ea7724
--- /dev/null
+++ b/src/cxx/include/common/InstallationPath.hh
@@ -0,0 +1,42 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file InstallatinPath.hh
+ *
+ * \brief Definition of InstallationPath.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_COMMON_INSTALLATION_PATH_HH
+#define BCL2FASTQ_COMMON_INSTALLATION_PATH_HH
+
+#include <boost/filesystem/path.hpp>
+
+namespace bcl2fastq
+{
+namespace common
+{
+
+class InstallationPath
+{
+private:
+ InstallationPath();
+
+ boost::filesystem::path installationRoot_;
+
+public:
+ static InstallationPath& getSingleton();
+
+ boost::filesystem::path expandPath(const boost::filesystem::path& path) const;
+};
+
+}
+}
+
+#endif // BCL2FASTQ_COMMON_INSTALLATION_PATH_HH
diff --git a/src/cxx/include/common/Logger.hh b/src/cxx/include/common/Logger.hh
new file mode 100644
index 0000000..52c7ad9
--- /dev/null
+++ b/src/cxx/include/common/Logger.hh
@@ -0,0 +1,153 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Logger.hh
+ *
+ * \brief Preprocessor-based logger mechanism.
+ *
+ * \author Roman Petrovski
+ * \author Marek Balint
+ */
+
+
+#include <ios>
+#include <string>
+
+#include <boost/atomic.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/thread.hpp>
+
+
+#ifndef BCL2FASTQ_COMMON_LOGGER_HH
+#define BCL2FASTQ_COMMON_LOGGER_HH
+
+
+/// \brief Logging stream.
+/// \param level Log level to be used.
+/// \todo Refactoring: Define separate macro for each logging level (e.g. @c BCL2FASTQ_LOG_ERROR, @c BCL2FASTQ_LOG_INFO, ...).
+#define BCL2FASTQ_LOG(level) \
+ if (::bcl2fastq::common::detail::LogStream s = ::bcl2fastq::common::detail::LogStream(level)) \
+ { \
+ } \
+ else \
+ std::clog \
+ << ::bcl2fastq::common::detail::ThreadTimestamp() \
+ /* << std::setfill(' ') << std::setw(7) */ << level << (level ? ": " : "") \
+ << ( \
+ /*(level == ::bcl2fastq::common::LogLevel::TRACE) \
+ ? \
+ std::string(__FILE__) + ":" \
+ + boost::lexical_cast<std::string>(__LINE__) + ": " \
+ :*/ \
+ "" \
+ ) \
+ /* no semicolon (';') here */
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+/// \brief Logging level.
+struct LogLevel
+{
+ enum value_type
+ {
+ NONE = 0, ///< Suppress all logging.
+ FATAL = 1, ///< Fatal errors only.
+ ERROR_TYPE = 2, ///< All errors.
+ WARNING = 3, ///< Warnings and all errors.
+ INFO = 4, ///< Informative messages and warnings/errors.
+ DEBUG = 5, ///< Debug messages, informative messages and warnings/errors.
+ TRACE = 6 ///< Log everything.
+ };
+};
+
+/// \brief Input operator for logging level.
+/// \param is Input stream.
+/// \param l Loging level to be input.
+/// \return Input stream.
+std::istream& operator>>(std::istream& is, LogLevel::value_type &l);
+
+/// \brief Output operator for logging level.
+/// \param os Output stream.
+/// \param l Loging level to be output.
+/// \return Output stream.
+std::ostream& operator<<(std::ostream& os, LogLevel::value_type l);
+
+/// \brief Minimum log level of the program.
+extern LogLevel::value_type BCL2FASTQ_MIN_LOG_LEVEL;
+
+
+namespace detail {
+
+/// \brief Stream for logging.
+class LogStream
+{
+public:
+
+ /// \brief Default constructor.
+ LogStream();
+
+ /// \brief Constructor.
+ /// \param logLevel Log level to be used.
+ explicit LogStream(LogLevel::value_type logLevel);
+
+ /// \brief Copy constructor.
+ /// \param that The other instance.
+ LogStream(const LogStream &that);
+
+ /// \brief Conversion operator.
+ operator bool () const;
+
+ static unsigned int getNumWarnings() { return numWarnings_; }
+ static unsigned int getNumErrors() { return numErrors_; }
+
+private:
+
+ /// \brief Mutex.
+ static boost::recursive_mutex clogMutex_;
+ static unsigned int numWarnings_;
+ static unsigned int numErrors_;
+
+ /// \brief Lock guard.
+ boost::lock_guard<boost::recursive_mutex> lock_;
+
+ /// \brief Log stream saved state.
+ boost::io::ios_base_all_saver ias_;
+
+ /// \brief Log level to be used.
+ LogLevel::value_type logLevel_;
+};
+
+
+/// \brief Thread timestamp tag for output operator overload.
+struct ThreadTimestamp
+{
+};
+
+/// \brief Formats time stamp and thread id to simplify threaded logging.
+/// \param os Output stream.
+/// \return Output stream.
+std::ostream & operator << (std::ostream &os, const ThreadTimestamp &);
+
+
+} // namespace detail
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_COMMON_LOGGER_HH
+
+
diff --git a/src/cxx/include/common/Options.hh b/src/cxx/include/common/Options.hh
new file mode 100644
index 0000000..d7e60f7
--- /dev/null
+++ b/src/cxx/include/common/Options.hh
@@ -0,0 +1,141 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Options.hh
+ *
+ * \brief Declaration of the program options processing.
+ *
+ * \author Come Raczy
+ * \author Mauricio Varea
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_OPTIONS_HH
+#define BCL2FASTQ_COMMON_OPTIONS_HH
+
+
+#include <boost/utility.hpp>
+#include <boost/program_options.hpp>
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+/// \brief Encapsulation of the processing of the command line options.
+/// \todo Features: Add config file and environment options.
+class Options : private boost::noncopyable
+{
+
+public:
+
+ /// \brief Program action.
+ struct Action
+ {
+ enum value_type
+ {
+ RUN, ///< Run.
+ HELP, ///< Display help and exit.
+ VERSION, ///< Display version and exit.
+ ABORT ///< Abort.
+ };
+ };
+
+public:
+
+ /// \brief Default constructor.
+ Options();
+
+ /// \brief Virtual destructor.
+ virtual ~Options() = 0;
+
+private:
+
+ /// \brief Initialize program options.
+ void initOptions();
+
+protected:
+
+ /// \brief Template method: initialize named options.
+ virtual void initNamedOptions(boost::program_options::options_description &options) = 0;
+
+ /// \brief Template method: initialize unnamed options.
+ virtual void initUnnamedOptions(boost::program_options::options_description &options) = 0;
+
+ /// \brief Tmeplate method: initialize positional options.
+ virtual void initPositionalOptions(boost::program_options::positional_options_description &options) = 0;
+
+public:
+
+ /// \brief Get program usage help.
+ /// \return String containing program usage help.
+ std::string usage() const;
+
+protected:
+
+ /// \brief Get program usage prefix.
+ /// \return String containing program usage prefix.
+ virtual std::string usagePrefix() const = 0;
+
+ /// \brief Get program usage suffix.
+ /// \return String containing program usage suffix.
+ virtual std::string usageSuffix() const;
+
+public:
+
+ /// \brief Parse command line arguments.
+ /// \param argc Number of command line arguments.
+ /// \param argv Command line arguments.
+ /// \return Action to be taked based on arguments.
+ Options::Action::value_type parse(int argc, char *argv[]);
+
+protected:
+
+ /// \brief Store the options from the sample sheet.
+ /// \param vm Options map where the settings are stored.
+ virtual void storeSampleSheetSettings(boost::program_options::variables_map &vm) = 0;
+
+ /// \brief Post-process parsed command line arguments.
+ /// \param vm Command line arguments to be post-process.
+ virtual void postProcess(boost::program_options::variables_map &vm) = 0;
+
+private:
+
+ /// \brief Flag determining whether option description has been initialized.
+ bool optionsInitialized_;
+
+ /// \brief Named options.
+ boost::program_options::options_description namedOptions_;
+
+ /// \brief Unnamed options.
+ boost::program_options::options_description unnamedOptions_;
+
+ /// \brief Positional options.
+ boost::program_options::positional_options_description positionalOptions_;
+};
+
+
+/// \brief Output operator for program Action.
+/// \param os Output stream.
+/// \param a Program action to be output.
+/// \return Output stream.
+std::ostream& operator<<(std::ostream& os, Options::Action::value_type a);
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_COMMON_OPTIONS_HH
+
+
diff --git a/src/cxx/include/common/Program.hh b/src/cxx/include/common/Program.hh
new file mode 100644
index 0000000..1170dfa
--- /dev/null
+++ b/src/cxx/include/common/Program.hh
@@ -0,0 +1,50 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Program.hh
+ *
+ * \brief Declaration of the skeleton of all c++ programs.
+ *
+ * \author Come Raczy
+ * \author Roman Petrovski
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_PROGRAM_HH
+#define BCL2FASTQ_COMMON_PROGRAM_HH
+
+
+#include <string>
+
+#include "common/ProgramInfo.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+/// \brief Unified behavior of all programs.
+template<typename O>
+void run(int(*callback)(const ProgramInfo &, O &), int argc, char *argv[]);
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#include "common/Program.hpp"
+
+
+#endif // BCL2FASTQ_COMMON_PROGRAM_HH
+
+
diff --git a/src/cxx/include/common/Program.hpp b/src/cxx/include/common/Program.hpp
new file mode 100644
index 0000000..ef413fb
--- /dev/null
+++ b/src/cxx/include/common/Program.hpp
@@ -0,0 +1,136 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Program.hpp
+ *
+ * \brief Implementation of the skeleton of all c++ programs.
+ *
+ * \author Come Raczy
+ * \author Mauricio Varea
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_PROGRAM_HPP
+#define BCL2FASTQ_COMMON_PROGRAM_HPP
+
+
+#include <iostream>
+#include <sstream>
+#include <algorithm>
+#include <string>
+#include <cstdlib>
+
+#include <boost/cstdlib.hpp>
+#include <boost/program_options.hpp>
+
+#include "common/ProgramInfo.hh"
+#include "common/Exceptions.hh"
+#include "common/Logger.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+template<typename O>
+void run(void(*callback)(const ProgramInfo &, O &), int argc, char *argv[])
+{
+#ifndef WIN32
+ setenv("LC_ALL", "C", 1);
+#endif
+
+ try {
+ ProgramInfo info(argv[0], boost::program_options::options_description::m_default_line_length);
+ std::clog << info << std::endl;
+
+ O options;
+ const typename O::Action::value_type action = options.parse(argc, argv);
+
+ switch(action) {
+ case O::Action::RUN:
+ {
+ callback(info, options);
+
+ BCL2FASTQ_LOG(LogLevel::NONE) << "Processing completed with " << ::bcl2fastq::common::detail::LogStream::getNumErrors()
+ << " errors and " << ::bcl2fastq::common::detail::LogStream::getNumWarnings()
+ << " warnings." << std::endl;
+ break;
+ }
+ case O::Action::HELP:
+ {
+ std::clog << options.usage();
+ std::exit(boost::exit_success);
+ }
+ case O::Action::VERSION:
+ {
+ std::exit(boost::exit_success);
+ }
+ case O::Action::ABORT:
+ {
+ std::exit(boost::exit_failure);
+ }
+ }
+ }
+ catch (const Exception &e)
+ {
+ BCL2FASTQ_LOG(LogLevel::ERROR_TYPE)
+ << "bcl2fastq::common::Exception: "
+ << e.getContext()
+ << std::endl
+ ;
+ std::exit(boost::exit_failure);
+ }
+ catch (const boost::exception &e)
+ {
+ BCL2FASTQ_LOG(LogLevel::ERROR_TYPE)
+ << "boost::exception: "
+ << boost::diagnostic_information(e)
+ << std::endl
+ ;
+ std::exit(boost::exit_failure);
+ }
+ catch (const std::runtime_error &e)
+ {
+ BCL2FASTQ_LOG(LogLevel::ERROR_TYPE)
+ << "std::runtime_error: "
+ << e.what()
+ << std::endl
+ ;
+ std::exit(boost::exit_failure);
+ }
+ catch (const std::logic_error &e)
+ {
+ BCL2FASTQ_LOG(LogLevel::ERROR_TYPE)
+ << "std::logic_error: "
+ << e.what()
+ << std::endl
+ ;
+ std::exit(boost::exit_failure);
+ }
+ catch(...) {
+ BCL2FASTQ_LOG(LogLevel::FATAL)
+ << "Unknown exception"
+ << std::endl
+ ;
+ std::exit(boost::exit_failure);
+ }
+}
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_COMMON_PROGRAM_HH
+
+
diff --git a/src/cxx/include/common/ProgramInfo.hh b/src/cxx/include/common/ProgramInfo.hh
new file mode 100644
index 0000000..787211b
--- /dev/null
+++ b/src/cxx/include/common/ProgramInfo.hh
@@ -0,0 +1,88 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ProgramInfo.hh
+ *
+ * \brief Basic program information.
+ *
+ * \author Mauricio Varea
+ * \author Marek Balint
+ */
+
+#ifndef BCL2FASTQ_COMMON_PROGRAMINFO_HH
+#define BCL2FASTQ_COMMON_PROGRAMINFO_HH
+
+
+#include <string>
+#include <iosfwd>
+
+#include <boost/filesystem.hpp>
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+/// \brief Program information.
+class ProgramInfo
+{
+
+public:
+
+ /// \brief Constructor.
+ /// \param cmd Command used to invoke the program (argv[0]).
+ /// \param width Line width (used for output).
+ ProgramInfo(const std::string &cmd, unsigned int width);
+
+public:
+
+ /// \brief Command used to invoke the program (argv[0]).
+ const boost::filesystem::path command;
+
+ /// \brief Name of binary executable.
+ const std::string binaryName;
+
+ /// \brief Short project name (one word).
+ const std::string projectNameShort;
+
+ /// \brief Long project name (descriptive sentence).
+ const std::string projectNameLong;
+
+ /// \brief Copyright string.
+ const std::string copyright;
+
+ /// \brief Version string (NN.YY.MM.DD)
+ const std::string version;
+
+private:
+
+ /// \brief Line width (used for output).
+ const unsigned int width_;
+
+ friend std::ostream& operator<<(std::ostream& os, ProgramInfo pi);
+};
+
+
+/// \brief Output operator for ProgramInfo.
+/// \param os Output stream.
+/// \param pi Program info to be output.
+/// \return Output stream.
+std::ostream& operator<<(std::ostream& os, ProgramInfo pi);
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_COMMON_PROGRAMINFO_HH
+
+
diff --git a/src/cxx/include/common/SampleMetadata.hh b/src/cxx/include/common/SampleMetadata.hh
new file mode 100644
index 0000000..a907fc9
--- /dev/null
+++ b/src/cxx/include/common/SampleMetadata.hh
@@ -0,0 +1,69 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Types.hh
+ *
+ * \brief Declaration of common data types
+ *
+ * \author Marek Balint
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_SAMPLE_METADATA_HH
+#define BCL2FASTQ_COMMON_SAMPLE_METADATA_HH
+
+
+#include "common/Types.hh"
+
+#include <string>
+#include <vector>
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+/// \brief Sample meta data.
+struct SampleMetadata
+{
+public:
+
+ /// \brief Sample barcodes container type definition.
+ typedef std::vector<std::vector<std::string> > BarcodesContainer;
+
+public:
+
+ /// \brief Sample ID.
+ std::string id_;
+
+ /// \brief Sample name.
+ std::string name_;
+
+ /// \brief Project name.
+ std::string project_;
+
+ /// \brief Lane number.
+ std::vector<common::LaneNumber> lanes_;
+
+ /// \brief Sample barcodes.
+ BarcodesContainer barcodes_;
+};
+
+/// \brief Sample metadta container type definition.
+typedef std::vector<SampleMetadata> SampleMetadataContainer;
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_COMMON_SAMPLE_METADATA_HH
+
+
diff --git a/src/cxx/include/common/StaticMemPool.hh b/src/cxx/include/common/StaticMemPool.hh
new file mode 100644
index 0000000..32dffe6
--- /dev/null
+++ b/src/cxx/include/common/StaticMemPool.hh
@@ -0,0 +1,91 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file StaticMemPool.hh
+ *
+ * \brief Declaration of simple static memory pool.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_STATICMEMPOOL_HH
+#define BCL2FASTQ_COMMON_STATICMEMPOOL_HH
+
+
+#include <cstddef>
+
+#include <boost/utility.hpp>
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+/// \brief Simple static memory pool.
+template<std::size_t PageSize, std::size_t PageCount>
+class StaticMemPool
+{
+public:
+
+ /// \brief Size type.
+ typedef std::size_t size_type;
+
+public:
+
+ /// \brief Default constructor.
+ StaticMemPool();
+
+ /// \brief Copy constructor.
+ /// \param that Other instance to copy from.
+ StaticMemPool(const StaticMemPool<PageSize, PageCount> &that);
+
+ /// \brief Assignment operator.
+ /// \param rhs Right-hand-side parameter.
+ StaticMemPool & operator=(const StaticMemPool<PageSize, PageCount> &rhs);
+
+ /// \brief Destructor.
+ ~StaticMemPool();
+
+public:
+
+ /// \brief Allocate memory from pool.
+ /// \param size Size of memory to be allocated.
+ /// \return Pointer to allocated memory or @c NULL in case of error.
+ void * allocate(size_type size);
+
+ /// \brief Free previously allocated memory.
+ /// \param ptr Pointer to memory previously allocated by this pool instanc.
+ void deallocate(void *ptr);
+
+private:
+
+ /// \brief Memory pool.
+ char pool_[PageCount][PageSize];
+
+ /// \brief Memory pool metadata.
+ /// \note Currently only boolean flag determining whether the given page
+ /// is free (@c true) or not (@c false).
+ bool metadata_[PageCount];
+};
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#include "common/StaticMemPool.hpp"
+
+
+#endif // BCL2FASTQ_COMMON_STATICMEMPOOL_HH
+
+
diff --git a/src/cxx/include/common/StaticMemPool.hpp b/src/cxx/include/common/StaticMemPool.hpp
new file mode 100644
index 0000000..9558bc0
--- /dev/null
+++ b/src/cxx/include/common/StaticMemPool.hpp
@@ -0,0 +1,118 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file StaticMemPool.hpp
+ *
+ * \brief Implementation of simple static memory pool.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_STATICMEMPOOL_HPP
+#define BCL2FASTQ_COMMON_STATICMEMPOOL_HPP
+
+
+#include <algorithm>
+
+#include <cstring>
+
+#include "common/Debug.hh"
+#include "common/Logger.hh"
+#include "common/StaticMemPool.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+template<std::size_t PageSize, std::size_t PageCount>
+StaticMemPool<PageSize, PageCount>::StaticMemPool()
+: pool_()
+, metadata_()
+{
+ std::fill(&metadata_[0], &metadata_[PageCount], true);
+}
+
+template<std::size_t PageSize, std::size_t PageCount>
+StaticMemPool<PageSize, PageCount>::StaticMemPool(const StaticMemPool<PageSize, PageCount> &that)
+: pool_()
+, metadata_()
+{
+ std::copy(&that.pool_[0][0], &that.pool[PageCount-1][PageSize], &pool_[0][0]);
+ std::copy(&that.metadata_[0], &that.metadata_[PageCount], &metadata_[0]);
+}
+
+template<std::size_t PageSize, std::size_t PageCount>
+StaticMemPool<PageSize, PageCount> & StaticMemPool<PageSize, PageCount>::operator=(const StaticMemPool<PageSize, PageCount> &rhs)
+{
+ if (this != &rhs)
+ {
+ std::copy(&rhs.pool_[0][0], &rhs.pool[PageCount-1][PageSize], &pool_[0][0]);
+ std::copy(&rhs.metadata_[0], &rhs.metadata_[PageCount], &metadata_[0]);
+ }
+ return *this;
+}
+
+template<std::size_t PageSize, std::size_t PageCount>
+StaticMemPool<PageSize, PageCount>::~StaticMemPool()
+{
+}
+
+template<std::size_t PageSize, std::size_t PageCount>
+void * StaticMemPool<PageSize, PageCount>::allocate(typename StaticMemPool<PageSize, PageCount>::size_type size)
+{
+ if (size > PageSize)
+ {
+ return NULL;
+ }
+
+ bool *freePage = std::find(&metadata_[0], &metadata_[PageCount], true);
+ if (freePage == &metadata_[PageCount])
+ {
+ return NULL;
+ }
+
+ size_type freePageIdx = freePage - &metadata_[0];
+ void *ret = static_cast<void *>(&pool_[freePageIdx][0]);
+ metadata_[freePageIdx] = false;
+
+ return ret;
+}
+
+template<std::size_t PageSize, std::size_t PageCount>
+void StaticMemPool<PageSize, PageCount>::deallocate(void *ptr)
+{
+ char *char_ptr = static_cast<char *>(ptr);
+ BCL2FASTQ_ASSERT_MSG(char_ptr >= &pool_[0][0], "Invalid pointer: " << ptr);
+ BCL2FASTQ_ASSERT_MSG(char_ptr < &pool_[PageCount][0], "Invalid pointer: " << ptr);
+ BCL2FASTQ_ASSERT_MSG((char_ptr - &pool_[0][0]) % PageSize == 0, "Invalid pointer: " << ptr);
+
+ size_type pageIdx = (char_ptr - &pool_[0][0]) / PageSize;
+ if (metadata_[pageIdx] == true)
+ {
+ BCL2FASTQ_LOG(LogLevel::DEBUG) << "Memory already free: " << ptr << std::endl;
+ return;
+ }
+ metadata_[pageIdx] = true;
+}
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_COMMON_STATICMEMPOOL_HPP
+
+
+
+
diff --git a/src/cxx/include/common/SystemCompatibility.hh b/src/cxx/include/common/SystemCompatibility.hh
new file mode 100644
index 0000000..238b6c1
--- /dev/null
+++ b/src/cxx/include/common/SystemCompatibility.hh
@@ -0,0 +1,54 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file SystemCompatibility.hh
+ *
+ * \brief Interface layer for system-dependent functionalities.
+ *
+ * \author Come Raczy
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_SYSTEMCOMPATIBILITY_HH
+#define BCL2FASTQ_COMMON_SYSTEMCOMPATIBILITY_HH
+
+#include <thread>
+
+namespace bcl2fastq {
+namespace common {
+
+
+/// \brief Generate a core dump with a meaningful backtrace.
+/// \note This function is guaranteed to never return.
+void terminateWithCoreDump();
+
+/// \brief Determine architecture's endianness.
+/// \retval true Little endian.
+/// \retval false Big endian.
+bool isLittleEndian();
+
+/// \brief Swap endianness.
+/// \param value Value which endianness is to be swapped.
+/// \return Value with swapped endianness.
+template<typename T>
+T swapEndian(T value);
+
+/// \brief Sets the max number of file handles
+void adjustMaxFileHandles();
+
+} // namespace common
+} // namespace bcl2fastq
+
+
+#include "common/SystemCompatibility.hpp"
+
+
+#endif // BCL2FASTQ_COMMON_SYSTEMCOMPATIBILITY_HH
+
+
diff --git a/src/cxx/include/common/SystemCompatibility.hpp b/src/cxx/include/common/SystemCompatibility.hpp
new file mode 100644
index 0000000..3fa283b
--- /dev/null
+++ b/src/cxx/include/common/SystemCompatibility.hpp
@@ -0,0 +1,60 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file SystemCompatibility.hpp
+ *
+ * \brief Interface layer for system-dependent functionalities.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_SYSTEMCOMPATIBILITY_HPP
+#define BCL2FASTQ_COMMON_SYSTEMCOMPATIBILITY_HPP
+
+
+#include <algorithm>
+#include <iterator>
+
+#include "common/SystemCompatibility.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+template<typename T>
+T swapEndian(T value)
+{
+ union {
+ T value;
+ unsigned char bytes[sizeof(T)];
+ } src, dst;
+
+ src.value = value;
+ std::copy(
+ &src.bytes[0],
+ &src.bytes[sizeof(T)],
+ std::reverse_iterator<unsigned char *>(&dst.bytes[sizeof(T)-1])
+ );
+
+ return dst.value;
+}
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_COMMON_SYSTEMCOMPATIBILITY_HPP
+
+
diff --git a/src/cxx/include/common/Threads.hh b/src/cxx/include/common/Threads.hh
new file mode 100644
index 0000000..24fb380
--- /dev/null
+++ b/src/cxx/include/common/Threads.hh
@@ -0,0 +1,199 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Threads.hh
+ *
+ * \brief Definition of helpers for thread management.
+ *
+ * \author Roman Petorvski
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_THREADS_HH
+#define BCL2FASTQ_COMMON_THREADS_HH
+
+
+#include <cstddef>
+
+#include <thread>
+#include <boost/noncopyable.hpp>
+#include <boost/ptr_container/ptr_vector.hpp>
+#include <boost/thread/mutex.hpp>
+#include <boost/thread/condition_variable.hpp>
+#include <boost/exception_ptr.hpp>
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+/// \brief Inversion of the boost::lock_guard.
+/// \todo Refactoring: Use Boost::ASIO: http://mostlycoding.blogspot.de/2009/05/asio-library-has-been-immensely-helpful.html
+template<typename Mutex>
+class unlock_guard : private boost::noncopyable
+{
+public:
+
+ /// \brief Constructor.
+ /// \param m Mutex.
+ explicit unlock_guard(Mutex& m);
+
+ /// \brief Destructor.
+ ~unlock_guard();
+
+private:
+
+ /// \brief Mutex.
+ Mutex& m_;
+};
+
+
+/// \brief Thread pool.
+template<bool CrashOnException>
+class BasicThreadVector : boost::noncopyable
+{
+public:
+
+ /// \brief Size type definition.
+ typedef std::size_t size_type;
+
+public:
+
+ /// \brief Constructor.
+ /// \param size Number of threads (a.k.a pool size).
+ explicit BasicThreadVector(size_type size);
+
+ /// \brief Destructor.
+ ~BasicThreadVector();
+
+public:
+
+ /// \brief Get pool size.
+ /// \return Pool size.
+ size_type size() const;
+
+public:
+
+ /// \brief Execute given function on given number of threads.
+ /// \param function The function to be executed.
+ /// \param threads Number of threads to be used.
+ /// \pre <tt>threads <= this->size()</tt>
+ template<typename FunctionT>
+ void execute(FunctionT function, size_type threads);
+
+ /// \brief Execute given function on all threads.
+ /// \param function The function to be executed.
+ template<typename FunctionT>
+ void execute(FunctionT function);
+
+private:
+
+ /// \brief Signal threads to execute workload.
+ /// \param threads Number of threads to use for workload execution.
+ void cycle(size_type threads);
+
+ /// \brief Wait for all threads to finish their current task.
+ /// \param lock Synchronisation lock.
+ void waitAll(boost::unique_lock<boost::mutex> &lock);
+
+ /// \brief Thread function for individual threads.
+ /// \param threadNum Thread number (0-based).
+ void threadFunction(size_type threadNum);
+
+ /// \brief Execute workload by the thread.
+ /// \param threadNum Thread number (0-based).
+ /// \param lock Synchronisation lock.
+ /// \note No exception handling.
+ void unsafeExecute(size_type threadNum, boost::unique_lock<boost::mutex> &lock);
+
+ /// \brief Execute workload by the thread.
+ /// \param threadNum Thread number (0-based).
+ /// \param lock Synchronisation lock.
+ /// \note With exception handling (exception thrown by the worker thread
+ /// will be rethrown on calling thread).
+ void safeExecute(size_type threadNum, boost::unique_lock<boost::mutex> &lock);
+
+private:
+
+ /// \brief Thread workload executor.
+ class Executor
+ {
+ public:
+
+ /// \brief Virtual destructor.
+ virtual ~Executor() {}
+
+ public:
+
+ /// \brief Execute thread workload.
+ /// \param threadNum Thread number (0-based).
+ virtual void execute(size_type threadNum) = 0;
+ };
+
+private:
+
+ /// \brief Container of threads.
+ boost::ptr_vector<std::thread> pool_;
+
+ /// \brief Thread synchronisation mutex.
+ boost::mutex mutex_;
+
+ /// \brief Thread synchronisation condition variable.
+ boost::condition_variable stateChangedCondition_;
+
+ /// \brief Current thread workload executor.
+ Executor *executor_;
+
+ /// \brief Number of threads currently processing the request.
+ size_type busyThreads_;
+
+ /// \brief Number of threads still required to process the request
+ size_type neededThreads_;
+
+ /// \brief Lowest idle thread
+ /// \note When executing with less threads than available,
+ /// this prevents the higher number threads to carry out the request.
+ size_type lowestBlockedThreadNumber_;
+
+ /// \brief True when the whole thing goes down.
+ bool terminateRequested_;
+
+ /// \brief Number of the current request.
+ /// \note Constantly-incrementing number to make sure
+ /// each thread processes one master call only once.
+ std::size_t currentRequest_;
+
+ /// \brief Exception thrown by worker thread.
+ boost::exception_ptr firstThreadException_;
+};
+
+/// \brief Thread pool with exception handling type definition.
+typedef BasicThreadVector<false> SafeThreadVector;
+
+/// \brief Thread pool without exception handling type definition.
+typedef BasicThreadVector<true> UnsafeThreadVector;
+
+/// \brief Thread pool type definition.
+typedef SafeThreadVector ThreadVector;
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#include "common/Threads.hpp"
+
+
+#endif // BCL2FASTQ_COMMON_THREADS_HH
+
+
diff --git a/src/cxx/include/common/Threads.hpp b/src/cxx/include/common/Threads.hpp
new file mode 100644
index 0000000..5ab0940
--- /dev/null
+++ b/src/cxx/include/common/Threads.hpp
@@ -0,0 +1,271 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Threads.hpp
+ *
+ * \brief Implementation of helpers for thread management.
+ *
+ * \author Roman Petorvski
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_THREADS_HPP
+#define BCL2FASTQ_COMMON_THREADS_HPP
+
+
+#include <algorithm>
+
+#include <boost/bind.hpp>
+#include <boost/thread/locks.hpp>
+
+#include "common/SystemCompatibility.hh"
+#include "common/Logger.hh"
+#include "common/Debug.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+template<typename Mutex>
+unlock_guard<Mutex>::unlock_guard(Mutex& m)
+: m_(m)
+{
+ m_.unlock();
+}
+
+template<typename Mutex>
+unlock_guard<Mutex>::~unlock_guard()
+{
+ m_.lock();
+}
+
+
+template<bool CrashOnException>
+BasicThreadVector<CrashOnException>::BasicThreadVector(typename BasicThreadVector<CrashOnException>::size_type size)
+: pool_()
+, mutex_()
+, stateChangedCondition_()
+, executor_(NULL)
+, busyThreads_(size)
+, neededThreads_(0)
+, lowestBlockedThreadNumber_(0)
+, terminateRequested_(false)
+, currentRequest_(0)
+, firstThreadException_()
+{
+ BCL2FASTQ_ASSERT_MSG(size > 0, "Inadequate pool size: " << size);
+ pool_.reserve(size);
+
+ boost::unique_lock<boost::mutex> lock(mutex_);
+ while (size--)
+ {
+ pool_.push_back(new std::thread(
+ &BasicThreadVector<CrashOnException>::threadFunction, this, size)
+ );
+ }
+ // Initial wait for all threads to initialize their 'processedRequest'
+ this->waitAll(lock);
+}
+
+template<bool CrashOnException>
+BasicThreadVector<CrashOnException>::~BasicThreadVector()
+{
+ {
+ /// \todo Refactoring: Locking is not really needed here, but it helps the assert to be a bit more definite.
+ boost::unique_lock<boost::mutex> lock(mutex_);
+ BCL2FASTQ_ASSERT_MSG(!busyThreads_, "Workers must not be running at this point");
+
+ terminateRequested_ = true;
+ stateChangedCondition_.notify_all();
+ }
+
+ std::for_each(
+ pool_.begin(),
+ pool_.end(),
+ boost::bind(&std::thread::join, _1)
+ );
+}
+
+template<bool CrashOnException>
+typename BasicThreadVector<CrashOnException>::size_type BasicThreadVector<CrashOnException>::size() const
+{
+ return pool_.size();
+}
+
+template<bool CrashOnException>
+template<typename FunctionT>
+void BasicThreadVector<CrashOnException>::execute(FunctionT function, size_type threads)
+{
+ BCL2FASTQ_ASSERT_MSG(threads <= pool_.size(), "Request must not exceed the amount of threads available");
+ BCL2FASTQ_ASSERT_MSG(executor_ == NULL, "Queueing is not supported");
+
+ class FunctionExecutor : public Executor
+ {
+ public:
+
+ FunctionExecutor(FunctionT &function)
+ : function_(function)
+ {
+ }
+
+ public:
+
+ virtual void execute(BasicThreadVector<CrashOnException>::size_type threadNum)
+ {
+ function_(threadNum);
+ }
+
+ private:
+
+ FunctionT &function_;
+
+ } executor(function);
+
+ executor_ = &executor;
+ this->cycle(threads);
+ executor_ = 0;
+}
+
+template<bool CrashOnException>
+template<typename FunctionT>
+void BasicThreadVector<CrashOnException>::execute(FunctionT function)
+{
+ this->execute(function, pool_.size());
+}
+
+template<bool CrashOnException>
+void BasicThreadVector<CrashOnException>::cycle(size_type threads)
+{
+ boost::unique_lock<boost::mutex> lock(mutex_);
+ BCL2FASTQ_ASSERT_MSG(busyThreads_ == 0, "Only one at a time outstanding request is allowed");
+
+ firstThreadException_ = boost::exception_ptr();
+ if (threads == 1)
+ {
+ // Special case for one to simplify debugging. Just do it on the calling thread.
+ executor_->execute(0);
+ }
+ else
+ {
+ lowestBlockedThreadNumber_ = threads;
+ neededThreads_ = threads;
+ ++currentRequest_;
+ stateChangedCondition_.notify_all();
+ this->waitAll(lock);
+ }
+ if (firstThreadException_)
+ {
+ BCL2FASTQ_LOG(LogLevel::WARNING) << "Rethrowing a thread exception " << std::endl;
+ boost::rethrow_exception(firstThreadException_);
+ }
+}
+
+template<bool CrashOnException>
+void BasicThreadVector<CrashOnException>::waitAll(boost::unique_lock<boost::mutex> &lock)
+{
+ while (busyThreads_ || neededThreads_)
+ {
+ stateChangedCondition_.wait(lock);
+ }
+}
+
+template<bool CrashOnException>
+void BasicThreadVector<CrashOnException>::threadFunction(size_type threadNum)
+{
+ BCL2FASTQ_LOG(LogLevel::DEBUG) << "Thread #" << threadNum << " created" << std::endl;
+
+ boost::unique_lock<boost::mutex> lock(mutex_);
+ while (!terminateRequested_)
+ {
+ BCL2FASTQ_ASSERT_MSG(busyThreads_ > 0, "Thread is not accounted for!!!");
+ --busyThreads_;
+
+ const std::size_t processedRequest = currentRequest_;
+
+ //BCL2FASTQ_LOG(LogLevel::DEBUG) << "Thread #" << threadNum << " waiting for new request" << std::endl;
+ stateChangedCondition_.notify_all();
+ while (!terminateRequested_ && processedRequest == currentRequest_ )
+ {
+ stateChangedCondition_.wait(lock);
+ }
+ ++busyThreads_;
+
+ if (!terminateRequested_)
+ {
+ //BCL2FASTQ_LOG(LogLevel::DEBUG) << "Thread #" << threadNum << " unblocked" << std::endl;
+ if (lowestBlockedThreadNumber_ > threadNum)
+ {
+ BCL2FASTQ_ASSERT_MSG(neededThreads_, "If thread is allowed to run, there must be a need for it!");
+ --neededThreads_;
+
+ if (CrashOnException)
+ {
+ this->unsafeExecute(threadNum, lock);
+ }
+ else
+ {
+ this->safeExecute(threadNum, lock);
+ }
+ // we're back under lock
+ }
+ }
+ }
+
+ BCL2FASTQ_LOG(LogLevel::DEBUG) << "Thread #" << threadNum << " terminated" << std::endl;
+}
+
+template<bool CrashOnException>
+void BasicThreadVector<CrashOnException>::unsafeExecute(size_type threadNum, boost::unique_lock<boost::mutex> &lock)
+{
+ unlock_guard<boost::unique_lock<boost::mutex> > unlock(lock);
+ executor_->execute(threadNum);
+}
+
+template<bool CrashOnException>
+void BasicThreadVector<CrashOnException>::safeExecute(size_type threadNum, boost::unique_lock<boost::mutex> &lock)
+{
+ try
+ {
+ this->unsafeExecute(threadNum, lock);
+ }
+ catch (...)
+ {
+ if (!firstThreadException_)
+ {
+ firstThreadException_ = boost::current_exception();
+ BCL2FASTQ_LOG(LogLevel::ERROR_TYPE)
+ << "Thread: " << threadNum
+ << " caught an exception first: " << boost::current_exception_diagnostic_information()
+ << std::endl
+ ;
+ }
+ else
+ {
+ BCL2FASTQ_LOG(LogLevel::ERROR_TYPE)
+ << "Thread: " << threadNum
+ << " also caught an exception: " << boost::current_exception_diagnostic_information()
+ << std::endl
+ ;
+ }
+ }
+}
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_COMMON_THREADS_HPP
+
+
diff --git a/src/cxx/include/common/Timer.hh b/src/cxx/include/common/Timer.hh
new file mode 100644
index 0000000..32ca427
--- /dev/null
+++ b/src/cxx/include/common/Timer.hh
@@ -0,0 +1,53 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Timer.hh
+ *
+ * \brief Declaration of common data types
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_TIMER_HH
+#define BCL2FASTQ_COMMON_TIMER_HH
+
+
+#include <chrono>
+
+#include <boost/noncopyable.hpp>
+
+namespace bcl2fastq {
+namespace common {
+
+class Timer : private boost::noncopyable
+{
+public:
+ Timer(size_t& timeInMicroseconds) :
+ timeInMicroseconds_(timeInMicroseconds),
+ startTime_(std::chrono::steady_clock::now())
+ {
+ }
+
+ ~Timer()
+ {
+ timeInMicroseconds_ += std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - startTime_).count();
+ }
+
+private:
+ size_t& timeInMicroseconds_;
+ std::chrono::time_point<std::chrono::steady_clock> startTime_;
+};
+
+} // namespace common
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_DATA_TIMER_HH
+
+
diff --git a/src/cxx/include/common/Types.hh b/src/cxx/include/common/Types.hh
new file mode 100644
index 0000000..449834d
--- /dev/null
+++ b/src/cxx/include/common/Types.hh
@@ -0,0 +1,143 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Types.hh
+ *
+ * \brief Declaration of common data types
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_TYPES_HH
+#define BCL2FASTQ_COMMON_TYPES_HH
+
+#include "common/Debug.hh"
+#include <boost/filesystem.hpp>
+#include <cstddef>
+#include <stdint.h>
+#include <ostream>
+#include <map>
+#include <vector>
+
+namespace bcl2fastq {
+namespace common {
+
+enum class NumBasesPerByte : uint32_t { ONE=1, TWO=2, FOUR=4 };
+
+/// \brief Read type definition.
+enum ReadType
+{
+ DATA,
+ INDEX,
+ UMI
+};
+
+
+/// \brief Raw data buffer type definition.
+class RawDataBuffer : public std::vector<char>
+{
+public:
+ RawDataBuffer() : std::vector<char>() { }
+ RawDataBuffer(size_t sz) : std::vector<char>(sz) { }
+
+ virtual ~RawDataBuffer() { }
+
+ boost::filesystem::path path_;
+};
+
+template<typename T>
+void resizeAndRealloc(T& container, size_t size)
+{
+/* if (container.capacity() < size)
+ {
+ // We need to allocate memory. Reserve 20% more than we need to reduce the
+ // liklihood we will need to do this again next time.
+ container.reserve(size*1.2);
+ }
+*/
+ container.resize(size);
+}
+
+template<typename T, typename TYPE>
+void resizeAndRealloc(T& container, size_t size, TYPE defaultValue)
+{
+/* if (container.capacity() < size)
+ {
+ // We need to allocate memory. Reserve 20% more than we need to reduce the
+ // liklihood we will need to do this again next time.
+ container.reserve(size*1.2);
+ }
+*/
+ container.resize(size, defaultValue);
+}
+
+/// \brief Lane number type definition.
+typedef std::size_t LaneNumber;
+
+/// \brief Sample number type definition.
+typedef std::size_t SampleNumber;
+
+/// \brief Tile number type definition.
+typedef std::size_t TileNumber;
+
+/// \brief Read number type definition.
+typedef std::size_t ReadNumber;
+
+/// \brief Cycle number type definition.
+typedef std::size_t CycleNumber;
+
+/// \brief Cycle number range.definition.
+typedef std::pair<CycleNumber, CycleNumber> CycleRange;
+
+/// \brief Number of clusters.
+typedef uint32_t ClustersCount;
+
+/// \brief Number of bases; throughput.
+typedef uint64_t Yield;
+
+/// \brief Number of bases; throughput.
+typedef uint64_t QualityScore;
+
+/// \brief Control flag type definition.
+typedef uint16_t ControlFlag;
+
+/// \brief TileAggregationMode type definition.
+enum class TileAggregationMode { NON_AGGREGATED, AGGREGATED, CBCL };
+
+/// brief type definition for map from lane and tile number to file name
+typedef std::vector<std::map<common::TileNumber, boost::filesystem::path>> TileFileMap;
+
+inline std::ostream& operator<<(std::ostream& os, ReadType readType)
+{
+ switch (readType)
+ {
+ case DATA:
+ os << "data read";
+ break;
+ case INDEX:
+ os << "index read";
+ break;
+ case UMI:
+ os << "umi read";
+ break;
+ default:
+ BCL2FASTQ_ASSERT_MSG(false, "Unrecognized ReadType");
+ break;
+ }
+
+ return os;
+}
+
+} // namespace common
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_DATA_TYPES_HH
+
+
diff --git a/src/cxx/include/config/Bcl2FastqOptions.hh b/src/cxx/include/config/Bcl2FastqOptions.hh
new file mode 100644
index 0000000..579ea82
--- /dev/null
+++ b/src/cxx/include/config/Bcl2FastqOptions.hh
@@ -0,0 +1,330 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Bcl2FastqOptions.hh
+ *
+ * \brief Declaration of the Bcl2Fastq options processing.
+ *
+ * \author Mauricio Varea
+ * \author Marek Balint
+ */
+
+#ifndef BCL2FASTQ_CONFIG_BCL2FASTQOPTIONS_HH
+#define BCL2FASTQ_CONFIG_BCL2FASTQOPTIONS_HH
+
+
+#include <iosfwd>
+#include <string>
+
+#include <boost/filesystem/path.hpp>
+#include <boost/optional.hpp>
+
+#include "config/SampleSheetCsv.hh"
+
+#include "common/Options.hh"
+#include "common/Threads.hh"
+
+
+namespace bcl2fastq {
+namespace config {
+
+/// \brief Maximum number of allowed mismatches in a barcode component type definition.
+typedef std::size_t MismatchesCount;
+
+/// \brief Maximum allowed mismatches mismatches in a barcode component container type definition.
+typedef std::vector<MismatchesCount> BarcodeMismatchCountsContainer;
+
+/// \brief This class circumvents a bug in boost that produces an error when
+/// spaces occur in boost::filesystem::path used with boost::program_options::value
+class BoostPath
+{
+public:
+ BoostPath() : path_() { }
+ BoostPath(const boost::filesystem::path& path) : path_(path) { }
+
+ friend std::istream& operator>>(std::istream& is, BoostPath& boostPath);
+
+ boost::filesystem::path path_;
+};
+
+
+/// \brief Bcl2Fastq custom options.
+class Bcl2FastqOptions : public common::Options
+{
+public:
+
+ /// \brief UseBasesMasks container type definition.
+ typedef std::vector<std::string> UseBasesMaskContainer;
+
+public:
+
+ /// \brief Default constructor.
+ Bcl2FastqOptions();
+
+public:
+
+ /// \brief Get input directory.
+ /// \return Path to the directory containing input BCL files.
+ const boost::filesystem::path& getInputDir() const { return inputDir_.path_; }
+
+ /// \brief Get runfolder directory.
+ /// \return Path to the project's root directory.
+ const boost::filesystem::path& getRunfolderDir() const { return runfolderDir_.path_; }
+
+ /// \brief Get intensities directory.
+ /// \return Path to the intensities directory.
+ const boost::filesystem::path& getIntensitiesDir() const { return intensitiesDir_.path_; }
+
+ /// \brief Get output directory.
+ /// \return Path to demultiplexed output directory.
+ const boost::filesystem::path& getOutputDir() const { return outputDir_.path_; }
+
+ /// \brief Get interop directory.
+ /// \return Path to demultiplexing statistics directory.
+ const boost::filesystem::path& getInteropDir() const { return interopDir_.path_; }
+
+ /// \brief Get stats directory.
+ /// \return Path to human-readable demultiplexing statistics directory.
+ const boost::filesystem::path& getStatsDir() const { return statsDir_.path_; }
+
+ /// \brief Get reports directory.
+ /// \return Path to reporting directory.
+ const boost::filesystem::path& getReportsDir() const { return reportsDir_.path_; }
+
+ /// \brief Get number of BCL loading threads.
+ /// \return Number of threds to be used for loading BCL data.
+ common::ThreadVector::size_type getBclLoaderThreadsCount() const;
+
+ /// \brief Get number of FASTQ creating threads.
+ /// \return Number of threads to be used for creation of FASTQ data.
+ common::ThreadVector::size_type getFastqCreatorThreadsCount() const;
+
+ /// \brief Get number of FASTQ writing threads.
+ /// \return Number of threads to be used for writing FASTQ data.
+ common::ThreadVector::size_type getFastqWriterThreadsCount() const;
+
+ /// \brief Get list of tiles to be processed.
+ /// \return List of tiles to be processed.
+ std::vector<std::string> getTilesFilterList() const;
+
+ /// \brief Get minimum read length after adapter trimming.
+ /// \return Minimum read length after adapter trimming.
+ std::size_t getMinimumTrimmedReadLength() const;
+
+ /// \brief Get the masks used to select which cycled are used.
+ /// \return The masks
+ const UseBasesMaskContainer& getUseBasesMasks() const;
+
+ /// \brief Get maximum number of useful bases in read after adapter trimming for which whole read is masked.
+ /// \return Maximum number of useful bases in read after adapter trimming for which whole read is masked.
+ std::size_t getMaskShortAdapterReads() const;
+
+ /// \brief Get adapter stringency.
+ /// \return Adapter stringency.
+ float getAdapterStringency() const;
+
+ /// \brief Get flag for ignore missing BCLs.
+ /// \retval true Ignore missing BCLs.
+ /// \retval false Fail on missing BCLs.
+ bool getIgnoreMissingBcls() const;
+
+ /// \brief Get flag for ignore missing filters.
+ /// \retval true Ignore missing filters.
+ /// \retval false Fail on missing filters.
+ bool getIgnoreMissingFilters() const;
+
+ /// \brief Get flag for ignore missing positions.
+ /// \retval true Ignore missing positions.
+ /// \retval false Fail on missing positions.
+ bool getIgnoreMissingPositions() const;
+
+ /// \brief Get flag for ignore missing controls.
+ /// \retval true Ignore missing controls.
+ /// \retval false Fail on missing controls.
+ bool getIgnoreMissingControls() const;
+
+ /// \brief Get flag for generate reverse complement FASTQs.
+ /// \retval true Generate reverse complement FASTQs.
+ /// \retval false Do not generate reverse complement FASTQs.
+ bool getGenerateReverseComplementFastqs() const;
+
+ /// \brief Get flag for include non-PF clusters.
+ /// \retval true Include non-PF clusters.
+ /// \retval false Do not include non-PF clusters.
+ bool getIncludeNonPfClusters() const;
+
+ /// \brief Get flag for create FASTQ files also for index reads.
+ /// \retval true Create FASTQ files also for index reads.
+ /// \retval false Do not create FASTQ files for index reads.
+ bool getCreateFastqsForIndexReads() const;
+
+ /// \brief Find adapters with indels.
+ /// \retval true if should find adapters with sliding window.
+ bool findAdaptersWithSlidingWindow() const { return findAdaptersWithSlidingWindow_; }
+
+ /// \brief Get flag for bgzf compression use.
+ /// retval True if bgzf compression should be used for fastq files
+ bool useBgzfCompression() const;
+
+ /// \brief Get compression level used by zlib for fastq files
+ /// retval compression level to be used by zlib
+ int getFastqCompressionLevel() const;
+
+ /// \brief Get flag for determining if it should automatically set allowed barcode mismatches to 0 on collision.
+ /// retval True if it should automatically set allowed barcode mismatches to 0 on collision.
+ bool getAutoSetToZeroBarcodeMismatches() const { return autoSetToZeroBarcodeMismatches_; }
+
+ /// \brief Get allowed barcode mismatch counts.
+ /// \return Barcode mismatch counts.
+ BarcodeMismatchCountsContainer& getBarcodeMismatches() { return barcodeMismatchCounts_; }
+
+ /// \brief Get allowed barcode mismatch counts.
+ /// \return Barcode mismatch counts.
+ const BarcodeMismatchCountsContainer& getBarcodeMismatches() const { return barcodeMismatchCounts_; }
+
+ /// \brief Return true if fastq files should not be split by lane
+ /// \return True if fastq files should not be split by lane.
+ bool noLaneSplitting() const { return noLaneSplitting_; }
+
+ const config::SampleSheetCsv& getSampleSheet() const { return *sampleSheet_; }
+
+public:
+
+ common::ThreadVector::size_type getAppliedFastqWriterThreadsCount() const;
+
+private:
+
+ virtual void initNamedOptions(boost::program_options::options_description &options);
+
+ virtual void initUnnamedOptions(boost::program_options::options_description &options);
+
+ virtual void initPositionalOptions(boost::program_options::positional_options_description &options);
+
+private:
+
+ virtual std::string usagePrefix() const;
+
+ virtual std::string usageSuffix() const;
+
+private:
+
+ /// \brief Store the options from the sample sheet.
+ /// \param vm Options map where the settings are stored.
+ virtual void storeSampleSheetSettings(boost::program_options::variables_map &vm);
+
+ /// \brief Post-process parsed command line arguments.
+ /// \param vm Command line arguments to be post-process.
+ virtual void postProcess(boost::program_options::variables_map &vm);
+
+private:
+
+ /// \brief Path to BaseCalls directory.
+ BoostPath inputDir_;
+
+ /// \brief Path to BaseCalls directory.
+ BoostPath runfolderDir_;
+
+ /// \brief Path to intensities directory.
+ BoostPath intensitiesDir_;
+
+ /// \brief Path to demultiplexed output.
+ BoostPath outputDir_;
+
+ /// \brief Path to demultiplexing stats directory.
+ BoostPath interopDir_;
+
+ /// \brief Path to human-readable demultiplexing stats directory.
+ BoostPath statsDir_;
+
+ /// \brief Path to reporting directory.
+ BoostPath reportsDir_;
+
+ /// \brief Path to sample sheet.
+ BoostPath sampleSheetPath_;
+
+ /// \brief Number of threads to be used for loading BCL data.
+ common::ThreadVector::size_type bclLoaderThreadsCount_;
+
+ /// \brief Number of threads to be used for demultiplexing.
+ common::ThreadVector::size_type demultiplexerThreadsCount_;
+
+ /// \brief Number of threads to be used for creation of FASTQ data.
+ common::ThreadVector::size_type fastqCreatorThreadsCount_;
+
+ /// \brief Number of threads to be used for writing FASTQ data.
+ common::ThreadVector::size_type fastqWriterThreadsCount_;
+
+ /// \brief List of tiles to be processed.
+ std::string tilesFilterString_;
+
+ /// \brief Minimum read length after adapter trimming.
+ std::size_t minimumTrimmedReadLength_;
+
+ /// \brief Identifies which cycles should be used
+ UseBasesMaskContainer useBasesMasks_;
+
+ /// \brief Maximum number of useful bases in read after adapter trimming for which whole read is masked.
+ std::size_t maskShortAdapterReads_;
+
+ /// \brief Adapter stringency.
+ float adapterStringency_;
+
+ /// \brief Generate reverse complement FASTQs flag.
+ bool generateReverseComplementFastqs_;
+
+ /// \brief Include non-PF clusters in created FASTQ files.
+ bool includeNonPfClusters_;
+
+ /// \brief Ignore missing BCLs (assume 'N'/'#' for missing calls).
+ bool ignoreMissingBcls_;
+
+ /// \brief Ignore missing filters (assume 'true' for missing filters).
+ bool ignoreMissingFilters_;
+
+ /// \brief Ignore missing positions (assume [0,0] for missing positions).
+ bool ignoreMissingPositions_;
+
+ /// \brief Ignore missing controls (assume 0 for missing controls).
+ bool ignoreMissingControls_;
+
+ /// \brief Create FASTQ files also for index reads.
+ bool createFastqsForIndexReads_;
+
+ /// \brief Find adapters with sliding window.
+ bool findAdaptersWithSlidingWindow_;
+
+ /// \brief Turn off bgzf compression.
+ bool noBgzfCompression_;
+
+ /// \brief Compression level used by zlib for fastq files
+ int fastqCompressionLevel_;
+
+ /// \brief Barcode mismatch counts delimited string.
+ std::string barcodeMismatchCountsDelimitedString_;
+
+ /// \brief Barcode mismatch counts.
+ BarcodeMismatchCountsContainer barcodeMismatchCounts_;
+
+ /// \brief If true, automatically set allowed barcode mismatches to 0 on collision
+ bool autoSetToZeroBarcodeMismatches_;
+
+ /// \brief If true, don't split fastq files by lane
+ bool noLaneSplitting_;
+
+ /// \brief Sample sheet.
+ config::SampleSheetCsvPtr sampleSheet_;
+};
+
+
+} // namespace config
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_CONFIG_BCL2FASTQOPTIONS_HH
+
+
diff --git a/src/cxx/include/config/SampleSheetCsv.hh b/src/cxx/include/config/SampleSheetCsv.hh
new file mode 100644
index 0000000..2d9b7f9
--- /dev/null
+++ b/src/cxx/include/config/SampleSheetCsv.hh
@@ -0,0 +1,227 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file SampleSheetCsv.hh
+ *
+ * \brief Declaration of sample sheet helper.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_CONFIG_SAMPLESHEETCSV_HH
+#define BCL2FASTQ_CONFIG_SAMPLESHEETCSV_HH
+
+
+#include <vector>
+#include <string>
+
+#include <boost/filesystem/path.hpp>
+#include <boost/noncopyable.hpp>
+#include "common/CsvGrammar.hh"
+#include "common/Types.hh"
+#include "common/SampleMetadata.hh"
+
+#include <boost/shared_ptr.hpp>
+
+namespace bcl2fastq {
+
+
+namespace config {
+
+/// \brief Sample sheet helper.
+class SampleSheetCsv : private boost::noncopyable
+{
+public:
+
+ class ExcludedTileRange
+ {
+ public:
+ ExcludedTileRange(const std::string& flowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber firstTile,
+ common::TileNumber lastTile);
+
+ bool isExcluded(const std::string& flowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber) const;
+
+ private:
+ std::string flowcell_;
+ common::LaneNumber laneNumber_;
+ common::TileNumber firstTile_;
+ common::TileNumber lastTile_;
+ };
+
+ /// \brief Adapters container type definition.
+ typedef std::vector<std::pair<common::ReadNumber,std::string> > AdaptersContainer;
+
+ /// \brief Excluded tiles container type definition
+ typedef std::vector<ExcludedTileRange> ExcludedTilesContainer;
+
+public:
+
+ /// \brief Constructor.
+ /// \param sampleSheetData Sample sheet CSV data.
+ /// \param directoryValidator Use to validate that there are no clashes
+ SampleSheetCsv(const common::CsvGrammarAttribute &sampleSheetData,
+ const boost::filesystem::path& outputDir);
+
+public:
+
+ // I would like to use boost::logic::tribool, but this expression bizarrely returns false:
+ // boost::logic::indeterminate(boost::tribool::indeterminate_value)
+ enum TriState { TRUE=0, FALSE, INDETERMINATE };
+
+ /// \brief Get beginning of adapters to be masked.
+ /// \return Iterator to beginning of adapters to be masked.
+ SampleSheetCsv::AdaptersContainer::const_iterator maskAdaptersBegin() const;
+
+ /// \brief Get end of adapters to be masked.
+ /// \return Iterator to end of adapters to be trimmed.
+ SampleSheetCsv::AdaptersContainer::const_iterator maskAdaptersEnd() const;
+
+ /// \brief Get beginning of adapters to be trimmed.
+ /// \return Iterator to beginning of adapters to be trimmed.
+ SampleSheetCsv::AdaptersContainer::const_iterator trimAdaptersBegin() const;
+
+ /// \brief Get end of adapters to be trimmed.
+ /// \return Iterator to end of adapters to be trimmed.
+ SampleSheetCsv::AdaptersContainer::const_iterator trimAdaptersEnd() const;
+
+ /// \brief Get sample metadata container.
+ /// \return Sample metadata container.
+ const common::SampleMetadataContainer& getSampleMetadata() const { return samples_; }
+
+ /// \brief Get the setting for createFastqForIndexReads
+ /// \return INDETERMINATE if the setting is not in the sample sheet.
+ TriState createFastqForIndexReads() const { return createFastqForIndexReads_; }
+
+ /// \brief Get the setting for findAdaptersWithIndels
+ /// \return INDETERMINATE if the setting is not in the sample sheet.
+ TriState findAdaptersWithIndels() const { return findAdaptersWithIndels_; }
+
+ /// \brief Generate reverse complement fastqs
+ /// \return INDTERMINATE if the setting is not in the sample sheet.
+ TriState generateReverseComplementFastqs() const { return generateReverseComplementFastqs_; }
+
+ /// \brief Get the read 1 end cycle
+ /// \return Read 1 end cycle number.
+ common::CycleNumber getRead1EndCycle() const { return read1EndWithCycle_; }
+
+ /// \brief Get the read 2 end cycle
+ /// \return Read 2 end cycle number.
+ common::CycleNumber getRead2EndCycle() const { return read2EndWithCycle_; }
+
+ /// \brief Get the read 1 start cycle
+ /// \return Read 1 start cycle number.
+ common::CycleNumber getRead1StartCycle() const { return read1StartFromCycle_; }
+
+ /// \brief Get the read 1 UMI length
+ /// \return Read 1 UMI length.
+ common::CycleNumber getRead1UmiLength() const { return read1UmiLength_; }
+
+ /// \brief Get the read 2 UMI length
+ /// \return Read 2 UMI length.
+ common::CycleNumber getRead2UmiLength() const { return read2UmiLength_; }
+
+ /// \brief Get the read 1 UMI start cycle
+ /// \return Read 1 UMI start cycle
+ common::CycleNumber getRead1UmiStartFromCycle() const { return read1UmiStartFromCycle_; }
+
+ /// \brief Get the read 2 UMI start cycle
+ /// \return Read 2 UMI start cycle
+ common::CycleNumber getRead2UmiStartFromCycle() const { return read2UmiStartFromCycle_; }
+
+ /// \brief Trim UMI if true
+ /// \return INDTERMINATE if the setting is not in the sample sheet.
+ TriState trimUmi() const { return trimUmi_; }
+
+ /// \brief Get the read 2 start cycle
+ /// \return Read 2 start cycle number.
+ common::CycleNumber getRead2StartCycle() const { return read2StartFromCycle_; }
+
+ /// \brief Return true if the tile number is excluded.
+ /// \param flowcell Flowcell id.
+ /// \param laneNumber Lane number.
+ /// \param tileNumber Tile number.
+ /// \return True if tile is excluded.
+ bool isTileExcluded(const std::string& flowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber) const;
+
+private:
+
+ /// \brief Adapters to be masked.
+ AdaptersContainer maskAdapters_;
+
+ /// \brief Adapters to be trimmed.
+ AdaptersContainer trimAdapters_;
+
+ /// \brief Sample metadata.
+ common::SampleMetadataContainer samples_;
+
+ /// \brief Create fastq for index reads.
+ TriState createFastqForIndexReads_;
+
+ /// \brief Find adapters with indels.
+ TriState findAdaptersWithIndels_;
+
+ /// \brief Generate reverse complement fastqs
+ TriState generateReverseComplementFastqs_;
+
+ /// \brief End cycle number for read 1
+ common::CycleNumber read1EndWithCycle_;
+
+ /// \brief End cycle number for read 2
+ common::CycleNumber read2EndWithCycle_;
+
+ /// \brief Start cycle number for read 1
+ common::CycleNumber read1StartFromCycle_;
+
+ /// \brief Start cycle number for read 2
+ common::CycleNumber read2StartFromCycle_;
+
+ /// \brief UMI length for read 1
+ common::CycleNumber read1UmiLength_;
+
+ /// \brief UMI length for read 2
+ common::CycleNumber read2UmiLength_;
+
+ /// \brief Read 1 UMI start cycle
+ common::CycleNumber read1UmiStartFromCycle_;
+
+ /// \brief Read 2 UMI start cycle
+ common::CycleNumber read2UmiStartFromCycle_;
+
+ /// \brief Trim UMI if true
+ TriState trimUmi_;
+
+ /// \brief Excluded tiles
+ ExcludedTilesContainer excludedTiles_;
+};
+
+typedef boost::shared_ptr<SampleSheetCsv> SampleSheetCsvPtr;
+
+/// \brief Create sample sheet object.
+/// \param runfolderDir Path to runfolder.
+/// \param directoryValidator Used to validate there are no clashes with existing directories
+/// \return Sample sheet object constructed with data from given runfolder.
+SampleSheetCsvPtr createSampleSheetCsv(const boost::filesystem::path& sampleSheetPath,
+ const boost::filesystem::path& outputDir);
+
+
+} // namespace config
+
+
+} // namespace bcl2fastq
+
+
+#endif //BCL2FASTQ_CONFIG_SAMPLESHEETCSV_HH
+
+
diff --git a/src/cxx/include/conversion/AdapterLocator.hh b/src/cxx/include/conversion/AdapterLocator.hh
new file mode 100644
index 0000000..a34ae40
--- /dev/null
+++ b/src/cxx/include/conversion/AdapterLocator.hh
@@ -0,0 +1,121 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file AdapterLocator.hh
+ *
+ * \brief Declaration of Adapter Locator.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_ADAPTER_LOCATOR_HH
+#define BCL2FASTQ_CONVERSION_ADAPTER_LOCATOR_HH
+
+#include "conversion/FastqIterator.hh"
+#include "common/Types.hh"
+
+#include <boost/noncopyable.hpp>
+
+namespace bcl2fastq
+{
+namespace conversion
+{
+
+template<common::NumBasesPerByte numBasesPerByte>
+class AdapterLocator : private boost::noncopyable
+{
+public:
+ AdapterLocator(const std::string& adapter,
+ float adapterStringency);
+
+ virtual ~AdapterLocator() { }
+
+ virtual FastqConstIterator<numBasesPerByte> identifyAdapter(const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd) const = 0;
+
+protected:
+ const std::string adapter_;
+ float adapterStringency_;
+};
+
+template<common::NumBasesPerByte numBasesPerByte>
+class AdapterLocatorSlidingWindow : public AdapterLocator<numBasesPerByte>
+{
+public:
+ AdapterLocatorSlidingWindow(const std::string& adapter,
+ float adapterStringency);
+
+ virtual ~AdapterLocatorSlidingWindow() { }
+
+ virtual FastqConstIterator<numBasesPerByte> identifyAdapter(const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd) const;
+};
+
+template<common::NumBasesPerByte numBasesPerByte>
+class AdapterLocatorWithIndels : public AdapterLocator<numBasesPerByte>
+{
+public:
+ AdapterLocatorWithIndels(const std::string& adapter,
+ float adapterStringency,
+ uint32_t minimumTrimmedReadLength);
+
+ virtual ~AdapterLocatorWithIndels() { }
+
+ virtual FastqConstIterator<numBasesPerByte> identifyAdapter(const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd) const;
+
+private:
+ struct MyersValues : private boost::noncopyable
+ {
+ MyersValues(uint32_t length,
+ uint64_t pv);
+
+ void updateValues(unsigned char b, const std::vector<uint64_t>& peq);
+
+ uint64_t mv_;
+ uint64_t count_;
+ uint64_t score_;
+ uint64_t compareLength_;
+ uint64_t pv_;
+ };
+
+ FastqConstIterator<numBasesPerByte> findPartialMatch(const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd,
+ uint32_t minScorePartialMatch) const;
+
+ FastqConstIterator<numBasesPerByte> findFullMatch(const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd,
+ uint32_t& minScorePartialMatch) const;
+
+ std::vector<uint64_t> getPeq(const std::vector<unsigned char>& adapterCalls) const;
+
+ FastqConstIterator<numBasesPerByte> findLongestExactSuffixMatch(const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd,
+ const std::vector<unsigned char>& adapterSubstring,
+ int tagLength) const;
+
+ void binarizeAdapterCalls(const std::string& adapter,
+ std::vector<unsigned char>& adapterCalls) const;
+
+ uint32_t minimumTrimmedReadLength_;
+ uint32_t minLengthMatch_;
+ uint32_t mismatchesToAccountFor_;
+ std::vector<uint32_t> maxAllowedMismatches_;
+ std::vector< std::vector<unsigned char> > adapterCallsSubsets_;
+
+ std::vector< std::vector<uint64_t> > peqs_;
+ std::vector<uint64_t> pv0s_;
+};
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+#include "conversion/AdapterLocator.hpp"
+
+#endif // BCL2FASTQ_CONVERSION_ADAPTER_LOCATOR_HH
diff --git a/src/cxx/include/conversion/AdapterLocator.hpp b/src/cxx/include/conversion/AdapterLocator.hpp
new file mode 100644
index 0000000..c567fed
--- /dev/null
+++ b/src/cxx/include/conversion/AdapterLocator.hpp
@@ -0,0 +1,394 @@
+/**
+* BCL to FASTQ file converter
+* Copyright (c) 2007-2017 Illumina, Inc.
+*
+* This software is covered by the accompanying EULA
+* and certain third party copyright/licenses, and any user of this
+* source file is bound by the terms therein.
+*
+* \file AdapterLocator.hpp
+*
+* \brief Implementation of adapter locator.
+*
+* \author Aaron Day
+*/
+
+#ifndef BCL2FASTQ_CONVERSION_ADAPTER_LOCATOR_HPP
+#define BCL2FASTQ_CONVERSION_ADAPTER_LOCATOR_HPP
+
+#include "conversion/AdapterLocator.hh"
+
+#include "conversion/BclBaseConversion.hh"
+#include "common/Debug.hh"
+
+namespace bcl2fastq
+{
+namespace conversion
+{
+
+template<common::NumBasesPerByte numBasesPerByte>
+AdapterLocator<numBasesPerByte>::AdapterLocator(const std::string& adapter,
+ float adapterStringency)
+: adapter_(adapter)
+, adapterStringency_(adapterStringency)
+{
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+AdapterLocatorSlidingWindow<numBasesPerByte>::AdapterLocatorSlidingWindow(const std::string& adapter,
+ float adapterStringency)
+: AdapterLocator<numBasesPerByte>(adapter, adapterStringency)
+{
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+FastqConstIterator<numBasesPerByte> AdapterLocatorSlidingWindow<numBasesPerByte>::identifyAdapter(const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd) const
+{
+ const std::string::size_type adapterLength = this->adapter_.size();
+
+ for (FastqConstIterator<numBasesPerByte> basesIter = basesBegin; basesIter != basesEnd; ++basesIter)
+ {
+ std::size_t mismatchCount = 0;
+ std::size_t matchCount = 0;
+ const size_t checkCycles = std::min(static_cast<typename FastqConstIterator<numBasesPerByte>::difference_type>(adapterLength), basesEnd - basesIter);
+ for (std::size_t adapterOffset = 0; adapterOffset < checkCycles; ++adapterOffset)
+ {
+ const char base = convertBcl2FastqBase(basesIter[adapterOffset]);
+ if (base == 'N')
+ {
+ // Ns are neither matches nor mismatches
+ continue;
+ }
+ if (base == this->adapter_.at(adapterOffset))
+ {
+ ++matchCount;
+ }
+ else
+ {
+ ++mismatchCount;
+ }
+
+ if ((mismatchCount > 1) && (mismatchCount > matchCount)) // Shortcut
+ {
+ // If at any point there are 2 or more mismatches and the number of mismatches is
+ // greater than the number of matches, stop scanning and start a new scan at
+ // the next initial position in the read sequence
+ break;
+ }
+ }
+
+ float sequenceIdentity = matchCount / (float)(matchCount + mismatchCount);
+ if (matchCount > 0 && sequenceIdentity >= this->adapterStringency_)
+ {
+ return basesIter;
+ }
+ }
+
+ return basesEnd;
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+AdapterLocatorWithIndels<numBasesPerByte>::AdapterLocatorWithIndels(const std::string& adapter,
+ float adapterStringency,
+ uint32_t minimumTrimmedReadLength)
+: AdapterLocator<numBasesPerByte>(adapter, adapterStringency)
+, minimumTrimmedReadLength_(minimumTrimmedReadLength)
+, minLengthMatch_(0)
+, mismatchesToAccountFor_(0)
+, maxAllowedMismatches_(adapter.size())
+, adapterCallsSubsets_(adapter.size())
+, peqs_(adapter.size())
+, pv0s_(adapter.size())
+{
+ BCL2FASTQ_ASSERT_MSG(adapter.size() <= 64,
+ "Adapter size must be 64 or less. Adapter: " << adapter);
+
+ std::vector<unsigned char> adapterCalls;
+ binarizeAdapterCalls(adapter,
+ adapterCalls);
+
+ size_t adapterLength = adapter.size();
+
+ minLengthMatch_ = std::min(adapterLength, static_cast<size_t>(std::ceil(adapterStringency * 10)));
+ mismatchesToAccountFor_ = std::ceil(adapterLength * (1.0 - adapterStringency));
+
+ // calculate all values between 1 and adapterLength (need shorter than
+ // minLengthMatch if mismatches found that reduce the search range to less then minLengthMatch)
+ for (size_t i = 0; i < adapterLength; ++i)
+ {
+ //maximum number of allowed mismatches for a full length match
+ maxAllowedMismatches_[i] = std::floor((i+1) * (1.0 - this->adapterStringency_)); //rounding down
+
+ adapterCallsSubsets_[i] = std::vector<unsigned char>(adapterCalls.begin(), adapterCalls.begin()+(i+1));
+
+ //calculate Peqs and Pv0s
+ peqs_[i] = getPeq(adapterCallsSubsets_[i]);
+
+ pv0s_[i] = (2L << i) - 1;
+ }
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+FastqConstIterator<numBasesPerByte> AdapterLocatorWithIndels<numBasesPerByte>::identifyAdapter(const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd) const
+{
+ if (basesBegin == basesEnd)
+ {
+ return basesEnd;
+ }
+
+ uint32_t minScorePartialMatch = this->adapter_.size();
+
+ FastqConstIterator<numBasesPerByte> pos(findFullMatch(basesBegin,
+ basesEnd,
+ minScorePartialMatch));
+
+ if (pos == basesEnd)
+ {
+ // try to find partial match at end since we did not find full match
+ return findPartialMatch(basesBegin,
+ basesEnd,
+ minScorePartialMatch);
+ }
+
+ return pos;
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+FastqConstIterator<numBasesPerByte> AdapterLocatorWithIndels<numBasesPerByte>::findPartialMatch(const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd,
+ uint32_t minScorePartialMatch) const
+{
+ FastqConstIterator<numBasesPerByte> trimPosition = basesEnd;
+
+ size_t adapterLength = this->adapter_.size();
+ uint64_t minScore = adapterLength;
+
+ // number of bases to consider for partial match
+ uint32_t tmpLength = (minScorePartialMatch > mismatchesToAccountFor_ ?
+ adapterLength - minScorePartialMatch + mismatchesToAccountFor_ - 1 :
+ adapterLength - 1);
+
+ if (tmpLength > std::distance(basesBegin, basesEnd))
+ {
+ return basesEnd;
+ }
+
+ while (trimPosition == basesEnd)
+ {
+ // only exact matches possible
+ if (minLengthMatch_ >= tmpLength)
+ {
+ return findLongestExactSuffixMatch(basesBegin,
+ basesEnd,
+ adapterCallsSubsets_[tmpLength - 1],
+ tmpLength);
+ }
+
+ // mismatches still allowed
+ MyersValues myersValues(tmpLength, pv0s_[tmpLength - 1]);
+ uint32_t maxAllowedMismatch = maxAllowedMismatches_[tmpLength - 1];
+
+ BOOST_REVERSE_FOREACH(char base, std::make_pair(basesBegin, basesEnd))
+ {
+ myersValues.updateValues(base, peqs_[tmpLength - 1]);
+
+ if (myersValues.score_ <= minScore)
+ {
+ minScore = myersValues.score_;
+
+ // partial adapter match
+ if (maxAllowedMismatch >= myersValues.score_)
+ {
+ trimPosition = basesEnd - myersValues.count_;
+ }
+ }
+
+ if (myersValues.count_ > myersValues.compareLength_)
+ {
+ break; // end foreach
+ }
+ }
+
+ if (trimPosition != basesEnd || minScore == tmpLength) // match found or all bases are mismatching
+ {
+ break; //end while loop
+ }
+
+ tmpLength -= (minScore > maxAllowedMismatch ?
+ (minScore - maxAllowedMismatch) :
+ 1);
+ }
+
+ return trimPosition;
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+FastqConstIterator<numBasesPerByte> AdapterLocatorWithIndels<numBasesPerByte>::findFullMatch(const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd,
+ uint32_t& minScorePartialMatch) const
+{
+ size_t adapterLength = this->adapter_.size();
+ MyersValues myersValues(adapterLength, pv0s_[adapterLength - 1]);
+
+ uint32_t minScore = adapterLength;
+ uint32_t cycleCount = std::distance(basesBegin, basesEnd);
+ int trimPosition = -1;
+ uint32_t lengthWithMismatch = cycleCount + maxAllowedMismatches_[adapterLength - 1];
+
+ //search for full matching adapters
+ for (FastqConstIterator<numBasesPerByte> pos = basesEnd - 1; pos >= basesBegin; --pos)
+ {
+ myersValues.updateValues(*pos, peqs_[adapterLength - 1]);
+
+ //full adapter match
+ if (myersValues.score_ <= maxAllowedMismatches_[adapterLength - 1] &&
+ (myersValues.score_ <= minScore || myersValues.count_ > (lengthWithMismatch - trimPosition)))
+ {
+ trimPosition = cycleCount - myersValues.count_;
+
+ // NOTE: There used to be code here to break out of this loop
+ // early, assuming that it could stop if there was a good enough
+ // match that will trim the read shorter than the required minimum
+ // read length. This was flawed, because indels and N's mean there
+ // could be more matches that would mask more.
+ }
+
+ //match with less mismatches than previous best match
+ if (myersValues.score_ <= minScore)
+ {
+ minScore = myersValues.score_;
+ //maximum number of matches within adapter length from end (used to find partial match)
+ if (myersValues.count_ < adapterLength)
+ {
+ minScorePartialMatch = minScore;
+ }
+
+ }
+ }
+
+ return (trimPosition == -1 ? basesEnd : basesBegin + trimPosition);
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+AdapterLocatorWithIndels<numBasesPerByte>::MyersValues::MyersValues(uint32_t length,
+ uint64_t pv)
+: mv_(0)
+, count_(0)
+, score_(length)
+, compareLength_(length-1)
+, pv_(pv)
+{
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+void AdapterLocatorWithIndels<numBasesPerByte>::MyersValues::updateValues(unsigned char b, const std::vector<uint64_t>& peq)
+{
+ // all zero for N and otherwise take the two right-most bits for basecalls (others are for quality calls)
+ uint64_t eq_ = peq[b];
+ uint64_t d0 = (((eq_ & pv_) + pv_) ^ pv_) | eq_ | mv_;
+
+ uint64_t ph = mv_ | ~(d0 | pv_);
+ uint64_t mh = pv_ & d0;
+
+ score_ += (ph >> compareLength_ & 1) - (mh >> compareLength_ & 1);
+
+ ph <<= 1;
+ pv_ = (mh << 1) | ~(d0 | ph);
+ mv_ = ph & d0;
+
+ //zero-based position from 3'-end or step count
+ ++count_;
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+std::vector<uint64_t> AdapterLocatorWithIndels<numBasesPerByte>::getPeq(const std::vector<unsigned char>& adapterCalls) const
+{
+ std::vector<uint64_t> peq(256, 0);
+ for (int peqBase = 1; peqBase <= 255; ++peqBase)
+ {
+ uint64_t& value = peq[peqBase];
+ BOOST_FOREACH(unsigned char base, std::make_pair(adapterCalls.begin(), adapterCalls.end()))
+ {
+ value <<= 1;
+
+ if ((peqBase & 3) == base)
+ {
+ value |= 1L;
+ }
+ }
+ }
+
+ return peq;
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+FastqConstIterator<numBasesPerByte> AdapterLocatorWithIndels<numBasesPerByte>::findLongestExactSuffixMatch(const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd,
+ const std::vector<unsigned char>& adapterSubstring,
+ int tagLength) const
+{
+ int cycleCount = std::distance(basesBegin, basesEnd);
+
+ for (int i = tagLength - 1; i >= 0; --i)
+ {
+ int match = 0;
+ int tmp = cycleCount - 1 - i;
+ for (int j = 0; j <= i; ++j)
+ {
+ if (( *(basesBegin + tmp + j) & 3) == adapterSubstring[j])
+ {
+ ++match;
+ }
+ else
+ {
+ break;
+ }
+ }
+
+ if (match > i)
+ {
+ return basesEnd - match;
+ }
+ }
+
+ return basesEnd;
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+void AdapterLocatorWithIndels<numBasesPerByte>::binarizeAdapterCalls(const std::string& adapter,
+ std::vector<unsigned char>& adapterCalls) const
+{
+ adapterCalls.resize(adapter.size());
+ for (size_t i = 0; i < adapter.size(); ++i)
+ {
+ switch (adapter[i])
+ {
+ case 'A':
+ case 'a':
+ adapterCalls[i] = 0;
+ break;
+ case 'C':
+ case 'c':
+ adapterCalls[i] = 1;
+ break;
+ case 'G':
+ case 'g':
+ adapterCalls[i] = 2;
+ break;
+ case 'T':
+ case 't':
+ adapterCalls[i] = 3;
+ break;
+ default:
+ break;
+ }
+ }
+}
+
+} // namepsace conversion
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_CONVERSION_ADAPTER_LOCATOR_HPP
+
diff --git a/src/cxx/include/conversion/BclBaseConversion.hh b/src/cxx/include/conversion/BclBaseConversion.hh
new file mode 100644
index 0000000..b7d457b
--- /dev/null
+++ b/src/cxx/include/conversion/BclBaseConversion.hh
@@ -0,0 +1,168 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BclBaseConversion.hh
+ *
+ * \brief Declaration of Bcl base conversion functions.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_CONVERSION_BCLBASECONVERSION_HH
+#define BCL2FASTQ_CONVERSION_BCLBASECONVERSION_HH
+
+namespace bcl2fastq
+{
+namespace conversion
+{
+
+static const char bases[256] =
+ {'N', 'N', 'N', 'N', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T',
+ 'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T'};
+
+static const char noQscoreBases[4] = { 'A', 'C', 'G', 'T' };
+
+static const char reverseComplementBases[256] =
+ {'N', 'N', 'N', 'N', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A',
+ 'T', 'G', 'C', 'A', 'T', 'G', 'C', 'A'};
+
+static const char noQscoreReverseComplementBases[4] = { 'T', 'G', 'C', 'A' };
+
+/// \brief Convert BCL byte to FASTQ base byte.
+/// \param bcl BCL byte to be converted.
+/// \return FASTQ base byte.
+inline char convertBcl2FastqBase(unsigned char bcl)
+{
+ return bases[bcl];
+}
+
+/// \brief Convert BCL byte to FASTQ base byte.
+/// \param bcl BCL byte to be converted.
+/// \return FASTQ base byte.
+inline char convertBcl2FastqBaseNoQscore(unsigned char bcl)
+{
+ return noQscoreBases[bcl];
+}
+
+/// \brief Convert BCL byte to FASTQ base complement byte.
+/// \param bcl BCL byte to be converted.
+/// \return FASTQ base complement byte.
+inline char convertBcl2FastqBaseComplement(unsigned char bcl)
+{
+ return reverseComplementBases[bcl];
+}
+
+/// \brief Convert BCL byte to FASTQ base complement byte.
+/// \param bcl BCL byte to be converted.
+/// \return FASTQ base complement byte.
+inline char convertBcl2FastqBaseComplementNoQscore(unsigned char bcl)
+{
+ return noQscoreReverseComplementBases[bcl];
+}
+
+/// \brief Convert BCL byte to FASTQ quality byte.
+/// \param bcl BCL byte to be converted.
+/// \return FASTQ quality byte.
+inline char convertBcl2FastqQuality(unsigned char bcl)
+{
+ const unsigned char quality = bcl >> 2;
+ return quality + 33 + (quality==0)*2;
+}
+
+/// \brief Convert BCL byte to FASTQ quality byte.
+/// \param bcl BCL byte to be converted.
+/// \param remappedQscores Map to actual qscore numbers for given qscore bin
+/// \return FASTQ quality byte.
+inline char convertBcl2FastqQuality2(unsigned char bcl, const std::vector<uint32_t>& remappedQscores)
+{
+ const unsigned char quality = remappedQscores[bcl >> 2];
+ return quality + 33 + (quality==0)*2;
+}
+
+/// \brief Convert BCL byte to FASTQ quality byte.
+/// \param bcl BCL byte to be converted.
+/// \return FASTQ quality byte.
+inline char convertBcl2FastqRawQuality(unsigned char bcl)
+{
+ return bcl >> 2;
+}
+
+/// \brief Convert BCL byte to FASTQ quality byte.
+/// \param bcl BCL byte to be converted.
+/// \param remappedQscores Map to actual qscore numbers for given qscore bin
+/// \return FASTQ quality byte.
+inline char convertBcl2FastqRawQuality2(unsigned char bcl, const std::vector<uint32_t>& remappedQscores)
+{
+ return remappedQscores[bcl >> 2];
+}
+
+} // namespace bcl2fastq
+} // namespace conversion
+
+#endif // BCL2FASTQ_CONVERSION_BCLBASECONVERSION_HH
+
diff --git a/src/cxx/include/conversion/BclLoader.hh b/src/cxx/include/conversion/BclLoader.hh
new file mode 100644
index 0000000..f7966b8
--- /dev/null
+++ b/src/cxx/include/conversion/BclLoader.hh
@@ -0,0 +1,464 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BclLoader.hh
+ *
+ * \brief Declaration of BCL loader.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_BCLLOADER_HH
+#define BCL2FASTQ_CONVERSION_BCLLOADER_HH
+
+
+#include "conversion/BclReader.hh"
+
+#include "layout/Layout.hh"
+#include "data/BclFile.hh"
+#include "data/CycleBCIFile.hh"
+#include "data/PositionsFile.hh"
+#include "data/BclBuffer.hh"
+#include "conversion/Stage.hh"
+#include "conversion/Task.hh"
+#include "conversion/SampleIndex.hh"
+
+#include <boost/filesystem/path.hpp>
+#include <boost/ptr_container/ptr_vector.hpp>
+#include <vector>
+#include <boost/shared_ptr.hpp>
+#include <mutex>
+#include <atomic>
+
+namespace bcl2fastq {
+
+
+namespace conversion {
+
+class BclLoaderTaskManager;
+
+/// \brief Task: Load BCL data into buffer.
+class BclLoadTask : public Task
+{
+public:
+
+ /// \brief Constructor.
+ /// \param bclFile BCL file to be loaded.
+ /// \param cycleBciFile Cycle BCI file.
+ /// \param aggregateTilesFlag All tiles aggregated into single file flag.
+ /// \param laneInfo Lane meata data.
+ /// \param cycleInfo Cycle meta data.
+ /// \param clustersCount Number of clusters to be loaded.
+ /// \param ignoreMissingBcls Ignore missing BCLs flag.
+ /// \param bclIdx bcl index
+ /// \param outputBuffer Buffer for data to be loaded to.
+ BclLoadTask(
+ std::shared_ptr<BclLoaderTaskManager>& taskManager,
+ data::RawBclBufferGroup& bclData,
+ std::mutex* aggregatedBclFileMutex,
+ common::TileAggregationMode aggregateTilesFlag,
+ const layout::LaneInfo &laneInfo,
+ const layout::CycleInfo &cycleInfo,
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfoIter,
+ bool ignoreMissingBcls,
+ size_t bclIdx,
+ DemuxBuffer& outputBuffer
+ );
+
+public:
+
+ virtual PriorityLevel getPriority() const { return PriorityLevel::Two; }
+
+ virtual bool execute(int32_t threadNum);
+
+private:
+
+ void checkBytesRead(int32_t bytesRead,
+ int32_t expectedBytes,
+ int32_t tileIdx,
+ const boost::filesystem::path& bclPath);
+
+ /// \brief Read all tiles (each tile from a separate file).
+ bool readSeparateTiles();
+
+ /// \brief Read all tiles (each tile from the same file).
+ bool readAggregatedTiles();
+
+ /// \brief Read all tiles (each tile from a CBCL file).
+ bool readCbclTiles();
+
+ std::stringstream &getTileDescriptor(std::stringstream &msg, data::PerCycleData perCycleData);
+
+ /// \brief BCL file to be loaded.
+ data::RawBclBufferGroup& bclData_;
+
+ std::mutex* aggregatedBclFileMutex_;
+
+ /// \brief ALl tiles aggregated into single file flag.
+ common::TileAggregationMode aggregateTilesMode_;
+
+ /// \brief Lane meta data.
+ const layout::LaneInfo &laneInfo_;
+
+ /// \brief Cycle meta data.
+ const layout::CycleInfo &cycleInfo_;
+
+ /// \brief Tile meta data.
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfoIter_;
+
+ /// \brief Ignore missing BCLs flag.
+ bool ignoreMissingBcls_;
+
+ /// \brief Index into bcl buffer
+ size_t bclIdx_;
+
+ /// \brief Buffer to load data to.
+ DemuxBuffer &outputBuffer_;
+};
+
+
+/// \brief Task: Load positions data into buffer.
+class PositionsLoadTask : public Task
+{
+public:
+
+ /// \brief Constructor.
+ /// \param positionsFile Positions file.
+ /// \param aggregateTilesFlag All tiles aggregated into single file flag.
+ /// \param laneInfo Lane meata data.
+ /// \param tileInfo Tile meta data.
+ /// \param clustersCount Number of clusters to be loaded.
+ /// \param ignoreMissingPositions Ignore missing positions flag.
+ /// \param outputBuffer Buffer for data to be loaded to.
+ PositionsLoadTask(
+ std::shared_ptr<BclLoaderTaskManager>& taskManager,
+ data::RawBclBufferGroup& bclData,
+ bool aggregateTilesFlag,
+ const layout::LaneInfo &laneInfo,
+ layout::LaneInfo::TileInfosContainer::const_iterator &tileInfo,
+ const std::vector<common::ClustersCount>& clustersCounts,
+ bool ignoreMissingPositions,
+ data::BclBufferVec& outputBuffer,
+ std::shared_ptr<data::BclBuffer::PatternedPositionsContainer>& patternedFlowcellPositions
+ );
+
+public:
+
+ // Since the positions could be shared if using a patterned flowcell, the priority is a bit higher.
+ virtual PriorityLevel getPriority() const { return PriorityLevel::One; }
+
+ virtual bool execute(int32_t threadNum);
+
+ bool execute(const common::RawDataBuffer& inputBuffer,
+ data::BclBuffer::PositionsContainer& outputBuffer);
+
+private:
+
+ /// \brief BCL file to be loaded.
+ data::RawBclBufferGroup& bclData_;
+
+ /// \brief All tiles aggregated into single file flag.
+ bool aggregateTilesFlag_;
+
+ /// \brief Lane meta data.
+ const layout::LaneInfo &laneInfo_;
+
+ /// \brief Tile meta data.
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfoIter_;
+
+ /// \brief Number of clusters to load.
+ const std::vector<common::ClustersCount>& clustersCounts_;
+
+ /// \brief Ignore missing positions flag.
+ bool ignoreMissingPositions_;
+
+ /// \brief Buffer to load data to.
+ data::BclBufferVec &outputBuffer_;
+
+ /// \brief Buffer for patterned flowcell positions
+ std::shared_ptr<data::BclBuffer::PatternedPositionsContainer>& patternedFlowcellPositions_;
+};
+
+
+/// \brief Task: Load filter data into buffer.
+class FilterLoadTask : public Task
+{
+public:
+
+ /// \brief Constructor.
+ /// \param filterFile Filter file.
+ /// \param aggregateTilesFlag All tiles aggregated into single file flag.
+ /// \param laneInfo Lane meata data.
+ /// \param tileInfo Tile meta data.
+ /// \param clustersCount Number of clusters to be loaded.
+ /// \param ignoreMissingFilters Ignore missing filters flag.
+ /// \param outputBuffer Buffer for data to be loaded to.
+ /// \param controlsOutputBuffer Buffer for controls data to be loaded to.
+ FilterLoadTask(
+ std::shared_ptr<BclLoaderTaskManager>& taskManager,
+ data::RawBclBufferGroup& bclData,
+ common::TileAggregationMode tileAggregationMode,
+ const layout::LaneInfo &laneInfo,
+ layout::LaneInfo::TileInfosContainer::const_iterator& tileInfo,
+ const std::vector<common::ClustersCount>& clustersCounts,
+ bool ignoreMissingFilters,
+ data::BclBufferVec& outputBuffer
+ );
+
+public:
+
+ virtual PriorityLevel getPriority() const { return PriorityLevel::Two; }
+
+ virtual bool execute(int32_t threadNum);
+
+private:
+
+ /// \brief Read the aggregate tiles filter file.
+ /// \param filePath Path to file.
+ void readAggregateTilesFilterFile(const layout::TileInfo& tileInfo,
+ common::ClustersCount clustersCount,
+ data::BclBuffer& outputBuffer,
+ const boost::filesystem::path& filePath = boost::filesystem::path());
+
+ /// \brief Read the filter file for a single tile.
+ /// \param filePath Path to file.
+ void readTileFilterFile(const boost::filesystem::path& filePath,
+ const layout::TileInfo& tileInfo,
+ data::BclBuffer& outputBuffer);
+
+ /// \brief Read the filter file.
+ /// \param filterFile File to read from.
+ void readFilterFile(common::RawDataBuffer& filterData,
+ const layout::TileInfo& tileInfo,
+ data::BclBuffer& outputBuffer,
+ bool isAggregatedTiles,
+ bool skipHeader);
+
+ /// \brief Validate the filter records
+ /// \param recordsRead Number of records read from file.
+ /// \param filePath Path to file.
+ void validateFilterRecords(std::size_t recordsRead,
+ common::ClustersCount clustersCount,
+ const boost::filesystem::path& filePath,
+ data::BclBuffer& outputBuffer);
+
+ data::RawBclBufferGroup& bclData_;
+
+ /// \brief File aggregation type.
+ common::TileAggregationMode tileAggregationMode_;
+ /// \brief All tiles aggregated into single file flag.
+ bool aggregateTilesFlag_;
+
+ /// \brief Lane meta data.
+ const layout::LaneInfo &laneInfo_;
+
+ /// \brief Tile meta data.
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfoIter_;
+
+ /// \brief Number of clusters to load.
+ const std::vector<common::ClustersCount>& clustersCounts_;
+
+ /// \brief Ignore missing filters flag.
+ bool ignoreMissingFilters_;
+
+ /// \brief Buffer to load control data to.
+ data::BclBufferVec& outputBuffer_;
+};
+
+
+class ControlLoadTask : public Task
+{
+public:
+ ControlLoadTask(
+ std::shared_ptr<BclLoaderTaskManager>& taskManager,
+ data::RawBclBufferGroup& bclData,
+ const layout::LaneInfo &laneInfo,
+ bool ignoreMissingControls,
+ data::BclBufferVec& outputBuffer);
+
+ virtual bool execute(int32_t threadNum);
+
+ virtual PriorityLevel getPriority() const { return PriorityLevel::Two; }
+
+private:
+ data::RawBclBufferGroup& bclData_;
+
+ /// \brief Lane meta data.
+ const layout::LaneInfo &laneInfo_;
+
+ /// \brief Tile meta data.
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfoIter_;
+
+ /// \brief Ignore missing controls flag.
+ bool ignoreMissingControls_;
+
+ /// \brief Buffer to load data to.
+ data::BclBufferVec& outputBuffer_;
+};
+
+class BclLoaderTaskManager : public TaskManager
+{
+public:
+ BclLoaderTaskManager(bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions,
+ bool ignoreMissingControls,
+ common::TileAggregationMode aggregateTilesFlag,
+ std::shared_ptr<data::RawBclBufferGroup>& inputBuffer,
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& inputBuffersToRecycle,
+ std::shared_ptr<DemuxBuffer>& outputBuffer,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBufferQueue,
+ std::shared_ptr<data::BclBuffer::PatternedPositionsContainer>& patternedFlowcellPositions);
+
+ virtual ~BclLoaderTaskManager();
+
+ std::shared_ptr<data::RawBclBufferGroup> getInputBuffer() { return inputBuffer_;}
+ std::shared_ptr<DemuxBuffer> getOutputBuffer() { return outputBuffer_; }
+
+ static void waitForAllTasksToFinish();
+
+private:
+ void postExecute(DemuxBuffer& postExBuffer);
+
+ void bclMismatchCount(const std::string& fileType,
+ common::CycleNumber cycleNumber,
+ common::TileNumber tileNumber,
+ common::RawDataBuffer::size_type realSize,
+ common::RawDataBuffer::size_type expectedSize);
+
+ void createUniqueFakePositions(data::BclBuffer& outputBuffer,
+ data::PerCycleData::BclsContainer::size_type bufferSize);
+
+ void validateCycleMetadata(data::PerCycleData& cycleData,
+ data::PerCycleData& prevCycleData);
+
+ bool ignoreMissingBcls_;
+ bool ignoreMissingFilters_;
+ bool ignoreMissingPositions_;
+ bool ignoreMissingControls_;
+
+ common::TileAggregationMode aggregateTilesFlag_;
+
+ /// \brief Unique index per read if reading the Posisions file failed.
+ data::PositionsFile::Record::ClusterCoordinate uniqueFailedReadIndex_;
+
+ std::shared_ptr<data::RawBclBufferGroup> inputBuffer_;
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& inputBuffersToRecycle_;
+ std::shared_ptr<DemuxBuffer> outputBuffer_;
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBufferQueue_;
+ std::shared_ptr<data::BclBuffer::PatternedPositionsContainer> patternedFlowcellPositions_;
+
+ static std::atomic<uint32_t> numTaskManagers_;
+ static std::condition_variable cvAllTaskManagersDone_;
+ static std::mutex mut_;
+};
+
+
+/// \brief BCL loader.
+class BclLoader : public Stage
+{
+public:
+
+ /// \brief Constructor.
+ /// \param threadsCount Number of threads.
+ /// \param layout Flowcell layout.
+ /// \param laneInfo Lane meata data.
+ /// \param ignoreMissingBcls Ignore missing BCLs flag.
+ /// \param ignoreMissingFilters Ignore missing filters flag.
+ /// \param ignoreMissingPositions Ignore missing positions flag.
+ BclLoader(
+ TaskQueue& taskQueue,
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& inputBuffersToUse,
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& inputBuffersToRecycle,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBuffersToSubmit,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBuffersToUse,
+ const layout::Layout &layout,
+ const layout::LaneInfo &laneInfo,
+ bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions,
+ bool ignoreMissingControls,
+ uint32_t numTilesPerBuffer
+ );
+
+ virtual std::shared_ptr<BclLoaderTaskManager> getNewTaskManager();
+
+ virtual bool startNewTasks();
+
+private:
+
+ virtual void terminate();
+
+ /// \brief BCL files container type definition.
+ typedef std::vector<std::shared_ptr<data::BclFile>> BclFilesContainer;
+
+ /// \brief Cycle BCI files container type definition.
+ typedef boost::ptr_vector<data::CycleBCIFile> CycleBciFilesContainer;
+
+ typedef boost::ptr_vector<std::mutex> MutexContainer;
+
+private:
+
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& inputBuffersToUse_;
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& inputBuffersToRecycle_;
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBuffersToSubmit_;
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBuffersToUse_;
+
+ /// \brief Layout.
+ const layout::Layout &layout_;
+
+ /// \brief Current lane.
+ const layout::LaneInfo &laneInfo_;
+
+ /// \brief Ignore missing BCLs flag.
+ bool ignoreMissingBcls_;
+
+ /// \brief Ignore missing filters flag.
+ bool ignoreMissingFilters_;
+
+ /// \brief Ignore missing positions flag.
+ bool ignoreMissingPositions_;
+
+ /// \brief Ignore missing controls.
+ bool ignoreMissingControls_;
+
+ /// \brief Tile being currently processed.
+ layout::LaneInfo::TileInfosContainer::const_iterator currentTileInfo_;
+
+ MutexContainer aggregatedBclMutexes_;
+
+ /// \brief Cycle BCI files (one per cycle).
+ CycleBciFilesContainer cycleBciFiles_;
+
+ /// \brief Positions for a patterned flowcell.
+ std::shared_ptr<data::BclBuffer::PatternedPositionsContainer> patternedFlowcellPositions_;
+
+ /// \brief Cluster counts for the current set of tiles.
+ std::vector<common::ClustersCount> clustersCounts_;
+
+ uint32_t numBuffersProcessed_;
+
+ uint32_t numBuffersCreated_;
+
+ const uint32_t numTilesPerBuffer_;
+
+ std::shared_ptr<data::RawBclBufferGroup> prevInputBuffer_;
+
+ bool terminated_;
+};
+
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_CONVERSION_BCLLOADER_HH
+
+
diff --git a/src/cxx/include/conversion/BclReader.hh b/src/cxx/include/conversion/BclReader.hh
new file mode 100644
index 0000000..c58d7ad
--- /dev/null
+++ b/src/cxx/include/conversion/BclReader.hh
@@ -0,0 +1,395 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BclReader.hh
+ *
+ * \brief Declaration of BCL reader.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_BCLREADER_HH
+#define BCL2FASTQ_CONVERSION_BCLREADER_HH
+
+
+#include "data/BclFileReader.hh"
+#include "data/RawBclBuffer.hh"
+#include "io/SyncFile.hh"
+#include "io/GzipDecompressor.hh"
+#include "io/FileBufWithReopen.hh"
+#include "layout/Layout.hh"
+#include "data/CycleBCIFile.hh"
+#include "data/BclFile.hh"
+#include "conversion/Stage.hh"
+#include "conversion/Task.hh"
+#include "conversion/SampleIndex.hh"
+#include "common/Types.hh"
+
+#include <vector>
+#include <boost/filesystem/path.hpp>
+#include <boost/ptr_container/ptr_vector.hpp>
+#include <atomic>
+
+namespace bcl2fastq {
+namespace conversion {
+
+class BclReaderTaskManager;
+
+
+class PatternedPositionsContainer : public common::RawDataBuffer
+{
+public:
+ PatternedPositionsContainer() : common::RawDataBuffer(), mut_(), cv_(), isReady_(false) { }
+
+ virtual ~PatternedPositionsContainer() { }
+
+ void setReady() { isReady_ = true; cv_.notify_all(); }
+
+ void waitForReady() { if (!isReady_) { std::unique_lock<std::mutex> lock(mut_); cv_.wait(lock, [this] { return isReady_.load(); }); } }
+
+private:
+ std::mutex mut_;
+ std::condition_variable cv_;
+ std::atomic<bool> isReady_;
+};
+
+/// \brief Task: Read BCL data into buffer.
+class BclReadTask : public Task
+{
+public:
+
+ /// \brief Constructor.
+ /// \param taskManager Manages all tasks.
+ /// \param bclFile BCL file to be loaded.
+ /// \param outputBuffer Buffer for data to be loaded to.
+ BclReadTask(
+ std::shared_ptr<BclReaderTaskManager>& taskManager,
+ data::BclFileReader& bclFileReader,
+ data::RawBclBufferGroup& outputBuffer
+ );
+
+public:
+
+ virtual bool execute(int32_t threadNum);
+
+private:
+
+ /// \brief BCL file reader.
+ data::BclFileReader &bclFileReader_;
+
+ /// \brief Tile meta data.
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfoIter_;
+
+ /// \brief Buffer to load data to.
+ data::RawBclBufferGroup &outputBuffer_;
+};
+
+
+/// \brief Task: Read positions data into buffer.
+class PositionsReadTask : public Task
+{
+public:
+
+ /// \brief Constructor.
+ /// \param positionsFile Positions file.
+ /// \param intensitiesDir Path to intensities directory.
+ /// \param aggregateTilesFlag All tiles aggregated into single file flag.
+ /// \param laneInfo Lane meata data.
+ /// \param tileInfo Tile meta data.
+ /// \param ignoreMissingPositions Ignore missing positions flag.
+ /// \param outputBuffer Buffer for data to be loaded to.
+ PositionsReadTask(
+ std::shared_ptr<BclReaderTaskManager>& taskManager,
+ std::shared_ptr<io::SyncFile> &positionsFile,
+ const boost::filesystem::path &intensitiesDir,
+ bool aggregateTilesFlag,
+ bool isPatternedFlowcell,
+ const layout::LaneInfo &laneInfo,
+ layout::LaneInfo::TileInfosContainer::const_iterator &tileInfo,
+ bool ignoreMissingPositions,
+ data::RawBclBufferGroup& outputBuffer,
+ std::shared_ptr<PatternedPositionsContainer>& patternedFlowcellPositions
+ );
+
+public:
+
+ virtual bool execute(int32_t threadNum);
+
+private:
+
+ bool readEntirePosFile(common::RawDataBuffer& outputBuffer);
+
+ bool readPartOfAggregatedPosFile(common::RawDataBuffer& outputBuffer);
+
+ /// \brief Positions file.
+ std::shared_ptr<io::SyncFile>& positionsFile_;
+
+ /// \brief Intensities directory path.
+ const boost::filesystem::path &intensitiesDir_;
+
+ /// \brief All tiles aggregated into single file flag.
+ bool aggregateTilesFlag_;
+
+ /// \brief Single location file flag.
+ bool isPatternedFlowcell_;
+
+ /// \brief Lane meta data.
+ const layout::LaneInfo &laneInfo_;
+
+ /// \brief Tile meta data.
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfoIter_;
+
+ /// \brief Ignore missing positions flag.
+ bool ignoreMissingPositions_;
+
+ /// \brief Buffer to load data to.
+ data::RawBclBufferGroup &outputBuffer_;
+
+ /// \brief Buffer for patterned flowcell positions
+ std::shared_ptr<PatternedPositionsContainer> patternedFlowcellPositions_;
+};
+
+
+/// \brief Task: Read filter data into buffer.
+class FilterReadTask : public Task
+{
+public:
+
+ /// \brief Constructor.
+ /// \param filterFile Filter file.
+ /// \param inputDir Path to input directory.
+ /// \param aggregateTilesFlag All tiles aggregated into single file flag.
+ /// \param laneInfo Lane meata data.
+ /// \param tileInfo Tile meta data.
+ /// \param ignoreMissingFilters Ignore missing filters flag.
+ /// \param outputBuffer Buffer for data to be loaded to.
+ /// \param controlsOutputBuffer Buffer for controls data to be loaded to.
+ FilterReadTask(
+ std::shared_ptr<BclReaderTaskManager>& taskManager,
+ std::shared_ptr<io::SyncFile> &filterFile,
+ const boost::filesystem::path &inputDir,
+ common::TileAggregationMode tileAggregationMode,
+ const layout::LaneInfo &laneInfo,
+ layout::LaneInfo::TileInfosContainer::const_iterator& tileInfo,
+ bool ignoreMissingFilters,
+ data::RawBclBufferGroup& outputBuffer
+ );
+
+public:
+
+ virtual bool execute(int32_t threadNum);
+
+private:
+
+ bool readEntireFilterFile(data::RawBclBuffer& rawBclBuffer);
+
+ bool readPartOfAggregatedFilterFile(common::RawDataBuffer& outputBuffer);
+
+ /// \brief Validate the filter records
+ /// \param recordsRead Number of records read from file.
+ /// \param filePath Path to file.
+ void validateFilterRecords(std::size_t recordsRead,
+ const boost::filesystem::path& filePath,
+ data::RawBclBuffer& outputBuffer);
+
+ /// \brief Filter file.
+ std::shared_ptr<io::SyncFile>& filterFile_;
+
+ /// \brief Input directory path.
+ const boost::filesystem::path &inputDir_;
+
+ /// \brief File aggregation type.
+ common::TileAggregationMode tileAggregationMode_;
+ /// \brief All tiles aggregated into single file flag.
+ bool aggregateTilesFlag_;
+
+
+ /// \brief Lane meta data.
+ const layout::LaneInfo &laneInfo_;
+
+ /// \brief Tile meta data.
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfoIter_;
+
+ /// \brief Ignore missing filters flag.
+ bool ignoreMissingFilters_;
+
+ /// \brief Buffer to load control data to.
+ data::RawBclBufferGroup& outputBuffer_;
+};
+
+
+class ControlReadTask : public Task
+{
+public:
+ ControlReadTask(std::shared_ptr<BclReaderTaskManager>& taskManager,
+ const boost::filesystem::path &inputDir,
+ const layout::LaneInfo &laneInfo,
+ layout::LaneInfo::TileInfosContainer::const_iterator &tileInfo,
+ bool ignoreMissingControls,
+ data::RawBclBufferGroup& outputBuffer);
+
+ virtual bool execute(int32_t threadNum);
+
+private:
+ bool readEntireControlFile(common::RawDataBuffer& outputBuffer);
+
+ /// \brief Input directory path.
+ const boost::filesystem::path &inputDir_;
+
+ /// \brief Lane meta data.
+ const layout::LaneInfo &laneInfo_;
+
+ /// \brief Tile meta data.
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfoIter_;
+
+ /// \brief Ignore missing controls flag.
+ bool ignoreMissingControls_;
+
+ /// \brief Buffer to load data to.
+ data::RawBclBufferGroup& outputBuffer_;
+};
+
+class BclReaderTaskManager : public TaskManager
+{
+public:
+ BclReaderTaskManager(bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions,
+ bool ignoreMissingControls,
+ std::shared_ptr<data::RawBclBufferGroup>& outputBuffer,
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& outputBufferQueue,
+ std::shared_ptr<PatternedPositionsContainer>& patternedFlowcellPositions);
+
+ virtual ~BclReaderTaskManager();
+
+ data::RawBclBufferGroup& getOutputBuffer() { return *outputBuffer_; }
+
+ static void waitForAllTasksToFinish();
+
+private:
+ void postExecute();
+
+ bool ignoreMissingBcls_;
+ bool ignoreMissingFilters_;
+ bool ignoreMissingPositions_;
+ bool ignoreMissingControls_;
+
+ std::shared_ptr<data::RawBclBufferGroup> outputBuffer_;
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& outputBufferQueue_;
+ std::shared_ptr<PatternedPositionsContainer> patternedFlowcellPositions_;
+
+ static std::atomic<uint32_t> numTaskManagers_;
+ static std::condition_variable cvAllTaskManagersDone_;
+ static std::mutex mut_;
+};
+
+
+/// \brief BCL loader.
+class BclReader : public Stage
+{
+public:
+
+ /// \brief Constructor.
+ /// \param threadsCount Number of threads.
+ /// \param layout Flowcell layout.
+ /// \param laneInfo Lane meata data.
+ /// \param ignoreMissingBcls Ignore missing BCLs flag.
+ /// \param ignoreMissingFilters Ignore missing filters flag.
+ /// \param ignoreMissingPositions Ignore missing positions flag.
+ /// \param inputDir Input directory.
+ /// \param intensitiesDir Intensities directory.
+ BclReader(
+ TaskQueue& taskQueue,
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& outputBuffersToSubmit,
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& outputBuffersToUse,
+ const layout::Layout &layout,
+ const layout::LaneInfo &laneInfo,
+ bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions,
+ bool ignoreMissingControls,
+ const boost::filesystem::path &inputDir,
+ const boost::filesystem::path &intensitiesDir
+ );
+
+uint32_t getNumTilesPerBuffer() const { return numTilesPerBuffer_; }
+
+private:
+
+ virtual void terminate();
+
+ void calculateNumTilesPerBuffer();
+
+ virtual std::shared_ptr<BclReaderTaskManager> getNewTaskManager();
+
+ virtual bool startNewTasks();
+
+ /// \brief BCL files container type definition.
+ typedef boost::ptr_vector<io::SyncFile> BclFilesContainer;
+
+ typedef boost::ptr_vector<data::BclFileReader> BclFileReadersContainer;
+
+ /// \brief Cycle BCI files container type definition.
+ typedef boost::ptr_vector<data::CycleBCIFile> CycleBciFilesContainer;
+
+private:
+
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& outputBuffersToSubmit_;
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& outputBuffersToUse_;
+
+ /// \brief Layout.
+ const layout::Layout &layout_;
+
+ /// \brief Current lane.
+ const layout::LaneInfo &laneInfo_;
+
+ /// \brief Ignore missing BCLs flag.
+ bool ignoreMissingBcls_;
+
+ /// \brief Ignore missing filters flag.
+ bool ignoreMissingFilters_;
+
+ /// \brief Ignore missing positions flag.
+ bool ignoreMissingPositions_;
+
+ /// \brief Ignore missing controls.
+ bool ignoreMissingControls_;
+
+ /// \brief Tile being currently processed.
+ layout::LaneInfo::TileInfosContainer::const_iterator currentTileInfo_;
+
+ /// \brief Input directory.
+ boost::filesystem::path inputDir_;
+
+ /// \brief Intensities directory.
+ boost::filesystem::path intensitiesDir_;
+
+ BclFileReadersContainer bclFileReaders_;
+
+ /// \brief Positions file.
+ std::shared_ptr<io::SyncFile> positionsFile_;
+
+ /// \brief Filter file.
+ std::shared_ptr<io::SyncFile> filterFile_;
+
+ /// \brief Positions for a patterned flowcell.
+ std::shared_ptr<PatternedPositionsContainer> patternedFlowcellPositions_;
+
+ uint32_t numTilesPerBuffer_;
+
+ uint32_t numBuffersCreated_;
+
+ uint32_t groupNumber_;
+};
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_CONVERSION_BCLREADER_HH
+
diff --git a/src/cxx/include/conversion/Converter.hh b/src/cxx/include/conversion/Converter.hh
new file mode 100644
index 0000000..c4216b2
--- /dev/null
+++ b/src/cxx/include/conversion/Converter.hh
@@ -0,0 +1,405 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Converter.hh
+ *
+ * \brief Declaration of BCL to FASTQ converter.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_CONVERTER_HH
+#define BCL2FASTQ_CONVERSION_CONVERTER_HH
+
+
+#include <vector>
+#include <map>
+
+//#include <boost/unordered_map.hpp>
+#include <boost/noncopyable.hpp>
+
+#include "data/InteropFile.hh"
+#include "common/Threads.hh"
+#include "config/Bcl2FastqOptions.hh"
+#include "layout/Layout.hh"
+#include "stats/TileStats.hpp"
+#include "stats/BarcodeStats.hh"
+#include "conversion/FastqBuffer.hh"
+#include "conversion/BclReader.hh"
+#include "conversion/BclLoader.hh"
+#include "conversion/Demultiplexer.hh"
+#include "conversion/FastqCreator.hh"
+#include "conversion/FastqWriter.hh"
+#include "conversion/ThreadPool.hh"
+
+
+namespace bcl2fastq
+{
+
+// Forward declarations
+namespace stats
+{
+class ConversionStatsXml;
+class JsonObject;
+class JsonArray;
+class JsonNumber;
+}
+
+namespace conversion
+{
+
+const uint32_t MAX_BCL_BUFFER_QUEUE_DEPTH = 5;
+
+namespace detail
+{
+
+class FastqSummaryStats
+{
+public:
+ FastqSummaryStats(common::SampleNumber sampleNumber,
+ const std::string& sampleName,
+ common::TileNumber tileNumber,
+ common::ClustersCount numberReadsRaw,
+ common::ClustersCount numberReadsPF);
+
+ bool operator<(const FastqSummaryStats& other) const;
+
+ common::SampleNumber sampleNumber_;
+ std::string sampleName_;
+ common::TileNumber tileNumber_;
+ common::ClustersCount numberReadsRaw_;
+ common::ClustersCount numberReadsPF_;
+};
+
+typedef std::vector<FastqSummaryStats> FastqSummaryStatsContainer;
+
+}
+
+/// \todo: consider managing interOp stats from here as well
+class ConverterStats
+{
+ typedef std::pair<layout::LaneInfo,stats::BarcodeHits> UnknownPair;
+ typedef std::map<layout::LaneInfo,stats::BarcodeHits> UnknownBarcodeHits;
+
+ typedef std::pair<layout::LaneInfo,DemultiplexTask::DemuxStats> DemuxPair;
+ typedef std::map<layout::LaneInfo,DemultiplexTask::DemuxStats> LaneBarcodeStats;
+
+ typedef std::map<layout::LaneInfo,ConversionStatsForSampleTileBarcode> LaneTileStats;
+
+ typedef std::map<std::string, stats::BarcodeStats> SampleLaneBarcodeStats;
+ typedef std::map<std::string, SampleLaneBarcodeStats> ProjectSampleLaneBarcodeStats;
+ typedef std::map<std::string, ProjectSampleLaneBarcodeStats> FlowcellProjectSampleLaneBarcodeStats;
+
+ typedef std::map<common::ReadNumber, stats::ReadBarcodeStats > AnnotatedTileReadBarcodeStats;
+ typedef std::pair<stats::TileBarcodeStats, AnnotatedTileReadBarcodeStats > TileReadStats;
+ typedef std::map<common::TileNumber, TileReadStats > TileReadBarcodeStats;
+ typedef std::map<std::string, TileReadBarcodeStats> BarcodeTileReadBarcodeStats;
+ typedef std::map<std::string, BarcodeTileReadBarcodeStats> SampleBarcodeTileReadBarcodeStats;
+ typedef std::map<std::string, SampleBarcodeTileReadBarcodeStats> ProjectSampleBarcodeTileReadBarcodeStats;
+ typedef std::map<std::string, ProjectSampleBarcodeTileReadBarcodeStats> FlowcellProjectSampleBarcodeTileReadBarcodeStats;
+
+ typedef std::map<std::string, stats::BarcodeStats> BarcodeStatsMap;
+ typedef std::map<int, BarcodeStatsMap> BarcodeLaneStatsMap;
+
+ // map the sampleId to sample index and the stats object
+ typedef std::map<std::string, std::pair<size_t, stats::TileBarcodeStats>> SampleReadBarcodeStats;
+
+ typedef std::map<std::string, SampleReadBarcodeStats> ProjectSampleReadBarcodeStats;
+
+public:
+ ConverterStats( const layout::Layout &layout,
+ const boost::filesystem::path &fastqSummaryPartialFileName,
+ const boost::filesystem::path &demuxSummaryPartialFileName,
+ const boost::filesystem::path &adapterTrimmingTxtFile,
+ const boost::filesystem::path &interopFile,
+ const boost::filesystem::path &demuxStatsXmlFile,
+ const boost::filesystem::path &conversionStatsXmlFile,
+ const boost::filesystem::path &statsJsonFile);
+
+ boost::filesystem::path getDemuxStatsXmlPath() const {return boost::filesystem::canonical( demuxStatsXmlFile_ );}
+ boost::filesystem::path getConversionStatsXmlPath() const {return boost::filesystem::canonical( conversionStatsXmlFile_ );}
+
+ DemultiplexTask::DemuxStats &getDemuxStats(const layout::LaneInfo &laneInfo) {return demuxStats_[laneInfo];}
+ ConversionStatsForSampleTileBarcode &getConversionStats(const layout::LaneInfo &laneInfo) {return conversionStats_[laneInfo];}
+ stats::BarcodeHits &getUnknownBarcodeStat(const layout::LaneInfo &laneInfo) {return topUnknownBarcodes_[laneInfo];}
+
+ void dumpInteropStats() const;
+ void dumpDemuxStats();
+ void dumpConversionStats();
+
+private:
+ void recordBarcodeStats(const BarcodeStatsMap& barcodeStatsMap,
+ const std::string& barcodeName,
+ stats::JsonArray& jsonIndexMetrics);
+
+ void recordReadStats(stats::ConversionStatsXml& conversionStatsXml,
+ const AnnotatedTileReadBarcodeStats& tileStats,
+ unsigned int laneNumber,
+ common::TileNumber tileNumber,
+ const std::string& barcodeName,
+ const std::string& project,
+ const std::string& sampleId,
+ const std::string& sampleName,
+ std::map<std::string, std::string>& uniqueSampleNameMap,
+ std::map<common::ReadNumber, stats::JsonNumber*>& readYieldMap,
+ std::map<common::ReadNumber, stats::JsonNumber*>& readYieldQ30Map,
+ std::map<common::ReadNumber, stats::JsonNumber*>& readQscoreSumMap,
+ std::map<std::pair<std::string, std::string>, std::vector<size_t>>& trimmedCountMap,
+ stats::JsonNumber* yield,
+ stats::JsonArray* jsonReadMetrics);
+
+ void recordUnknownBarcodes(stats::JsonObject& jsonStats,
+ stats::ConversionStatsXml& conversionStatsXml) const;
+
+ void writeAdapterStats(const layout::LaneInfo& laneInfo,
+ const ProjectSampleReadBarcodeStats& stats,
+ std::ostream& adapterOstr,
+ std::ostream& trimLengthOstr,
+ std::map<std::pair<std::string, std::string>, std::vector<size_t>>& trimmedBaseMap) const;
+
+ void getSampleNameAndNumber(const layout::LaneInfo& laneInfo,
+ const std::string& project,
+ const std::string& sampleId,
+ std::string& sampleName,
+ common::SampleNumber& sampleNumber) const;
+
+ void writeSummaryStats(const ProjectSampleBarcodeTileReadBarcodeStats& stats,
+ const layout::LaneInfo& laneInfo) const;
+
+ void writeFastqSummaryStats(common::LaneNumber laneNumber,
+ const detail::FastqSummaryStatsContainer& fastqSummaryStats) const;
+
+ void writeDemuxSummaryStats(common::LaneNumber laneNumber,
+ const detail::FastqSummaryStatsContainer& fastqSummaryStats) const;
+
+ void writeAdapterStats(common::LaneNumber laneNumber,
+ const std::string& project,
+ const std::string& sampleId,
+ const std::string& sampleName,
+ common::SampleNumber sampleNumber,
+ const stats::AllReadsStats::TrimLengthCountForReads& trimLengthCountForReads,
+ std::ostream& adapterOstr,
+ std::ostream& trimLengthOstr,
+ std::map<std::pair<std::string, std::string>, std::vector<size_t>>& trimmedBaseMap) const;
+
+ void deserializeMetadata(const layout::LaneInfo &laneInfo,
+ size_t sampleIndex,
+ size_t barcodeIndex,
+ std::string &indexName,
+ std::string &sampleId,
+ std::string &projectName,
+ std::string &sampleName,
+ common::SampleNumber &sampleNumber) const;
+
+ const layout::Layout &layout_;
+
+ /// \brief Path to FastqSummaryF1* file.
+ const boost::filesystem::path fastqSummaryPartialFileName_;
+
+ /// \brief Path to DemuxSummaryF1* file.
+ const boost::filesystem::path demuxSummaryPartialFileName_;
+
+ /// \brief Path to AdapterTrimming.txt file
+ const boost::filesystem::path adapterTrimmingTxtFile_;
+
+ /// \brief Path to IndexMetricsOut.bin file
+ const boost::filesystem::path interopFile_;
+
+ /// \brief Path to DemultiplexingStats.xml file
+ const boost::filesystem::path demuxStatsXmlFile_;
+
+ /// \brief Path to ConversionStats.xml file
+ const boost::filesystem::path conversionStatsXmlFile_;
+
+ /// \brief Path to Stats.json file.
+ const boost::filesystem::path statsJsonFile_;
+
+ /// \brief Demultiplexing statistics (one per lane).
+ LaneBarcodeStats demuxStats_;
+
+ /// \brief Basecalling statistics (one per lane).
+ LaneTileStats conversionStats_;
+
+ /// \brief Table of unknown barcodes (one per lane).
+ UnknownBarcodeHits topUnknownBarcodes_;
+
+ std::vector<stats::AllReadsStats::TrimLengthCountForReads> trimLengthCountForReadsPerLane_;
+
+ /// \brief Map the index name to the BarcodeStats object
+ BarcodeLaneStatsMap barcodeLaneStatsMap_;
+};
+
+
+/// \brief BCL to FASTQ converter.
+///
+/// \dot
+/// digraph LAYOUT {
+/// fontsize=12;
+///
+/// source [ label="BCLs" shape=folder ];
+/// stage1 [ label="load BCLs" URL="\ref BclLoader" shape=component ];
+/// buffer1 [ label="BCL Buffer" URL="\ref BclBuffer" shape=box3d ];
+/// stage2 [ label="demultiplex" URL="\ref Demultiplexer" shape=component ];
+/// buffer2 [ label="BCL Buffer" URL="\ref BclBuffer" shape=box3d ];
+/// stage3 [ label="create FASTQs" URL="\ref FastqCreator" shape=component ];
+/// buffer3 [ label="FASTQ Buffer" URL="\ref FastqBuffer" shape=box3d ];
+/// stage4 [ label="write FASTQs" URL="\ref FastqWriter" shape=component ];
+/// sink [ label="FASTQs" shape=folder ];
+///
+/// {
+/// rank=same;
+/// source -> stage1;
+/// }
+/// {
+/// rank=same;
+/// stage1 -> buffer1;
+/// }
+/// {
+/// rank=same;
+/// buffer1 -> stage2;
+/// }
+/// {
+/// rank=same;
+/// stage2 -> buffer2;
+/// }
+/// {
+/// rank=same;
+/// buffer2 -> stage3;
+/// }
+/// {
+/// rank=same;
+/// stage3 -> buffer3;
+/// }
+/// {
+/// rank=same;
+/// buffer3 -> stage4;
+/// }
+/// {
+/// rank=same;
+/// stage4 -> sink;
+/// }
+///
+/// \enddot
+class Converter : private boost::noncopyable
+{
+public:
+ /// \brief Constructor.
+ /// \param options Program options.
+ /// \param layout Flowcell layout.
+ /// \param laneInfo Lane metadata.
+ Converter(
+ const bcl2fastq::config::Bcl2FastqOptions &options,
+ const layout::Layout &layout,
+ const layout::LaneInfo &laneInfo,
+ bcl2fastq::conversion::ConverterStats &allStats
+ );
+
+public:
+
+ static std::unique_ptr<Stage> createNewFastqCreator(TaskQueue& processTaskQueue,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inUseDemuxBufferQueue,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& recycledBclBufferQueue,
+ ThreadSafeMap<uint32_t, std::shared_ptr<FastqBuffer>>& inUseFastqBufferMap,
+ ThreadSafeQueue<std::shared_ptr<FastqBuffer>>& recycledFastqBufferQueue,
+ const layout::Layout& layout,
+ const layout::LaneInfo& laneInfo,
+ const bcl2fastq::config::Bcl2FastqOptions& options,
+ bcl2fastq::conversion::ConverterStats& allStats);
+
+ /// \brief Run conversion.
+ void run();
+
+private:
+
+ /// \brief Stage references container type definition.
+ typedef std::vector<boost::reference_wrapper<Stage> > StageRefsContainer;
+
+ /// \brief Thread function executing given stage.
+ /// \param stages Individual stages to be executed.
+ /// \param threadNum Thread number.
+ void stageThreadFunction(StageRefsContainer &stages, common::ThreadVector::size_type threadNum);
+
+private:
+
+ class StatsFinalizer : private boost::noncopyable
+ {
+ public:
+ StatsFinalizer(bool isOnly1Sample,
+ const ConversionStatsForSampleTileBarcode& conversionStats,
+ DemultiplexTask::DemuxStats& demuxStats);
+
+ ~StatsFinalizer();
+
+ private:
+ DemultiplexTask::DemuxStats& demuxStats_;
+ const ConversionStatsForSampleTileBarcode& conversionStats_;
+ bool isOnly1Sample_;
+ };
+
+ /// \brief Flowcell layout.
+ const layout::Layout &layout_;
+
+ /// \brief lane info
+ const layout::LaneInfo& laneInfo_;
+
+ /// \brief Updates stats upon destruction.
+ StatsFinalizer statsFinalizer_;
+
+ /// \brief Stats
+ ConverterStats allStats_;
+
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>> inUseRawBclBufferQueue_;
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>> recycledRawBclBufferQueue_;
+
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>> inUseBclBufferQueue_;
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>> recycledBclBufferQueue_;
+
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>> inUseDemuxBufferQueue_;
+
+ ThreadSafeMap<uint32_t, std::shared_ptr<FastqBuffer>> inUseFastqBufferMap_;
+ ThreadSafeQueue<std::shared_ptr<FastqBuffer>> recycledFastqBufferQueue_;
+
+ TaskQueue processTaskQueue_;
+ TaskQueue ioReadTaskQueue_;
+ TaskQueue ioWriteTaskQueue_;
+
+ Terminator terminator_;
+
+ /// \brief Barcode to sample translation table.
+ const layout::BarcodeTranslationTable barcodeTranslationTable_;
+
+ /// \brief BCL reader.
+ BclReader bclReader_;
+
+ /// \brief BCL loader.
+ BclLoader bclLoader_;
+
+ /// \brief Demultiplexer.
+ Demultiplexer demultiplexer_;
+
+ /// \brief FASTQ creator.
+ std::unique_ptr<Stage> fastqCreator_;
+
+ /// \brief FASTQ writer.
+ FastqWriter fastqWriter_;
+
+ ThreadPool processThreadPool_;
+ ThreadPool ioReadThreadPool_;
+ ThreadPool ioWriteThreadPool_;
+};
+
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_CONVERSION_CONVERTER_HH
+
+
diff --git a/src/cxx/include/conversion/Demultiplexer.hh b/src/cxx/include/conversion/Demultiplexer.hh
new file mode 100644
index 0000000..4fb67b3
--- /dev/null
+++ b/src/cxx/include/conversion/Demultiplexer.hh
@@ -0,0 +1,242 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Demultiplexer.hh
+ *
+ * \brief Declaration of demultiplexer.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_DEMULTIPLEXER_HH
+#define BCL2FASTQ_CONVERSION_DEMULTIPLEXER_HH
+
+
+#include "data/BclBuffer.hh"
+#include "layout/Layout.hh"
+#include "layout/LaneInfo.hh"
+#include "layout/BarcodeTranslationTable.hh"
+#include "stats/DemultiplexingStatsXml.hh"
+#include "stats/BarcodeStats.hh"
+#include "conversion/Stage.hh"
+#include "conversion/Task.hh"
+#include "conversion/SampleIndex.hh"
+
+#include "conversion/FastqIterator.hh"
+
+namespace bcl2fastq
+{
+namespace conversion
+{
+
+class DemuxTaskManager;
+
+/// \brief Task: Demultiplex.
+class DemultiplexTask : public Task
+{
+public:
+
+ /// \brief Demultiplexing statistics. One stats::BarcodeStats per sample, barcode, tile.
+ typedef std::vector<std::vector<std::vector<stats::BarcodeStats>>> DemuxStats;
+
+ class DemuxStatsSubset
+ {
+ public:
+ DemuxStatsSubset(DemuxStats& summaryDemuxStats);
+ ~DemuxStatsSubset();
+
+ DemuxStats& getStats() { return stats_; }
+
+ private:
+ DemuxStats& summaryDemuxStats_;
+ DemuxStats stats_;
+ };
+
+public:
+
+ /// \brief Constructor.
+ /// \param inputBuffer Input buffer.
+ /// \param readsInfos Read infos.
+ /// \param offsetBegin Starting offset.
+ /// \param offsetEnd Ending offset (one past the end).
+ /// \param barcodeTranslationTable Barcode translation table.
+ /// \param tileInfosBegin tile info
+ /// \param numIndexReads number of index reads
+ /// \param includeNonPfClusters If true, include non PF clusters.
+ /// \param numBasesPerByte Number of bases in a byte.
+ /// \param demuxStats Demultiplexing statistics.
+ /// \param outputBuffer Output buffer.
+ DemultiplexTask(
+ std::shared_ptr<DemuxTaskManager>& taskManager,
+ const data::BclBuffer &inputBuffer,
+ const layout::ReadInfosContainer& readInfos,
+ data::BclBuffer::BclCycleContainer::size_type offsetBegin,
+ data::BclBuffer::BclCycleContainer::size_type offsetEnd,
+ const layout::BarcodeTranslationTable &barcodeTranslationTable,
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfosBegin,
+ size_t numIndexReads,
+ bool includeNonPfClusters,
+ common::NumBasesPerByte numBasesPerByte,
+ std::vector<std::shared_ptr<DemultiplexTask::DemuxStatsSubset>>& demuxStats,
+ data::BclBuffer::SamplesContainer &outputBuffer
+ );
+
+ virtual ~DemultiplexTask() { }
+
+ // Demux has a serial step. Give it the highest priority to avoid waiting on this.
+ virtual PriorityLevel getPriority() const { return PriorityLevel::Zero; }
+
+ virtual bool execute(int32_t threadNum);
+
+ template<common::NumBasesPerByte numBasesPerByte>
+ bool execute(int32_t threadNum);
+
+private:
+
+ /// \brief Input buffer.
+ const data::BclBuffer &inputBuffer_;
+
+ /// \brief Read infos.
+ const layout::ReadInfosContainer& readInfos_;
+
+ /// \brief Starting offset.
+ const data::BclBuffer::BclCycleContainer::size_type offsetBegin_;
+
+ /// \brief Ending offset (one past the end).
+ const data::BclBuffer::BclCycleContainer::size_type offsetEnd_;
+
+ /// \brief Barcode to sample translation table.
+ const layout::BarcodeTranslationTable &barcodeTranslationTable_;
+
+ /// \brief Reference to the first tile, used to calculate index.
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfosBegin_;
+
+ /// \brief Number of index reads.
+ size_t numIndexReads_;
+
+ /// \brief Include non-PF clusters
+ bool includeNonPfClusters_;
+
+ /// \brief Number of bases per byte.
+ common::NumBasesPerByte numBasesPerByte_;
+
+ /// \brief Demultiplexing statistics.
+ std::vector<std::shared_ptr<DemultiplexTask::DemuxStatsSubset>>& demuxStats_;
+
+ /// \brief Output buffer.
+ data::BclBuffer::SamplesContainer &outputBuffer_;
+};
+
+class DemuxTaskManager : public TaskManager
+{
+public:
+ DemuxTaskManager(std::shared_ptr<DemuxBuffer>& buffer,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBufferQueue);
+
+ virtual ~DemuxTaskManager();
+
+ DemuxBuffer& getBuffer() { return *buffer_; }
+
+ static void waitForAllTasksToFinish();
+
+private:
+ std::shared_ptr<DemuxBuffer> buffer_;
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBufferQueue_;
+
+ static std::atomic<uint32_t> numTaskManagers_;
+ static std::condition_variable cvAllTaskManagersDone_;
+ static std::mutex mut_;
+};
+
+/// \brief Demultiplexer
+class Demultiplexer : public Stage
+{
+public:
+
+ /// \brief Maximum number of allowed mismatches in a barcode component type definition.
+ typedef std::size_t MismatchesCount;
+
+ /// \brief Maximum allowed mismatches mismatches in a barcode component container type definition.
+ typedef std::vector<MismatchesCount> BarcodeMismatchCountsContainer;
+
+public:
+
+ /// \brief Constructor.
+ /// \param threadsCount Number of threads.
+ /// \param layout Flowcell layout.
+ /// \param laneInfo Lane meata data.
+ /// \param barcodeTranslationTable Table to translate barcodes to samples.
+ /// \includeNonPfClusters If true, include non PF clusters.
+ /// \param interopDir Interop directory.
+ Demultiplexer(
+ TaskQueue& taskQueue,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inputBuffersToUse,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBuffersToSubmit,
+ const layout::Layout &layout,
+ const layout::LaneInfo &laneInfo,
+ const layout::BarcodeTranslationTable &barcodeTranslationTable,
+ bool includeNonPfClusters,
+ int32_t numThreads,
+ DemultiplexTask::DemuxStats &summaryDemuxStats
+ );
+
+ virtual ~Demultiplexer();
+
+ virtual bool startNewTasks();
+
+ void calcFilterOffsets(data::BclBufferVec& buffers);
+
+private:
+
+ virtual void terminate();
+
+ virtual std::shared_ptr<DemuxTaskManager> getNewTaskManager();
+
+ void createTileStats(const layout::BarcodeTranslationTable::SampleMetadata &sampleMetadata);
+
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inputBuffersToUse_;
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBuffersToSubmit_;
+
+ /// \brief Layout.
+ const layout::Layout &layout_;
+
+ /// \brief Current lane.
+ const layout::LaneInfo &laneInfo_;
+
+ /// \brief Barcode to sample translation table.
+ const layout::BarcodeTranslationTable &barcodeTranslationTable_;
+
+ /// \brief Include non-PF clusters
+ bool includeNonPfClusters_;
+
+ /// \brief Number of threads.
+ common::ThreadVector::size_type threadsCount_;
+
+ /// \brief Demultiplexing statistics (summary).
+ DemultiplexTask::DemuxStats &summaryDemuxStats_;
+
+ /// \brief Flag determining whether demultiplexing should take place.
+ bool doDemultiplex_;
+
+ /// \brief Number of index reads.
+ size_t numIndexReads_;
+
+ std::vector<std::shared_ptr<DemultiplexTask::DemuxStatsSubset>> stats_;
+};
+
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+#include "conversion/Demultiplexer.hpp"
+
+#endif // BCL2FASTQ_CONVERSION_DEMULTIPLEXER_HH
+
+
diff --git a/src/cxx/include/conversion/Demultiplexer.hpp b/src/cxx/include/conversion/Demultiplexer.hpp
new file mode 100644
index 0000000..63b2a43
--- /dev/null
+++ b/src/cxx/include/conversion/Demultiplexer.hpp
@@ -0,0 +1,80 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Demultiplexer.hpp
+ *
+ * \brief Declaration of Demultiplexer.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_DEMULTIPLEXER_HPP
+#define BCL2FASTQ_CONVERSION_DEMULTIPLEXER_HPP
+
+
+#include "conversion/Demultiplexer.hh"
+
+namespace bcl2fastq {
+namespace conversion {
+
+template<common::NumBasesPerByte numBasesPerByte>
+bool DemultiplexTask::execute(int32_t threadNum)
+{
+ for (data::BclBuffer::BclCycleContainer::size_type offset = offsetBegin_; offset < offsetEnd_; ++offset)
+ {
+ const bool filterFlag = inputBuffer_.filters_.at(offset).data_;
+
+ layout::Barcode& barcode = outputBuffer_.at(offset).second;
+ // Clear the barcode. This does not free memory, which is good.
+ // We don't want to reallocate the string every time, which
+ // is particularly bad since this is multithreaded and the memory allocation will block.
+ barcode.reset(numIndexReads_);
+
+ if (filterFlag || includeNonPfClusters_)
+ {
+ size_t i = 0;
+ layout::ReadInfo::CycleInfosContainer::const_iterator::difference_type previousReadsLength = 0;
+ for (const auto& readInfo : readInfos_)
+ {
+ size_t currentReadLength = readInfo.cycleInfos().size();
+ if (readInfo.isIndexRead() && currentReadLength != 0)
+ {
+ const FastqConstIterator<numBasesPerByte> begin = FastqConstIterator<numBasesPerByte>(inputBuffer_, offset) + previousReadsLength;
+ const FastqConstIterator<numBasesPerByte> end = FastqConstIterator<numBasesPerByte>(inputBuffer_, offset) + previousReadsLength + currentReadLength;
+
+ size_t dist = end-begin;
+ std::string& barcodeString = barcode.components_[i].getString();
+ barcodeString.resize(dist);
+ for (size_t j = 0; j < dist; ++j)
+ {
+ barcodeString[j] = convertBcl2FastqBase(begin[j]);
+ }
+ ++i;
+ }
+ previousReadsLength += readInfo.cyclesToLoad().size();
+ }
+ unsigned int mismatches(0);
+ layout::BarcodeTranslationTable::SampleMetadata sampleMetadata = barcodeTranslationTable_.translateBarcode(barcode,mismatches);
+
+ outputBuffer_.at(offset).first = sampleMetadata;
+ demuxStats_[threadNum]->getStats().at(sampleMetadata.sampleIndex_)[sampleMetadata.barcodeIndex_][inputBuffer_.tileInfo_ - tileInfosBegin_].incrementBarcodeCount(mismatches);
+ }
+ else
+ {
+ outputBuffer_.at(offset).first = layout::BarcodeTranslationTable::SampleMetadata();
+ }
+ }
+ return true;
+}
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_CONVERSION_DEMULTIPLEXER_HPP
+
diff --git a/src/cxx/include/conversion/FastqBuffer.hh b/src/cxx/include/conversion/FastqBuffer.hh
new file mode 100644
index 0000000..ff36464
--- /dev/null
+++ b/src/cxx/include/conversion/FastqBuffer.hh
@@ -0,0 +1,54 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FastqBuffer.hh
+ *
+ * \brief Declaration of FASTQ buffer.
+ *
+ * \author Marek Balint
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_FASTQBUFFER_HH
+#define BCL2FASTQ_CONVERSION_FASTQBUFFER_HH
+
+#include <stdint.h>
+#include <vector>
+
+namespace bcl2fastq {
+namespace conversion {
+
+
+/// \brief FASTQ buffer.
+struct FastqBuffer
+{
+public:
+ FastqBuffer();
+
+ /// \brief FASTQ data container type definition.
+ /// \note Container is indexed by sample, read and task
+ /// (task is needed for parallelization).
+ typedef std::vector<std::vector<std::vector<std::vector<char> > > > FastqsContainer;
+
+public:
+
+ /// \brief Buffers for FASTQ data.
+ FastqsContainer fastqs_;
+ uint32_t groupNumber_;
+
+ uint32_t getGroupNumber() const { return groupNumber_; }
+ void setGroupNumber(uint32_t groupNumber) { groupNumber_ = groupNumber; }
+};
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_CONVERSION_FASTQBUFFER_HH
+
diff --git a/src/cxx/include/conversion/FastqCreator.hh b/src/cxx/include/conversion/FastqCreator.hh
new file mode 100644
index 0000000..55431af
--- /dev/null
+++ b/src/cxx/include/conversion/FastqCreator.hh
@@ -0,0 +1,408 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FastqCreator.hh
+ *
+ * \brief Declaration of FASTQ creator.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_FASTQCREATOR_HH
+#define BCL2FASTQ_CONVERSION_FASTQCREATOR_HH
+
+#include "layout/Layout.hh"
+#include "stats/TileStats.hpp"
+#include "stats/BarcodeHits.hh"
+#include "data/BclBuffer.hh"
+#include "conversion/FastqBuffer.hh"
+#include "conversion/Stage.hh"
+#include "conversion/Task.hh"
+#include "conversion/FastqIterator.hh"
+#include "conversion/AdapterLocator.hh"
+#include "conversion/SampleIndex.hh"
+
+#include <boost/ptr_container/ptr_vector.hpp>
+
+namespace bcl2fastq
+{
+
+namespace io
+{
+ class GzipCompressor;
+}
+
+namespace conversion
+{
+
+class FastqCreateTaskManager;
+
+/// \brief BaseCalling statistics (one per tile, sample, barcode).
+typedef std::pair< stats::TileBarcodeStats,
+ std::vector<stats::ReadBarcodeStats> > ConversionStats;
+
+// First index is tile, second is sample, third is barcode
+typedef std::vector<std::vector<std::vector<ConversionStats>>> ConversionStatsForSampleTileBarcode;
+
+typedef ConversionStatsForSampleTileBarcode::value_type ConversionStatsForTileBarcode;
+
+// There will be 1 of these objects for every processing thread.
+struct AllConversionStats : private boost::noncopyable
+{
+ AllConversionStats(ConversionStatsForSampleTileBarcode& summaryConversionStats,
+ stats::BarcodeHits& summaryUnknownBarcodes);
+
+ // Record to the summary stats in the destructor
+ ~AllConversionStats();
+
+private:
+ ConversionStatsForSampleTileBarcode& summaryConversionStats_;
+ stats::BarcodeHits& summaryUnknownBarcodes_;
+
+public:
+ ConversionStatsForSampleTileBarcode conversionStats_;
+ stats::BarcodeHits unknownBarcodes_;
+};
+
+// Stats for all tasks (index is task number)
+ typedef std::vector<std::shared_ptr<AllConversionStats>> FastqCreateStats;
+
+ typedef std::shared_ptr<FastqCreateStats> FastqCreateStatsPtr;
+
+
+/// \brief Task: Convert BCLs to FASTQs.
+template<common::NumBasesPerByte numBasesPerByte>
+class FastqCreateTask : public Task
+{
+public:
+
+ /// \brief Constructor.
+ /// \param bclBuffer BCL buffer.
+ /// \param cyclesBegin Beginning of BCL cycles to process.
+ /// \param cyclesEnd End of BCL cycles to process.
+ /// \param flowcellInfo Flowcell meta data.
+ /// \param laneInfo Lane meata data.
+ /// \param readInfo Read meta data.
+ /// \param offsetNumber Offset number.
+ /// \param offsetsBegin Beginning of FASTQ offsets range.
+ /// \param offsetsEnd End of FASTQ offsets range.
+ /// \param maskShortAdapterReads Maximum number of useful bases in read after adapter trimming for which whole read is masked.
+ /// \param adapterStringency Adapter stringency.
+ /// \param maskAdapters Mask adapters.
+ /// \param trimAdapters Trim adapters.
+ /// \param generateReverseComplementFastqs Generate reverse complement FASTQs flag.
+ /// \param includeNonPfClusters Include non-PF clusters in created FASTQ files flag.
+ /// \param layout Flowcell layout.
+ /// \param useBgzf If true, use BGZF compression
+ /// \param compressionLevel Compression level used by zlib
+ /// \param findAdaptersWithSlidingWindow If true, find adapters with the sliding window algorithm.
+ /// \param tileStats Statistics collected for a tile
+ /// \param outputBuffer FASTQ output buffer.
+ FastqCreateTask(
+ std::shared_ptr<FastqCreateTaskManager>& taskManager,
+ const data::BclBufferVec& bclBuffers,
+ size_t cyclesIndex,
+ size_t cycleIndexEnd,
+ const layout::FlowcellInfo &flowcellInfo,
+ const layout::LaneInfo &laneInfo,
+ const layout::ReadInfo& currentReadInfo,
+ SampleIndex::FastqOffsetsContainer::const_iterator offsetsBegin,
+ SampleIndex::FastqOffsetsContainer::const_iterator offsetsEnd,
+ std::size_t maskShortAdapterReads,
+ float adapterStringency,
+ const boost::ptr_vector<AdapterLocator<numBasesPerByte>>& maskAdapters,
+ const boost::ptr_vector<AdapterLocator<numBasesPerByte>>& trimAdapters,
+ bool generateReverseComplementFastqs,
+ bool includeNonPfClusters,
+ bool useBgzf,
+ int compressionLevel,
+ bool findAdaptersWithSlidingWindow,
+ FastqCreateStats &stats,
+ FastqBuffer::FastqsContainer::value_type::value_type::value_type &outputBuffer
+ );
+
+ virtual PriorityLevel getPriority() const { return PriorityLevel::Three; }
+
+ virtual bool execute(int32_t threadNum);
+
+private:
+
+ /// \brief Update tileStats_ accordingly.
+ /// \param basesBegin Beginning of data.
+ /// \param basesEnd End of data.
+ /// \param barcode Index for this read.
+ /// \param filterFlag Whether data passes chastity filter or not.
+ /// \param trimmedCount Number of trimmed bases.
+ /// \return Whether or not the barcode has been found in the table.
+ bool computeStatistics(
+ const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd,
+ SampleIndex::FastqOffsetsContainer::value_type offset,
+ int32_t threadNum,
+ bool filterFlag,
+ size_t trimmedCount,
+ const std::vector<uint32_t>& remappedQscores
+ );
+
+ /// \brief Crate FASTQ.
+ /// \param offset Offset of the FASTQ in the input buffer.
+ /// \param compressor Gzip compressor.
+ void fastqCreate(
+ SampleIndex::FastqOffsetsContainer::value_type offset,
+ int32_t threadNum,
+ io::GzipCompressor& compressor
+ );
+
+ /// \brief Create the barcode strings for the current read.
+ /// \param barcodeStrings Vector of barcode strings.
+ /// \param offset Offset of the FASTQ in the input buffer.
+ void createBarcodeStrings(std::vector<std::string>& barcodeStrings,
+ SampleIndex::FastqOffsetsContainer::value_type offset);
+
+ /// \brief Write a header element.
+ /// \param element Element to write.
+ void writeHeaderElement(const std::string& element);
+
+ /// \brief Create the header.
+ /// \param offset Offset of the FASTQ in the input buffer.
+ /// \param filterFlag True if filter passed.
+ void createHeader(SampleIndex::FastqOffsetsContainer::value_type offset,
+ bool filterFlag);
+
+ /// \brief Create the bases and quality scores.
+ /// \param offset Offset of the FASTQ in the input buffer.
+ /// \param compressor Gzip compressor.
+ /// \param trimmedCount Number of trimmed bases.
+ void createBasesAndQualities(SampleIndex::FastqOffsetsContainer::value_type offset,
+ io::GzipCompressor& compressor,
+ size_t& trimmedCount);
+
+private:
+
+ /// \brief BCL buffer.
+ const data::BclBufferVec& bclBuffers_;
+
+ /// \brief Beginning of BCL cycles to process.
+ const size_t cycleIndex_;
+
+ /// \brief End of BCL cycles to process.
+ const size_t cycleIndexEnd_;
+
+ /// \brief Flowcell meta data.
+ const layout::FlowcellInfo &flowcellInfo_;
+
+ /// \brief Lane meta data.
+ const layout::LaneInfo &laneInfo_;
+
+ /// \brief Read meta data.
+ const layout::ReadInfo &readInfo_;
+
+ /// \brief Sample number.
+ const layout::BarcodeTranslationTable::SampleMetadata sampleMetadata_;
+
+ /// \brief Beginning of FASTQ offsets range.
+ const SampleIndex::FastqOffsetsContainer::const_iterator offsetsBegin_;
+
+ /// \brief End of FASTQ offsets range.
+ const SampleIndex::FastqOffsetsContainer::const_iterator offsetsEnd_;
+
+ /// \brief Buffer to write FASTQs to.
+ FastqBuffer::FastqsContainer::value_type::value_type::value_type &outputBuffer_;
+
+ /// \brief Intermediate buffer.
+ FastqBuffer::FastqsContainer::value_type::value_type::value_type buffer_;
+
+ /// \brief Use BGZF compression if true.
+ bool useBgzf_;
+
+ /// \brief Compression level.
+ int compressionLevel_;
+
+ /// \brief Find adapters with the sliding window algorithm if true.
+ bool findAdaptersWithSlidingWindow_;
+
+ /// \brief Minimum read length after adapter trimming.
+ std::size_t minimumTrimmedReadLength_;
+
+ /// \brief Maximum number of useful bases in read after adapter trimming for which whole read is masked.
+ std::size_t maskShortAdapterReads_;
+
+ /// \brief Adapter stringency.
+ float adapterStringency_;
+
+ /// \brief Mask adapters.
+ const boost::ptr_vector<AdapterLocator<numBasesPerByte>>& maskAdapters_;
+
+ /// \brief Trim adapters.
+ const boost::ptr_vector<AdapterLocator<numBasesPerByte>>& trimAdapters_;
+
+ /// \brief Generate reverse complement FASTQs flag.
+ bool generateReverseComplementFastqs_;
+
+ /// \brief Include non-PF clusters in created FASTQ files.
+ bool includeNonPfClusters_;
+
+ /// \brief Conversion statistics.
+ FastqCreateStats &stats_;
+
+ /// \brief UMI cycles.
+ std::vector< common::CycleRange > umiCycles_;
+};
+
+class FastqCreateTaskManager : public TaskManager
+{
+public:
+ FastqCreateTaskManager(std::shared_ptr<DemuxBuffer>& inputBuffer,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inputBuffersToRecycle,
+ std::shared_ptr<FastqBuffer>& outputBuffer,
+ ThreadSafeMap<uint32_t, std::shared_ptr<FastqBuffer>>& outputBufferMap);
+
+ virtual ~FastqCreateTaskManager();
+
+ static void waitForAllTasksToFinish();
+
+ DemuxBuffer& getInputBuffer() { return *inputBuffer_; }
+ FastqBuffer& getOutputBuffer() { return *outputBuffer_; }
+
+private:
+ std::shared_ptr<DemuxBuffer> inputBuffer_;
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inputBuffersToRecycle_;
+ std::shared_ptr<FastqBuffer> outputBuffer_;
+ ThreadSafeMap<uint32_t, std::shared_ptr<FastqBuffer>>& outputBufferMap_;
+
+ static std::atomic<uint32_t> numTaskManagers_;
+ static std::condition_variable cvAllTaskManagersDone_;
+ static std::mutex mut_;
+};
+
+/// \brief FASTQ creator.
+template<common::NumBasesPerByte numBasesPerByte>
+class FastqCreator : public Stage
+{
+public:
+
+ /// \brief Constructor.
+ /// \param threadsCount Number of threads.
+ /// \param layout Flowcell layout.
+ /// \param laneInfo Lane meata data.
+ /// \param maskShortAdapterReads Maximum number of useful bases in read after adapter trimming for which whole read is masked.
+ /// \param adapterStringency Adapter stringency.
+ /// \param generateReverseComplementFastqs Generate reverse complement FASTQs.
+ /// \param includeNonPfClusters Include non-PF clusters in created FASTQ files flag.
+ /// \param createFastqsForIndexReads Create FASTQ files also for index reads flag.
+ /// \param useBgzf If true, use BGZF compression
+ /// \param compressionLevel Compression level used by zlib
+ /// \param findAdaptersWithSlidingWindow If true, find adapters with the sliding window algorithm.
+ /// \param tileStats Statistics collected for a tile
+ /// \param unknownBarcodeHits Counts of unknown barcodes
+ FastqCreator(
+ TaskQueue& taskQueue,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inputBuffersToUse,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inputBuffersToRecycle,
+ ThreadSafeMap<uint32_t, std::shared_ptr<FastqBuffer>>& outputBuffersToSubmit,
+ ThreadSafeQueue<std::shared_ptr<FastqBuffer>>& outputBuffersToUse,
+ const layout::Layout &layout,
+ const layout::LaneInfo &laneInfo,
+ std::size_t maskShortAdapterReads,
+ float adapterStringency,
+ bool generateReverseComplementFastqs,
+ bool includeNonPfClusters,
+ bool createFastqsForIndexReads,
+ bool useBgzf,
+ int compressionLevel,
+ bool findAdaptersWithSlidingWindow,
+ uint32_t numProcessingThreads,
+ ConversionStatsForSampleTileBarcode &summaryTileStats,
+ stats::BarcodeHits &unknownBarcodeHits
+ );
+
+ virtual ~FastqCreator();
+
+public:
+
+ virtual bool startNewTasks();
+
+private:
+
+ virtual void terminate();
+
+ virtual std::shared_ptr<FastqCreateTaskManager> getNewTaskManager();
+
+ void createTileStats();
+
+ void createAdapters(boost::ptr_vector<AdapterLocator<numBasesPerByte>>& adapters,
+ layout::ReadInfo::AdaptersContainer::const_iterator begin,
+ layout::ReadInfo::AdaptersContainer::const_iterator end) const;
+
+private:
+
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inputBuffersToUse_;
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inputBuffersToRecycle_;
+ ThreadSafeMap<uint32_t, std::shared_ptr<FastqBuffer>>& outputBuffersToSubmit_;
+ ThreadSafeQueue<std::shared_ptr<FastqBuffer>>& outputBuffersToUse_;
+
+ /// \brief Layout.
+ const layout::Layout &layout_;
+
+ /// \brief Current lane.
+ const layout::LaneInfo &laneInfo_;
+
+ /// \brief Maximum number of useful bases in read after adapter trimming for which whole read is masked.
+ std::size_t maskShortAdapterReads_;
+
+ /// \brief Adapter stringency.
+ float adapterStringency_;
+
+ /// \brief Mask adapters.
+ boost::ptr_vector< boost::ptr_vector<AdapterLocator<numBasesPerByte>> > maskAdapters_;
+
+ /// \brief Trim adapters.
+ boost::ptr_vector< boost::ptr_vector<AdapterLocator<numBasesPerByte>> > trimAdapters_;
+
+ /// \brief Generate reverse complement FASTQs flag.
+ bool generateReverseComplementFastqs_;
+
+ /// \brief Include non-PF clusters in created FASTQ files.
+ bool includeNonPfClusters_;
+
+ /// \brief Create FASTQ files also for index reads.
+ bool createFastqsForIndexReads_;
+
+ /// \brief Use BGZF compression if true.
+ bool useBgzf_;
+
+ /// \brief Compression level used by zlib
+ int compressionLevel_;
+
+ /// \brief Find adapters with the sliding window algorithm if true.
+ bool findAdaptersWithSlidingWindow_;
+
+ /// \brief Conversion statistics (summary).
+ ConversionStatsForSampleTileBarcode &summaryTileStats_;
+
+ /// \brief Unknown barcodes statistics (summary).
+ stats::BarcodeHits &unknownBarcodesHits_;
+
+ FastqCreateStats statsForEachThread_;
+
+ uint32_t prevGroupNumber_;
+
+ uint32_t numBuffersCreated_;
+};
+
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+#include "conversion/FastqCreator.hpp"
+
+#endif // BCL2FASTQ_CONVERSION_FASTQCREATOR_HH
+
+
diff --git a/src/cxx/include/conversion/FastqCreator.hpp b/src/cxx/include/conversion/FastqCreator.hpp
new file mode 100644
index 0000000..18b241b
--- /dev/null
+++ b/src/cxx/include/conversion/FastqCreator.hpp
@@ -0,0 +1,808 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FastqCreator.hpp
+ *
+ * \brief Declaration of FastqCreator.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_FASTQ_CREATOR_HPP
+#define BCL2FASTQ_CONVERSION_FASTQ_CREATOR_HPP
+
+
+#include "conversion/FastqCreator.hh"
+
+#include "common/FastIo.hh"
+#include "io/GzipCompressor.hh"
+#include "conversion/BclBaseConversion.hh"
+
+#include <boost/foreach.hpp>
+
+namespace bcl2fastq {
+namespace conversion {
+
+template<common::NumBasesPerByte numBasesPerByte>
+FastqCreateTask<numBasesPerByte>::FastqCreateTask(
+ std::shared_ptr<FastqCreateTaskManager>& taskManager,
+ const data::BclBufferVec &bclBuffers,
+ size_t cycleIndex,
+ size_t cycleIndexEnd,
+ const layout::FlowcellInfo &flowcellInfo,
+ const layout::LaneInfo &laneInfo,
+ const layout::ReadInfo& currentReadInfo,
+ SampleIndex::FastqOffsetsContainer::const_iterator offsetsBegin,
+ SampleIndex::FastqOffsetsContainer::const_iterator offsetsEnd,
+ std::size_t maskShortAdapterReads,
+ float adapterStringency,
+ const boost::ptr_vector<AdapterLocator<numBasesPerByte>>& maskAdapters,
+ const boost::ptr_vector<AdapterLocator<numBasesPerByte>>& trimAdapters,
+ bool generateReverseComplementFastqs,
+ bool includeNonPfClusters,
+ bool useBgzf,
+ int compressionLevel,
+ bool findAdaptersWithSlidingWindow,
+ FastqCreateStats& stats,
+ FastqBuffer::FastqsContainer::value_type::value_type::value_type &outputBuffer
+)
+: Task(taskManager)
+, bclBuffers_(bclBuffers)
+, cycleIndex_(cycleIndex + currentReadInfo.getBclBufferOffset())
+, cycleIndexEnd_(cycleIndex_ + currentReadInfo.cycleInfos().size())
+, flowcellInfo_(flowcellInfo)
+, laneInfo_(laneInfo)
+, readInfo_(currentReadInfo)
+, sampleMetadata_(bclBuffers_.at(offsetsBegin->first).samples_.at(offsetsBegin->second).first)
+, offsetsBegin_(offsetsBegin)
+, offsetsEnd_(offsetsEnd)
+, outputBuffer_(outputBuffer)
+, buffer_()
+, useBgzf_(useBgzf)
+, compressionLevel_(compressionLevel)
+, findAdaptersWithSlidingWindow_(findAdaptersWithSlidingWindow)
+, minimumTrimmedReadLength_(laneInfo_.getMinimumTrimmedReadLength())
+, maskShortAdapterReads_(maskShortAdapterReads)
+, adapterStringency_(adapterStringency)
+, maskAdapters_(maskAdapters)
+, trimAdapters_(trimAdapters)
+, generateReverseComplementFastqs_(generateReverseComplementFastqs)
+, includeNonPfClusters_(includeNonPfClusters)
+, stats_(stats)
+, umiCycles_()
+{
+ common::CycleNumber umiCycleOffset = 0;
+ for (const auto& readInfo : laneInfo_.readInfos())
+ {
+ // We do not load BCL files for cycles we don't use.
+ // The index into the BCL buffer needs to account for this.
+
+ const common::CycleRange& umiCycles = readInfo.getUmiCycles();
+ if (umiCycles.second != 0)
+ {
+ // These are the indexes into the bcl buffer
+ umiCycles_.push_back(common::CycleRange(umiCycles.first + umiCycleOffset, umiCycles.second + umiCycleOffset));
+ }
+
+ umiCycleOffset += readInfo.cyclesToLoad().size();
+ }
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+bool FastqCreateTask<numBasesPerByte>::execute(int32_t threadNum)
+{
+ io::GzipCompressor compressor(outputBuffer_,
+ useBgzf_,
+ boost::iostreams::gzip_params(compressionLevel_, // config param with default=4
+ boost::iostreams::zlib::deflated, // default
+ 15, // default
+ 9));
+
+ for (auto offset = offsetsBegin_; offset != offsetsEnd_; ++offset)
+ {
+ fastqCreate(*offset,
+ threadNum,
+ compressor);
+ }
+
+ return true;
+}
+
+
+namespace detail {
+
+
+static const char fastqHeaderStart = '@';
+static const char fastqHeaderSeparator = '\n';
+static const char fastqHeaderValuesSeparator = ':';
+static const char fastqHeaderPositionsSeparator = ' ';
+static const char fastqHeaderUmiSeparator = '+';
+static const std::string fastqBasesSeparator("\n+\n");
+static const char fastqQualitiesSeparator('\n');
+static const char fastqMaskedBase('N');
+static const char fastqMaskedQuality('#');
+
+
+} // namespace detail
+
+template<common::NumBasesPerByte numBasesPerByte>
+bool FastqCreateTask<numBasesPerByte>::computeStatistics( const FastqConstIterator<numBasesPerByte>& basesBegin,
+ const FastqConstIterator<numBasesPerByte>& basesEnd,
+ SampleIndex::FastqOffsetsContainer::value_type offset,
+ int32_t threadNum,
+ bool filterFlag,
+ size_t trimmedCount,
+ const std::vector<uint32_t>& remappedQscores )
+{
+ layout::LaneInfo::TileInfosContainer::size_type tileIndex = std::distance( laneInfo_.getTileInfos().begin(),
+ bclBuffers_[offset.first].tileInfo_ );
+ stats::ReadStats readStats;
+ readStats.yield = std::distance(basesBegin, basesEnd);
+ readStats.yieldQ30 = remappedQscores.empty() ?
+ std::count_if(basesBegin,
+ basesEnd,
+ [](unsigned char bcl)
+ { const unsigned char quality = bcl >> 2;
+ return quality >= 30; }) :
+ std::count_if(basesBegin,
+ basesEnd,
+ [&remappedQscores](unsigned char bcl)
+ { const unsigned char quality = remappedQscores[bcl>>2];
+ return quality >= 30; });
+
+ readStats.qualityScoreSum = remappedQscores.empty() ?
+ std::accumulate(basesBegin,
+ basesEnd,
+ common::ClustersCount(0),
+ [](common::ClustersCount total, unsigned char bcl)
+ { return total + (bcl>>2); }) :
+ std::accumulate(basesBegin,
+ basesEnd,
+ common::ClustersCount(0),
+ [&remappedQscores](common::ClustersCount total, unsigned char bcl)
+ { return total + remappedQscores[bcl>>2]; });
+
+ auto& statsForBarcode = stats_.at(threadNum)->conversionStats_.at(sampleMetadata_.sampleIndex_).at(tileIndex).at(sampleMetadata_.barcodeIndex_);
+ statsForBarcode.first[stats::TileBarcodeStats::RAW].RecordAdapterStats(trimmedCount, readInfo_.getNumber());
+ statsForBarcode.second.at(readInfo_.getNumber() - 1)[stats::TileBarcodeStats::RAW] += readStats;
+ if( filterFlag )
+ {
+ if( 1 == readInfo_.getNumber() ) { statsForBarcode.first[stats::TileBarcodeStats::PF].clusterCount += 1; }
+ statsForBarcode.first[stats::TileBarcodeStats::PF].RecordAdapterStats(trimmedCount, readInfo_.getNumber());
+ statsForBarcode.second.at(readInfo_.getNumber() - 1)[stats::TileBarcodeStats::PF] += readStats;
+ }
+
+ return (0 != sampleMetadata_.sampleIndex_);
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+void FastqCreateTask<numBasesPerByte>::fastqCreate(
+ SampleIndex::FastqOffsetsContainer::value_type offset,
+ int32_t threadNum,
+ io::GzipCompressor& compressor
+)
+{
+ const data::BclBuffer &bclBuffer = bclBuffers_.at(offset.first);
+ layout::LaneInfo::TileInfosContainer::size_type tileIndex = std::distance( laneInfo_.getTileInfos().begin(),
+ bclBuffer.tileInfo_ );
+ auto& statsForBarcode = stats_.at(threadNum)->conversionStats_.at(sampleMetadata_.sampleIndex_).at(tileIndex).at(sampleMetadata_.barcodeIndex_);
+ if( 1 == readInfo_.getNumber() ) { statsForBarcode.first[stats::TileBarcodeStats::RAW].clusterCount += 1; }
+
+ const bool filterFlag = bclBuffer.filters_[offset.second].data_;
+ const FastqConstIterator<numBasesPerByte> begin = FastqConstIterator<numBasesPerByte>(bclBuffer, offset.second) + cycleIndex_;
+ const FastqConstIterator<numBasesPerByte> end = FastqConstIterator<numBasesPerByte>(bclBuffer, offset.second) + cycleIndexEnd_;
+
+ size_t trimmedCount = 0;
+ if (includeNonPfClusters_ || filterFlag)
+ {
+ buffer_.clear();
+ createHeader(offset,
+ filterFlag);
+
+ createBasesAndQualities(offset,
+ compressor,
+ trimmedCount);
+ }
+
+ if (!filterFlag && !bclBuffer.bcls_.back().includeNonPf_)
+ {
+ return;
+ }
+
+ if (readInfo_.isDataRead())
+ {
+ if (!computeStatistics(begin, end, offset, threadNum, filterFlag, trimmedCount, bclBuffer.bcls_[0].remappedQscores_))
+ {
+ if( 1 == readInfo_.getNumber() && (includeNonPfClusters_ || filterFlag) ) { stats_.at(threadNum)->unknownBarcodes_.record( bclBuffer.samples_[offset.second].second ); }
+ }
+ }
+
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+void FastqCreateTask<numBasesPerByte>::writeHeaderElement(const std::string& element)
+{
+ size_t bufferSize = buffer_.size();
+ buffer_.resize(bufferSize + element.size() + 1);
+ std::copy(element.begin(), element.end(), buffer_.begin() + bufferSize);
+ buffer_.back() = detail::fastqHeaderValuesSeparator;
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+void FastqCreateTask<numBasesPerByte>::createHeader(SampleIndex::FastqOffsetsContainer::value_type offset,
+ bool filterFlag)
+{
+ const auto& bclBuffer = bclBuffers_.at(offset.first);
+ const common::ControlFlag controlFlag = bclBuffer.controls_.at(offset.second).data_;
+
+ // HEADER
+ buffer_.push_back(detail::fastqHeaderStart);
+ // instrument
+ writeHeaderElement(flowcellInfo_.getInstrument());
+ // run number
+ writeHeaderElement(flowcellInfo_.getRunNumber());
+ // flow cell ID
+ writeHeaderElement(flowcellInfo_.getFlowcellId());
+
+ // lane number
+ BOOST_ASSERT(laneInfo_.getNumber() < 1000);
+
+ if (laneInfo_.getNumber() >= 100)
+ {
+ buffer_.push_back(char((laneInfo_.getNumber() / 100) + '0'));
+ int32_t remainder = laneInfo_.getNumber() % 100;
+ buffer_.push_back(char((remainder / 10) + '0'));
+ buffer_.push_back(char((remainder % 10) + '0'));
+ }
+ else if (laneInfo_.getNumber() >= 10)
+ {
+ buffer_.push_back(char((laneInfo_.getNumber() / 10) + '0'));
+ buffer_.push_back(char((laneInfo_.getNumber() % 10) + '0'));
+ }
+ else
+ {
+ buffer_.push_back(char(laneInfo_.getNumber() + '0'));
+ }
+ buffer_.push_back(detail::fastqHeaderValuesSeparator);
+ // tile number
+ common::putUnsignedInteger(bclBuffer.tileInfo_->getNumber(), buffer_);
+ buffer_.push_back(detail::fastqHeaderValuesSeparator);
+ // x position
+ common::putUnsignedInteger( bclBuffer.positions_->size() > offset.second
+ ? (*bclBuffer.positions_)[offset.second].x_
+ : 0, buffer_ );
+ buffer_.push_back(detail::fastqHeaderValuesSeparator);
+ // y position
+ common::putUnsignedInteger( bclBuffer.positions_->size() > offset.second
+ ? (*bclBuffer.positions_)[offset.second].y_
+ : 0, buffer_ );
+ // UMI
+ if (!umiCycles_.empty())
+ {
+ buffer_.push_back(detail::fastqHeaderValuesSeparator);
+ size_t i = 0;
+ for (const auto& cycle : umiCycles_)
+ {
+ if (i > 0)
+ {
+ buffer_.push_back(detail::fastqHeaderUmiSeparator);
+ }
+ std::transform(
+ FastqConstIterator<numBasesPerByte>(bclBuffer, offset.second) + cycle.first,
+ FastqConstIterator<numBasesPerByte>(bclBuffer, offset.second) + cycle.second,
+ std::back_inserter(buffer_),
+ bclBuffer.bcls_[0].numBitsPerQscore_ == 0 ? convertBcl2FastqBaseNoQscore : &convertBcl2FastqBase
+ );
+
+ ++i;
+ }
+ }
+ buffer_.push_back(detail::fastqHeaderPositionsSeparator);
+
+ // read number
+ BOOST_ASSERT(readInfo_.getNumber() <= 9);
+ buffer_.push_back(char(readInfo_.getNumber() + '0'));
+ buffer_.push_back(detail::fastqHeaderValuesSeparator);
+ // is-filtered
+ buffer_.push_back(filterFlag ? 'N' : 'Y');
+ buffer_.push_back(detail::fastqHeaderValuesSeparator);
+ // control number
+ common::putUnsignedInteger(controlFlag, buffer_);
+ buffer_.push_back(detail::fastqHeaderValuesSeparator);
+
+ // barcode
+ if (bclBuffer.samples_[offset.second].second.componentsBegin() == bclBuffer.samples_[offset.second].second.componentsEnd())
+ {
+ common::putUnsignedInteger((laneInfo_.sampleInfosBegin()+sampleMetadata_.sampleIndex_)->getNumber(), buffer_);
+ }
+ else
+ {
+ bclBuffer.samples_[offset.second].second.output(buffer_);
+ }
+ buffer_.push_back(detail::fastqHeaderSeparator);
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+void FastqCreateTask<numBasesPerByte>::createBasesAndQualities(SampleIndex::FastqOffsetsContainer::value_type offset,
+ io::GzipCompressor& compressor,
+ size_t& trimmedCount)
+{
+ const auto& bclBuffer = bclBuffers_.at(offset.first);
+ const FastqConstIterator<numBasesPerByte> begin = FastqConstIterator<numBasesPerByte>(bclBuffer, offset.second) + cycleIndex_;
+
+ // The last cycle we want to put in the FASTQ file
+ const FastqConstIterator<numBasesPerByte> end = FastqConstIterator<numBasesPerByte>(bclBuffer, offset.second) + cycleIndexEnd_;
+
+ // BASES
+ FastqConstIterator<numBasesPerByte> maskBegin(end);
+ for (const AdapterLocator<numBasesPerByte>& adapterLocator : maskAdapters_)
+ {
+ const FastqConstIterator<numBasesPerByte> adapterPos = adapterLocator.identifyAdapter(begin, end);
+ maskBegin = std::min(maskBegin, adapterPos);
+ }
+
+ FastqConstIterator<numBasesPerByte> trimBegin(end);
+ for (const AdapterLocator<numBasesPerByte>& adapterLocator : trimAdapters_)
+ {
+ const FastqConstIterator<numBasesPerByte> adapterPos = adapterLocator.identifyAdapter(begin, end);
+ trimBegin = std::min(trimBegin, adapterPos);
+ }
+
+ trimmedCount = end - std::min(trimBegin, maskBegin);
+ if (readInfo_.isDataRead())
+ {
+ if ((trimBegin - begin) < typename FastqConstIterator<numBasesPerByte>::difference_type(minimumTrimmedReadLength_))
+ {
+ maskBegin = std::min(trimBegin, maskBegin);
+ trimBegin = begin + minimumTrimmedReadLength_;
+ }
+
+ if ((maskBegin - begin) < typename FastqConstIterator<numBasesPerByte>::difference_type(maskShortAdapterReads_))
+ {
+ maskBegin = begin;
+ }
+ }
+
+ const typename FastqConstIterator<numBasesPerByte>::difference_type maskedCount = trimBegin - maskBegin;
+ FastqConstIterator<numBasesPerByte> basesEnd(std::min(maskBegin, trimBegin));
+ size_t bufferSize = buffer_.size();
+
+ size_t numBases = std::distance(begin, basesEnd);
+ size_t numBasesPlusMask = numBases;
+ if (maskedCount > 0)
+ {
+ numBasesPlusMask += maskedCount;
+ }
+
+ // Resize for bases + BasesSeparator + quality + QualitySeparator
+ buffer_.resize(bufferSize + 2*numBasesPlusMask + 1 + detail::fastqBasesSeparator.size());
+ std::vector<char>::iterator iter(buffer_.begin());
+ std::advance(iter, bufferSize);
+ if (generateReverseComplementFastqs_)
+ {
+ if (maskedCount > 0)
+ {
+ std::fill_n(iter, maskedCount, detail::fastqMaskedBase);
+ std::advance(iter, maskedCount);
+ }
+
+ std::transform(
+ FastqConstReverseIterator<numBasesPerByte>(basesEnd),
+ FastqConstReverseIterator<numBasesPerByte>(begin),
+ iter,
+ bclBuffer.bcls_[0].numBitsPerQscore_ == 0 ? &convertBcl2FastqBaseComplementNoQscore : &convertBcl2FastqBaseComplement
+ );
+
+ std::advance(iter, numBases);
+ }
+ else
+ {
+ std::transform(
+ begin,
+ basesEnd,
+ iter,
+ bclBuffer.bcls_[0].numBitsPerQscore_ == 0 ? &convertBcl2FastqBaseNoQscore : &convertBcl2FastqBase
+ );
+
+ std::advance(iter, numBases);
+ if (maskedCount > 0)
+ {
+ std::fill_n(iter, maskedCount, detail::fastqMaskedBase);
+ std::advance(iter, maskedCount);
+ }
+ }
+
+ std::copy(detail::fastqBasesSeparator.begin(), detail::fastqBasesSeparator.end(), iter);
+ std::advance(iter, detail::fastqBasesSeparator.size());
+
+ // QUALITIES
+ if (generateReverseComplementFastqs_)
+ {
+ if (maskedCount > 0)
+ {
+ std::fill_n(iter, maskedCount, detail::fastqMaskedQuality);
+ std::advance(iter, maskedCount);
+ }
+
+ if ( bclBuffers_[offset.first].bcls_[0].remappedQscores_.empty())
+ {
+ std::transform(FastqConstReverseIterator<numBasesPerByte>(basesEnd),
+ FastqConstReverseIterator<numBasesPerByte>(begin),
+ iter,
+ &convertBcl2FastqQuality);
+ }
+ else
+ {
+ std::transform(FastqConstReverseIterator<numBasesPerByte>(basesEnd),
+ FastqConstReverseIterator<numBasesPerByte>(begin),
+ iter,
+ [this, &offset] (unsigned char bcl) { return convertBcl2FastqQuality2(bcl, bclBuffers_[offset.first].bcls_[0].remappedQscores_); });
+ }
+
+ std::advance(iter, numBases);
+ }
+ else
+ {
+ if (bclBuffers_[offset.first].bcls_[0].remappedQscores_.empty())
+ {
+ std::transform(begin, basesEnd, iter, &convertBcl2FastqQuality);
+ }
+ else
+ {
+ std::transform(begin,
+ basesEnd,
+ iter,
+ [this, &offset] (unsigned char bcl) { return convertBcl2FastqQuality2(bcl, bclBuffers_[offset.first].bcls_[0].remappedQscores_); });
+ }
+
+ std::advance(iter, numBases);
+ if (maskedCount > 0)
+ {
+ std::fill_n(iter, maskedCount, detail::fastqMaskedQuality);
+ std::advance(iter, maskedCount);
+ }
+ }
+ *iter = detail::fastqQualitiesSeparator;
+
+ std::streamsize bytesWritten = compressor.write(&*buffer_.begin(), buffer_.size());
+ BCL2FASTQ_ASSERT_MSG( static_cast<FastqBuffer::FastqsContainer::value_type::value_type::value_type::size_type>(bytesWritten) == buffer_.size(),
+ "Only " << bytesWritten << " of " << buffer_.size() << " bytes have been written" );
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+FastqCreator<numBasesPerByte>::FastqCreator(
+ TaskQueue& taskQueue,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inputBuffersToUse,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inputBuffersToRecycle,
+ ThreadSafeMap<uint32_t, std::shared_ptr<FastqBuffer>>& outputBuffersToSubmit,
+ ThreadSafeQueue<std::shared_ptr<FastqBuffer>>& outputBuffersToUse,
+ const layout::Layout &layout,
+ const layout::LaneInfo &laneInfo,
+ std::size_t maskShortAdapterReads,
+ float adapterStringency,
+ bool generateReverseComplementFastqs,
+ bool includeNonPfClusters,
+ bool createFastqsForIndexReads,
+ bool useBgzf,
+ int compressionLevel,
+ bool findAdaptersWithSlidingWindow,
+ uint32_t numProcessingThreads,
+ ConversionStatsForSampleTileBarcode &summaryTileStats,
+ stats::BarcodeHits &unknownBarcodeHits
+)
+: Stage("Fastq creating", taskQueue)
+, inputBuffersToUse_(inputBuffersToUse)
+, inputBuffersToRecycle_(inputBuffersToRecycle)
+, outputBuffersToSubmit_(outputBuffersToSubmit)
+, outputBuffersToUse_(outputBuffersToUse)
+, layout_(layout)
+, laneInfo_(laneInfo)
+, maskShortAdapterReads_(maskShortAdapterReads)
+, adapterStringency_(adapterStringency)
+, maskAdapters_()
+, trimAdapters_()
+, generateReverseComplementFastqs_(generateReverseComplementFastqs)
+, includeNonPfClusters_(includeNonPfClusters)
+, createFastqsForIndexReads_(createFastqsForIndexReads)
+, useBgzf_(useBgzf)
+, compressionLevel_(compressionLevel)
+, findAdaptersWithSlidingWindow_(findAdaptersWithSlidingWindow)
+, summaryTileStats_(summaryTileStats)
+, unknownBarcodesHits_(unknownBarcodeHits)
+, statsForEachThread_(numProcessingThreads)
+, prevGroupNumber_(0)
+, numBuffersCreated_(0)
+{
+ createTileStats();
+
+ for (const auto& readInfo : laneInfo_.readInfos())
+ {
+ if (readInfo.shouldCreateFastqForRead(createFastqsForIndexReads_))
+ {
+ maskAdapters_.push_back(new boost::ptr_vector<AdapterLocator<numBasesPerByte>>());
+ trimAdapters_.push_back(new boost::ptr_vector<AdapterLocator<numBasesPerByte>>());
+
+ createAdapters(maskAdapters_.back(),
+ readInfo.maskAdaptersBegin(),
+ readInfo.maskAdaptersEnd());
+
+ createAdapters(trimAdapters_.back(),
+ readInfo.trimAdaptersBegin(),
+ readInfo.trimAdaptersEnd());
+ }
+ }
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+FastqCreator<numBasesPerByte>::~FastqCreator()
+{
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+void FastqCreator<numBasesPerByte>::createTileStats()
+{
+ size_t nonIndexReadLength = 0;
+ size_t numNonIndexReads = 0;
+ for (const auto& readInfo : laneInfo_.readInfos())
+ {
+ if (readInfo.isDataRead())
+ {
+ nonIndexReadLength = std::max(nonIndexReadLength, readInfo.cycleInfos().size());
+ numNonIndexReads = std::max(numNonIndexReads, readInfo.getNumber());
+ }
+ }
+
+ layout::LaneInfo::TileInfosContainer::size_type tilesCount = laneInfo_.getTileInfos().size();
+
+ summaryTileStats_.resize(laneInfo_.getSampleInfos().size(),
+ ConversionStatsForTileBarcode(tilesCount));
+
+ const auto& sampleInfos = laneInfo_.getSampleInfos();
+ size_t sampleIndex = 0;
+ for (auto& summaryStatsForSample : summaryTileStats_)
+ {
+ size_t tileIndex = 0;
+ for (auto& summaryStatsForTile : summaryStatsForSample)
+ {
+ size_t barcodeIndex = 0;
+ summaryStatsForTile.resize(std::max(static_cast<size_t>(1UL), sampleInfos.at(sampleIndex).getBarcodes().size()));
+ for (auto& summaryStatsForBarcode : summaryStatsForTile)
+ {
+ summaryStatsForBarcode.first[stats::TileBarcodeStats::RAW].init(nonIndexReadLength, numNonIndexReads);
+ summaryStatsForBarcode.first[stats::TileBarcodeStats::PF].init(nonIndexReadLength, numNonIndexReads);
+
+ layout::BarcodeTranslationTable::SampleMetadata sampleMetadata(sampleIndex,barcodeIndex);
+ for (const auto& readInfo : laneInfo_.readInfos())
+ {
+ if (readInfo.isDataRead())
+ {
+ summaryStatsForBarcode.second.push_back(stats::ReadBarcodeStats(readInfo.getNumber()));
+ }
+ }
+
+ ++barcodeIndex;
+ }
+
+ ++tileIndex;
+ }
+
+ ++sampleIndex;
+ }
+
+ for (auto& statsForThread : statsForEachThread_)
+ {
+ statsForThread = std::make_shared<AllConversionStats>(
+ summaryTileStats_,
+ unknownBarcodesHits_);
+ }
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+bool FastqCreator<numBasesPerByte>::startNewTasks()
+{
+ static int sequentialId = 0; ++sequentialId;
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "FastqCreator::startNewTask " << sequentialId << std::endl;
+
+ TaskQueue &taskQueue = this->getTaskQueue();
+ if (taskQueue.isTerminated() || inputBuffersToUse_.isTerminated() )
+ {
+ // Only occurs if there was an error
+ outputBuffersToSubmit_.terminate();
+ inputBuffersToUse_.terminate();
+
+ return false;
+ }
+
+ auto taskManager = getNewTaskManager();
+ if (taskManager.get() == NULL)
+ {
+ return false;
+ }
+
+ FastqBuffer &outputBuffer = taskManager->getOutputBuffer();
+ const auto& inputBuffer = taskManager->getInputBuffer().bclBuffers_;
+ const auto& sampleIndex = *taskManager->getInputBuffer().sampleIndex_;
+
+ layout::LaneInfo::SampleInfosContainer::size_type sampleInfosCounts = laneInfo_.getSampleInfos().size();
+
+ // Check if tile file is missing.
+ DemuxBuffer & demuxBuffer = taskManager->getInputBuffer();
+ if (!demuxBuffer.hasCycleData())
+ {
+ // 'clear" the outputBuffer;
+ for( layout::LaneInfo::SampleInfosContainer::size_type sampleInfoIndex = 0;
+ sampleInfoIndex < sampleInfosCounts;
+ ++sampleInfoIndex )
+ {
+ const layout::ReadInfosContainer::size_type readInfoIndex = 0;
+ for (const auto& readInfo : laneInfo_.readInfos())
+ {
+ outputBuffer.fastqs_.at(sampleInfoIndex).at(readInfoIndex).resize(0);
+ }
+ }
+ return true;
+ }
+
+ BCL2FASTQ_ASSERT_MSG(inputBuffer.front().bcls_.front().bcls_.size() > 0, "No BCL cycles to process");
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Clusters/BCL: " << inputBuffer.front().bcls_.front().getNumClustersPerByte() * inputBuffer.front().bcls_.front().bcls_.size() << std::endl;
+
+ uint32_t taskNum = 0;
+ for( layout::LaneInfo::SampleInfosContainer::size_type sampleInfoIndex = 0;
+ sampleInfoIndex < sampleInfosCounts;
+ ++sampleInfoIndex )
+ {
+ size_t cycleIndex = 0;
+
+ layout::ReadInfosContainer::size_type readInfoIndex = 0;
+ for (const auto& readInfo : laneInfo_.readInfos())
+ {
+ // For index reads, use unmasked cycles.
+ size_t nextCycleIndex = cycleIndex + readInfo.getNumCyclesToLoad();
+
+ // cycleIndex += readInfo.getBclBufferOffset();
+ if (readInfo.shouldCreateFastqForRead(createFastqsForIndexReads_))
+ {
+ SampleIndex::FastqOffsetsContainer::size_type offsetsIndex = 0;
+ const SampleIndex::FastqOffsetsContainer::const_iterator offsetsEnd = sampleIndex.offsetsEnd(sampleInfoIndex);
+ SampleIndex::FastqOffsetsContainer::const_iterator offset = sampleIndex.offsetsBegin(sampleInfoIndex);
+
+ outputBuffer.fastqs_.at(sampleInfoIndex).at(readInfoIndex).resize((offsetsEnd-offset + data::ClustersPerTask - 1) / data::ClustersPerTask);
+
+ while(offset != offsetsEnd)
+ {
+ BCL2FASTQ_ASSERT_MSG(offsetsEnd - offset > 0, "Invalid offset iterator");
+ const SampleIndex::FastqOffsetsContainer::const_iterator nextOffset =
+ offset
+ +
+ (static_cast<std::size_t>(offsetsEnd - offset) < data::ClustersPerTask ? offsetsEnd - offset : data::ClustersPerTask);
+
+ taskQueue.addData(std::make_shared<FastqCreateTask<numBasesPerByte>>(
+ taskManager,
+ inputBuffer,
+ cycleIndex,
+ nextCycleIndex,
+ layout_.getFlowcellInfo(),
+ laneInfo_,
+ readInfo,
+ offset,
+ nextOffset,
+ maskShortAdapterReads_,
+ adapterStringency_,
+ maskAdapters_[readInfoIndex],
+ trimAdapters_[readInfoIndex],
+ generateReverseComplementFastqs_,
+ includeNonPfClusters_,
+ useBgzf_,
+ compressionLevel_,
+ findAdaptersWithSlidingWindow_,
+ statsForEachThread_,
+ outputBuffer.fastqs_.at(sampleInfoIndex).at(readInfoIndex).at(offsetsIndex)
+ ));
+
+ ++taskNum;
+ ++offsetsIndex;
+ offset = nextOffset;
+ }
+ ++readInfoIndex;
+ }
+
+ cycleIndex = nextCycleIndex;
+ }
+ }
+
+ return true;
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+void FastqCreator<numBasesPerByte>::terminate()
+{
+ this->getTaskQueue().terminate();
+ inputBuffersToUse_.terminate();
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+std::shared_ptr<FastqCreateTaskManager> FastqCreator<numBasesPerByte>::getNewTaskManager()
+{
+ std::shared_ptr<DemuxBuffer> inputBuffer;
+ if (!inputBuffersToUse_.tryGetData(inputBuffer))
+ {
+ FastqCreateTaskManager::waitForAllTasksToFinish();
+
+ // We're done. Notify the next stage (fastq writing).
+ outputBuffersToSubmit_.setFinished(prevGroupNumber_);
+ return std::shared_ptr<FastqCreateTaskManager>();
+ }
+
+ prevGroupNumber_ = inputBuffer->getGroupNumber();
+ std::shared_ptr<FastqBuffer> outputBuffer;
+
+ if (numBuffersCreated_ < 3)
+ {
+ if (!outputBuffersToUse_.tryGetDataCheckEmpty(outputBuffer))
+ {
+ // There aren't any buffers available. Make a new one.
+ // The memory allocation isn't too bad here, since we're recycling the buffers.
+
+ outputBuffer = std::make_shared<FastqBuffer>();
+ ++numBuffersCreated_;
+
+ layout::LaneInfo::SampleInfosContainer::size_type sampleInfosCounts = laneInfo_.getSampleInfos().size();
+
+ outputBuffer->fastqs_.resize(sampleInfosCounts);
+
+ layout::ReadInfosContainer::size_type readsCount = std::count_if(laneInfo_.readInfos().begin(),
+ laneInfo_.readInfos().end(),
+ [this] (const layout::ReadInfo& readInfo)
+ { return !readInfo.cycleInfos().empty() &&
+ (createFastqsForIndexReads_ || readInfo.isDataRead()); });
+
+ for( layout::LaneInfo::SampleInfosContainer::size_type sampleInfoIndex = 0;
+ sampleInfoIndex < sampleInfosCounts;
+ ++sampleInfoIndex )
+ {
+ outputBuffer->fastqs_[sampleInfoIndex].resize(readsCount);
+ }
+ }
+ }
+ else
+ {
+ if (!outputBuffersToUse_.tryGetData(outputBuffer))
+ {
+ return std::shared_ptr<FastqCreateTaskManager>();
+ }
+ }
+
+ outputBuffer->setGroupNumber(inputBuffer->getGroupNumber());
+
+ return std::make_shared<FastqCreateTaskManager>(inputBuffer,
+ inputBuffersToRecycle_,
+ outputBuffer,
+ outputBuffersToSubmit_);
+}
+
+template<common::NumBasesPerByte numBasesPerByte>
+void FastqCreator<numBasesPerByte>::createAdapters(boost::ptr_vector<AdapterLocator<numBasesPerByte>>& adapters,
+ layout::ReadInfo::AdaptersContainer::const_iterator begin,
+ layout::ReadInfo::AdaptersContainer::const_iterator end) const
+{
+ size_t minimumTrimmedReadLength = laneInfo_.getMinimumTrimmedReadLength();
+ BOOST_FOREACH (const layout::ReadInfo::AdaptersContainer::value_type& adapter, std::make_pair(begin, end))
+ {
+ findAdaptersWithSlidingWindow_ ?
+ adapters.push_back(new AdapterLocatorSlidingWindow<numBasesPerByte>(adapter, adapterStringency_)) :
+ adapters.push_back(new AdapterLocatorWithIndels<numBasesPerByte>(adapter, adapterStringency_, minimumTrimmedReadLength));
+ }
+}
+
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_CONVERSION_FASTQ_CREATOR_HPP
+
diff --git a/src/cxx/include/conversion/FastqIterator.hh b/src/cxx/include/conversion/FastqIterator.hh
new file mode 100644
index 0000000..c68a5ce
--- /dev/null
+++ b/src/cxx/include/conversion/FastqIterator.hh
@@ -0,0 +1,197 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FastqIterator.hh
+ *
+ * \brief Declaration of FASTQ iterator.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_FASTQITERATOR_HH
+#define BCL2FASTQ_CONVERSION_FASTQITERATOR_HH
+
+
+#include <vector>
+#include <iterator>
+
+#include <boost/iterator/iterator_facade.hpp>
+#include <boost/iterator/reverse_iterator.hpp>
+#include <boost/mpl/if.hpp>
+
+#include "data/BclBuffer.hh"
+#include "data/CbclFile.hh"
+
+namespace bcl2fastq {
+namespace conversion {
+
+// VERY IMPORTANT: This class is templated on common::NumBasesPerByte because
+// the dereference method must be very fast, since it is called on every base.
+// Even a simple switch statement in dereference add 30% to the entire run time.
+
+// NOTE: This class is fundamentally flawed in that it does not allow for the
+// use of auto-vecotrization. This is a core piece of the code, and so it should
+// be rewritten in such a way that the compiler can vectorize the instructions.
+
+/// \brief FASTQ iterator.
+/// \note As Boost::Iterator documentation notes, the superior way to implement
+/// this should be by using @c boost::iterator_adaptor. However, that class
+/// is flawed, because it allows interoperability of completely unrelated
+/// iterators (@c boost::is_convertible is @c true for @c T and @c const @c T,
+/// but also for e.g. @c char and @c int).
+template<bool IsConstant, common::NumBasesPerByte numBasesPerByte>
+class FastqIteratorTemplate : public boost::iterator_facade<
+ FastqIteratorTemplate<IsConstant, numBasesPerByte>, // Derived
+ typename boost::mpl::if_c< // Value
+ IsConstant,
+ const char,
+ char
+ >::type,
+ boost::random_access_traversal_tag, // CategoryOrTraversal
+ typename boost::mpl::if_c< // Reference
+ IsConstant,
+ const char,
+ char
+ >::type
+>
+{
+public:
+
+ /// \brief Constructor.
+ /// \param iterator BCL chunk iterator.
+ /// \param offset Offset from beginning of the BCL chunk.
+ FastqIteratorTemplate(
+ const data::BclBuffer& bclBuffer,
+ data::PerCycleData::BclsContainer::size_type offset
+ )
+ : iterator_(bclBuffer.bcls_.begin())
+ , offset_(offset / static_cast<uint32_t>(numBasesPerByte))
+ , offsetSubByte_(offset % static_cast<uint32_t>(numBasesPerByte))
+ , offsetCbclFiltered_(bclBuffer.cbclFilterOffsets_.empty() ? 0 : bclBuffer.cbclFilterOffsets_.at(offset) / static_cast<uint32_t>(numBasesPerByte))
+ , offsetCbclFilteredSubByte_(bclBuffer.cbclFilterOffsets_.empty() ? 0 : bclBuffer.cbclFilterOffsets_.at(offset) % static_cast<uint32_t>(numBasesPerByte))
+ {
+ }
+
+private:
+
+ friend class boost::iterator_core_access;
+
+ /// \brief Dereference core operation.
+ /// \return Reference to element being pointed to.
+ /// \pre Iterator is dereferencable.
+ typename FastqIteratorTemplate<IsConstant, numBasesPerByte>::reference dereference() const;
+/* {
+ auto cycleByte = iterator_->includeNonPf_ ? iterator_->bcls_[offset_] : iterator_->bcls_[offsetCbclFiltered_];
+ auto subByteIndex = iterator_->includeNonPf_ ? offsetSubByte_ : offsetCbclFilteredSubByte_;
+ switch (numBasesPerByte)
+ {
+ case common::NumBasesPerByte::ONE:
+ return cycleByte;
+ break;
+ case common::NumBasesPerByte::TWO:
+ return (cycleByte >> 4*subByteIndex) & 0x0f;
+ break;
+ case common::NumBasesPerByte::FOUR:
+ return (cycleByte >> 2*subByteIndex) & 0x03;
+ break;
+ }
+
+ BCL2FASTQ_ASSERT_MSG(false, "Invalid number of bases per byte");
+
+ return 0;
+ }
+*/
+ /// \brief Equality comparison core operation.
+ /// \param rhs Right-hand-side parameter.
+ /// \retval true Left-hand-side and right-hand-side parameters are equal.
+ /// \retval false Left-hand-side and right-hand-side parameters are not equal.
+ template<bool IsRhsConstant>
+ bool equal(const FastqIteratorTemplate<IsRhsConstant, numBasesPerByte> &rhs) const
+ {
+ return (iterator_ == rhs.iterator_) &&
+ (offset_ == rhs.offset_);
+ }
+
+ /// \brief Increment core operation.
+ /// \pre Iterator is dereferencable.
+ /// \post Iterator points to the next element.
+ void increment()
+ {
+ ++iterator_;
+ }
+
+ /// \brief Decrement core operation.
+ /// \pre Iterator is dereferencable.
+ /// \post Iterator points to the next element.
+ void decrement()
+ {
+ --iterator_;
+ }
+
+ /// \brief Advance core operation.
+ /// \param diff Distance by which the iterator is to be advanced.
+ /// \pre Iterator is dereferencable.
+ /// \post Iterator has been advanced by specified distance.
+ void advance(typename FastqIteratorTemplate<IsConstant, numBasesPerByte>::difference_type diff)
+ {
+ iterator_ += diff;
+ }
+
+ /// \brief Distance core operation.
+ /// \param rhs Right-hand-side parameter.
+ /// \return Distance between this iterator and right-hand-side iterator.
+ template<bool IsRhsConstant>
+ typename FastqIteratorTemplate<IsConstant, numBasesPerByte>::difference_type distance_to(const FastqIteratorTemplate<IsRhsConstant, numBasesPerByte> &rhs) const
+ {
+ return rhs.iterator_ - iterator_;
+ }
+
+private:
+
+ /// \brief BCL chunk iterator.
+ typename boost::mpl::if_c<IsConstant, data::BclBuffer::BclCycleContainer::const_iterator, data::BclBuffer::BclCycleContainer::iterator>::type iterator_;
+
+ /// \brief Offset from beginning of the BCL chunk.
+ data::PerCycleData::BclsContainer::size_type offset_;
+
+ data::PerCycleData::BclsContainer::size_type offsetSubByte_;
+
+ /// \brief Offset from beginning of the CBCL chunk (for cycles which don't include non-pf clusters).
+ data::PerCycleData::BclsContainer::size_type offsetCbclFiltered_;
+
+ data::PerCycleData::BclsContainer::size_type offsetCbclFilteredSubByte_;
+};
+
+/// \brief FASTQ iterator.
+//typedef FastqIteratorTemplate<false, NumBasesPerByte::ONE> FastqIterator;
+template<common::NumBasesPerByte numBasesPerByte>
+using FastqIterator = FastqIteratorTemplate<false, numBasesPerByte>;
+
+/// \brief FASTQ reverse iterator.
+//typedef boost::reverse_iterator<FastqIterator> FastqReverseIterator;
+template<common::NumBasesPerByte numBasesPerByte>
+using FastqReverseIterator = boost::reverse_iterator<FastqIterator<numBasesPerByte>>;
+
+/// \brief FASTQ constant iterator.
+//typedef FastqIteratorTemplate<true, NumBasesPerByte::ONE> FastqConstIterator;
+template<common::NumBasesPerByte numBasesPerByte>
+using FastqConstIterator = FastqIteratorTemplate<true, numBasesPerByte>;
+
+/// \brief FASTQ constant reverse iterator.
+//typedef boost::reverse_iterator<FastqConstIterator> FastqConstReverseIterator;
+template<common::NumBasesPerByte numBasesPerByte>
+using FastqConstReverseIterator = boost::reverse_iterator<FastqConstIterator<numBasesPerByte>>;
+
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+#include "conversion/FastqIterator.hpp"
+
+#endif // BCL2FASTQ_CONVERSION_FASTQITERATOR_HH
diff --git a/src/cxx/include/conversion/FastqIterator.hpp b/src/cxx/include/conversion/FastqIterator.hpp
new file mode 100644
index 0000000..71fa8d0
--- /dev/null
+++ b/src/cxx/include/conversion/FastqIterator.hpp
@@ -0,0 +1,75 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file TaskQueue.hpp
+ *
+ * \brief Declaration of task queue.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_FASTQITERATOR_HPP
+#define BCL2FASTQ_CONVERSION_FASTQITERATOR_HPP
+
+#include "conversion/FastqIterator.hh"
+
+namespace bcl2fastq {
+namespace conversion {
+
+// Silence a g++ warning where the template class function warns the const qualifier is not
+// // needed BUT it is for the class function. I think it looks at this as a non-class function.
+#pragma GCC diagnostic ignored "-Wignored-qualifiers"
+
+template<>
+inline typename FastqIteratorTemplate<true, common::NumBasesPerByte::ONE>::reference FastqIteratorTemplate<true, common::NumBasesPerByte::ONE>::dereference() const
+{
+ return iterator_->bcls_[offset_];
+}
+template<>
+inline typename FastqIteratorTemplate<false, common::NumBasesPerByte::ONE>::reference FastqIteratorTemplate<false, common::NumBasesPerByte::ONE>::dereference() const
+{
+ return iterator_->bcls_[offset_];
+}
+
+
+template<>
+inline typename FastqIteratorTemplate<true, common::NumBasesPerByte::TWO>::reference FastqIteratorTemplate<true, common::NumBasesPerByte::TWO>::dereference() const
+{
+ auto cycleByte = iterator_->includeNonPf_ ? iterator_->bcls_[offset_] : iterator_->bcls_[offsetCbclFiltered_];
+ auto subByteIndex = iterator_->includeNonPf_ ? offsetSubByte_ : offsetCbclFilteredSubByte_;
+ return (cycleByte >> 4*subByteIndex) & 0x0f;
+}
+template<>
+inline typename FastqIteratorTemplate<false, common::NumBasesPerByte::TWO>::reference FastqIteratorTemplate<false, common::NumBasesPerByte::TWO>::dereference() const
+{
+ auto cycleByte = iterator_->includeNonPf_ ? iterator_->bcls_[offset_] : iterator_->bcls_[offsetCbclFiltered_];
+ auto subByteIndex = iterator_->includeNonPf_ ? offsetSubByte_ : offsetCbclFilteredSubByte_;
+ return (cycleByte >> 4*subByteIndex) & 0x0f;
+}
+
+template<>
+inline typename FastqIteratorTemplate<true, common::NumBasesPerByte::FOUR>::reference FastqIteratorTemplate<true, common::NumBasesPerByte::FOUR>::dereference() const
+{
+ auto cycleByte = iterator_->includeNonPf_ ? iterator_->bcls_[offset_] : iterator_->bcls_[offsetCbclFiltered_];
+ auto subByteIndex = iterator_->includeNonPf_ ? offsetSubByte_ : offsetCbclFilteredSubByte_;
+ return (cycleByte >> 2*subByteIndex) & 0x03;
+}
+template<>
+inline typename FastqIteratorTemplate<false, common::NumBasesPerByte::FOUR>::reference FastqIteratorTemplate<false, common::NumBasesPerByte::FOUR>::dereference() const
+{
+ auto cycleByte = iterator_->includeNonPf_ ? iterator_->bcls_[offset_] : iterator_->bcls_[offsetCbclFiltered_];
+ auto subByteIndex = iterator_->includeNonPf_ ? offsetSubByte_ : offsetCbclFilteredSubByte_;
+ return (cycleByte >> 2*subByteIndex) & 0x03;
+}
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_CONVERSION_FASTQITERATOR_HPP
+
diff --git a/src/cxx/include/conversion/FastqWriter.hh b/src/cxx/include/conversion/FastqWriter.hh
new file mode 100644
index 0000000..7aabf00
--- /dev/null
+++ b/src/cxx/include/conversion/FastqWriter.hh
@@ -0,0 +1,189 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FastqWriter.hh
+ *
+ * \brief Declaration of FASTQ writer.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_FASTQWRITER_HH
+#define BCL2FASTQ_CONVERSION_FASTQWRITER_HH
+
+
+#include "layout/Layout.hh"
+#include "data/FastqFile.hh"
+#include "conversion/FastqBuffer.hh"
+#include "conversion/Stage.hh"
+#include "conversion/Task.hh"
+
+#include <condition_variable>
+#include <mutex>
+
+namespace bcl2fastq {
+
+
+namespace conversion {
+
+class FastqWriteTaskManager : public TaskManager
+{
+public:
+ FastqWriteTaskManager(std::shared_ptr<FastqBuffer>& inputBuffer,
+ ThreadSafeQueue<std::shared_ptr<FastqBuffer>>& inputBuffersToRecycle);
+
+ virtual ~FastqWriteTaskManager();
+
+ static void waitForAllTasksToFinish();
+
+ FastqBuffer& getInputBuffer() { return *inputBuffer_; }
+
+private:
+ std::shared_ptr<FastqBuffer> inputBuffer_;
+ ThreadSafeQueue<std::shared_ptr<FastqBuffer>>& inputBuffersToRecycle_;
+
+ static std::atomic<uint32_t> numTaskManagers_;
+ static std::condition_variable cvAllTaskManagersDone_;
+ static std::mutex mut_;
+};
+
+/// \brief Task: Write FASTQs from buffer to file.
+class FastqWriteTask : public Task
+{
+public:
+
+ /// \brief FASTQ file external state with metadata type definition.
+ struct FastqFileStateInfo
+ {
+ public:
+
+ /// \brief Default constructor.
+ FastqFileStateInfo();
+
+ /// \brief Constructor.
+ explicit FastqFileStateInfo(data::FastqFile::FastqFileStatePtr fastqFileStatePtr);
+
+ FastqFileStateInfo(FastqFileStateInfo&& tmp);
+
+ void waitForGroupNumber(uint32_t groupNumber);
+
+ void signalWriteComplete();
+
+ public:
+
+ /// \brief FASTQ file external state.
+ data::FastqFile::FastqFileStatePtr fastqFileStatePtr_;
+
+ uint32_t nextGroupNumber_;
+ std::condition_variable cvGroupNumber_;
+ std::mutex mtx_;
+ };
+
+public:
+
+ /// \brief Constructor.
+ /// \param fastqFiles FASTQ files (one per thread).
+ /// \param fastqFileStateInfo External state and metadata of FASTQ file to be written to.
+ FastqWriteTask(std::shared_ptr<FastqWriteTaskManager>& taskManager,
+ FastqFileStateInfo &fastqFileStateInfo,
+ const FastqBuffer::FastqsContainer::value_type::value_type &inputBuffer,
+ uint32_t groupNumber);
+
+public:
+
+ virtual bool execute(int32_t threadNum);
+
+ virtual PriorityLevel getPriority() const { return PriorityLevel::Zero; }
+
+private:
+
+ /// \brief File state meta data of the FASTQ file to be written to.
+ FastqFileStateInfo& fastqFileStateInfo_;
+
+ /// \brief Buffer to read data from.
+ const FastqBuffer::FastqsContainer::value_type::value_type& inputBuffer_;
+
+ uint32_t groupNumber_;
+};
+
+
+/// \brief FASTQ writer.
+class FastqWriter : public Stage // public SinkStage<FastqBuffer>
+{
+public:
+
+ /// \brief Constructor.
+ /// \param threadsCount Number of threads.
+ /// \param layout Flowcell layout.
+ /// \param laneInfo Lane meata data.
+ /// \param createFastqsForIndexReads Create FASTQ files also for index reads flag.
+ /// \param outputDir Output directory.
+ FastqWriter(
+ TaskQueue& taskQueue,
+ ThreadSafeMap<uint32_t, std::shared_ptr<FastqBuffer>>& inputBuffersToUse,
+ ThreadSafeQueue<std::shared_ptr<FastqBuffer>>& inputBuffersToRecycle,
+ const layout::Layout &layout,
+ const layout::LaneInfo &laneInfo,
+ bool createFastqsForIndexReads,
+ const boost::filesystem::path& outputDir,
+ bool noLaneSplitting
+ );
+
+public:
+
+ virtual std::shared_ptr<FastqWriteTaskManager> getNewTaskManager();
+
+ virtual bool startNewTasks();
+
+private:
+
+ virtual void terminate();
+
+ /// \brief FASTQ file external states container type definition.
+ typedef std::vector<std::vector<std::shared_ptr<FastqWriteTask::FastqFileStateInfo>>> FastqFileStatesContainer;
+
+private:
+// Class to delete an empty file on destruction
+class EmptyFileDeleter
+{
+public:
+ EmptyFileDeleter() : filePaths_() { }
+ ~EmptyFileDeleter();
+
+ void addFile(boost::filesystem::path& filePath) { filePaths_.push_back(filePath); }
+
+private:
+ std::vector<boost::filesystem::path> filePaths_;
+};
+
+ ThreadSafeMap<uint32_t, std::shared_ptr<FastqBuffer>>& inputBuffersToUse_;
+ ThreadSafeQueue<std::shared_ptr<FastqBuffer>>& inputBuffersToRecycle_;
+
+ /// \brief Layout.
+ const layout::Layout &layout_;
+
+ /// \brief Delete empty files on destruction
+ EmptyFileDeleter emptyFileDeleter_;
+
+ /// \brief FASTQ file states.
+ FastqFileStatesContainer fastqFileStates_;
+
+ uint32_t groupNumber_;
+};
+
+
+} // namespace conversion
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_CONVERSION_FASTQWRITER_HH
+
+
diff --git a/src/cxx/include/conversion/SampleIndex.hh b/src/cxx/include/conversion/SampleIndex.hh
new file mode 100644
index 0000000..f333be8
--- /dev/null
+++ b/src/cxx/include/conversion/SampleIndex.hh
@@ -0,0 +1,243 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file SampleIndex.hh
+ *
+ * \brief Declaration of sample index.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_SAMPLEINDEX_HH
+#define BCL2FASTQ_CONVERSION_SAMPLEINDEX_HH
+
+
+#include <boost/noncopyable.hpp>
+
+#include "layout/LaneInfo.hh"
+#include "data/BclBuffer.hh"
+
+
+namespace bcl2fastq {
+namespace conversion {
+
+
+/// \brief Index of FASTQ offsets and their assignments to samples.
+class SampleIndex : private boost::noncopyable
+{
+public:
+ /// \brief FASTQ offsets container type definition.
+ typedef std::vector<std::pair<data::BclBufferVec::size_type, common::RawDataBuffer::size_type>> FastqOffsetsContainer;
+
+ static const FastqOffsetsContainer::size_type defaultFastqStep_ = 1;
+
+public:
+
+ /// \brief Constructor.
+ /// \param samplesCount Total number of samples.
+ /// \param maxFastqsCount Maximum total number of indexed FASTQ sequences.
+ SampleIndex(
+ layout::LaneInfo::SampleInfosContainer::size_type samplesCount,
+ SampleIndex::FastqOffsetsContainer::size_type maxFastqsCount
+ );
+
+ /// \brief Reset to initial state.
+ /// \param bclBuffers buffers to index
+ void reset(const data::BclBufferVec& bclBuffers);
+
+private:
+
+ /// \brief Get the total number of clusters.
+ /// \param bclBuffers bcl buffers.
+ /// \return Iterator to end of offsets.
+ size_t getTotalClusters(const data::BclBufferVec& bclBuffers) const;
+
+ /// \brief Reset to initial state.
+ /// \param fastqsCount Actual total number of indexed FASTQ sequences.
+ /// \pre Value passed in @c fastqsCount parameter must not be larger than
+ /// value passed to constructor's @c maxFastqsCount paramteter.
+ void reset(SampleIndex::FastqOffsetsContainer::size_type fastqsCount);
+
+ /// \brief Increment count of FASTQ offsets for given sample.
+ /// \param sample Sample index.
+ /// \param count Number of FASTQ offsets to be counted in.
+ /// \pre Neither @c reserve nor @c finalize has been called since
+ /// last call to @c reset, which must have been called at least once.
+ void incrementFastqCount(
+ layout::LaneInfo::SampleInfosContainer::size_type sample,
+ SampleIndex::FastqOffsetsContainer::size_type count = defaultFastqStep_
+ );
+
+ /// \brief Create data structures for specified amounts of FASTQ offsets in individual samples.
+ /// \pre Neither @c reserve nor @c finalize has been called since
+ /// last call to @c reset, which must have been called at least once.
+ void reserve();
+
+ /// \brief Add FASTQ offset for given sample.
+ /// \param sample Sample index.
+ /// \param offset FASTQ offset to be added.
+ /// \pre Neither @c reset nor @c finalize has been called since
+ /// last call to @c reserve, which must have been called at least once.
+ void addOffset(
+ layout::LaneInfo::SampleInfosContainer::size_type sample,
+ const FastqOffsetsContainer::value_type& offset
+ );
+
+ /// \brief Finalize index.
+ /// \pre Neither @c finalize nor @c reset has been called since
+ /// last call to @c reserve, which must have been called at least once.
+ void finalize();
+
+public:
+
+ /// \brief Get beginning of offsets for given sample.
+ /// \param sample Sample index.
+ /// \return Iterator to beginning of offsets.
+ /// \pre Neither @c reset nor @c reserve has been called since
+ /// last call to @c finalize, which must have been called at least once.
+ SampleIndex::FastqOffsetsContainer::const_iterator offsetsBegin(layout::LaneInfo::SampleInfosContainer::size_type sample) const;
+
+ /// \brief Get end of offsets for given sample.
+ /// \param sample Sample index.
+ /// \return Iterator to end of offsets.
+ /// \pre Neither @c reset nor @c reserve has been called since
+ /// last call to @c finalize, which must have been called at least once.
+ SampleIndex::FastqOffsetsContainer::const_iterator offsetsEnd(layout::LaneInfo::SampleInfosContainer::size_type sample) const;
+
+ /// \brief Get beginning of offsets for given sample.
+ /// \param sample Sample index.
+ /// \return Iterator to beginning of offsets.
+ /// \pre Neither @c reset nor @c reserve has been called since
+ /// last call to @c finalize, which must have been called at least once.
+ SampleIndex::FastqOffsetsContainer::iterator offsetsBegin(layout::LaneInfo::SampleInfosContainer::size_type sample);
+
+ /// \brief Get end of offsets for given sample.
+ /// \param sample Sample index.
+ /// \return Iterator to end of offsets.
+ /// \pre Neither @c reset nor @c reserve has been called since
+ /// last call to @c finalize, which must have been called at least once.
+ SampleIndex::FastqOffsetsContainer::iterator offsetsEnd(layout::LaneInfo::SampleInfosContainer::size_type sample);
+
+private:
+
+ /// \brief Internal state.
+ struct State
+ {
+ enum value_type
+ {
+ INITIALIZING, ///< Object has not been fully initialized, yet.
+ CALCULATING_SIZES, ///< Write-only state: accumulating sample sizes.
+ BUILDING_INDEX, ///< Write-only state: acquiring declared numbers of FASTQ offsets.
+ INDEX_BUILT ///< Read-only state: index is ready for use.
+ };
+ };
+
+ /// \brief Sample meta-data.
+ struct SampleMetaData
+ {
+ /// \brief Beginning of offsets list.
+ FastqOffsetsContainer::iterator begin;
+
+ /// \brief End of offsets list.
+ FastqOffsetsContainer::iterator end;
+
+ /// \brief Next offset record to be filled in.
+ FastqOffsetsContainer::iterator next;
+ };
+
+private:
+
+ /// \brief Helper for reserve method.
+ /// \param offset Reference to offset variable, being used inside the algorithm.
+ /// \param sampleMetaData Sample meta data structure.
+ static void reserveHelper(
+ std::vector<SampleIndex::SampleMetaData>::size_type &offset,
+ SampleIndex::SampleMetaData &sampleMetaData
+ );
+
+ /// \brief Helper predicate for finalize method.
+ /// \param offsetsBegin Beginning of FASTQ offset array.
+ /// \param offsetsEnd End of FASTQ offset array.
+ /// \param sampleMetaData Sample meta data structure.
+ /// \retval true Sample meta data structure *IS* *NOT* invalid.
+ /// \retval false Sample meta data structure *IS* valid.
+ static bool finalizePredicate(
+ SampleIndex::FastqOffsetsContainer::const_iterator offsetsBegin,
+ SampleIndex::FastqOffsetsContainer::const_iterator offsetsEnd,
+ const SampleIndex::SampleMetaData &sampleMetaData
+ );
+
+private:
+public:
+ /// \brief Internal state.
+ State::value_type state_;
+
+ /// \brief FASTQ offset array.
+ FastqOffsetsContainer offsets_;
+
+ /// \brief Samples metadata.
+ std::vector<SampleMetaData> samplesMetaData_;
+
+ /// \brief Number of FASTQ sequences (actual number as opposed to maximum number).
+ FastqOffsetsContainer::size_type fastqsCount_;
+};
+
+struct DemuxBuffer
+{
+ DemuxBuffer()
+ : bclBuffers_(), sampleIndex_(), groupNumber_(0)
+ {
+ }
+
+ DemuxBuffer(layout::LaneInfo::SampleInfosContainer::size_type samplesCount,
+ SampleIndex::FastqOffsetsContainer::size_type maxFastqsCount,
+ size_t numCycles)
+ : bclBuffers_(),
+ sampleIndex_(std::make_shared<SampleIndex>(samplesCount, maxFastqsCount)),
+ groupNumber_(0)
+ {
+ }
+
+ void init(layout::LaneInfo::SampleInfosContainer::size_type samplesCount,
+ SampleIndex::FastqOffsetsContainer::size_type maxFastqsCount)
+ {
+ sampleIndex_ = std::make_shared<SampleIndex>(samplesCount, maxFastqsCount);
+ }
+
+ void calculateIndex()
+ {
+ sampleIndex_->reset(bclBuffers_);
+ }
+
+ bool hasCycleData() const
+ {
+ for (const data::BclBuffer &bclBuffer : bclBuffers_) {
+ if (bclBuffer.hasCycleData())
+ return true;
+ }
+ return false;
+ }
+
+ uint32_t getGroupNumber() const { return groupNumber_; }
+ void setGroupNumber(uint32_t groupNumber) { groupNumber_ = groupNumber; }
+
+ data::BclBufferVec bclBuffers_;
+ std::shared_ptr<SampleIndex> sampleIndex_;
+ uint32_t groupNumber_;
+};
+
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_CONVERSION_SAMPLEINDEX_HH
+
+
diff --git a/src/cxx/include/conversion/Stage.hh b/src/cxx/include/conversion/Stage.hh
new file mode 100644
index 0000000..57b7354
--- /dev/null
+++ b/src/cxx/include/conversion/Stage.hh
@@ -0,0 +1,94 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Stage.hh
+ *
+ * \brief Declaration of a processing stage.
+ *
+ * \author Marek Balint
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_STAGE_HH
+#define BCL2FASTQ_CONVERSION_STAGE_HH
+
+
+#include <boost/noncopyable.hpp>
+#include <boost/thread/mutex.hpp>
+#include <future>
+#include <mutex>
+
+#include "conversion/Task.hh"
+#include "conversion/TaskQueue.hh"
+
+
+namespace bcl2fastq {
+namespace conversion {
+
+class TaskManager;
+
+/// \brief Processing stage.
+class Stage : private boost::noncopyable
+{
+public:
+
+ /// \brief Constructor.
+ /// \param stageName Name of the stage.
+ explicit Stage(const std::string& stageName,
+ TaskQueue& taskQueue);
+
+ /// \brief Virtual destructor.
+ virtual ~Stage() = 0;
+
+ /// \brief Run the processing stage.
+ void run();
+
+protected:
+
+ virtual bool startNewTasks() = 0;
+
+ /// \brief Get task queue.
+ /// \return Task queue.
+ TaskQueue & getTaskQueue();
+
+public:
+
+ /// \brief Announce that an error has occurred and thread is going down.
+ /// \note This function should be overridden
+ /// by processing stages to let other threads that this thread is going
+ /// down and no more data will be either consumed nor produced by it.
+ virtual void terminate() = 0;
+
+private:
+ /// \brief Task queue.
+ TaskQueue& taskQueue_;
+
+protected:
+ /// \brief Name of stage
+ std::string stageName_;
+};
+
+class Terminator : private boost::noncopyable
+{
+public:
+ Terminator() : stages_() { }
+
+ void add(Stage* stage) { stages_.push_back(stage); }
+ void terminate() { for (auto& stage : stages_) { stage->terminate(); } }
+
+private:
+ std::vector<Stage*> stages_;
+};
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_CONVERSION_STAGE_HH
+
diff --git a/src/cxx/include/conversion/Task.hh b/src/cxx/include/conversion/Task.hh
new file mode 100644
index 0000000..9581b9c
--- /dev/null
+++ b/src/cxx/include/conversion/Task.hh
@@ -0,0 +1,80 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Task.hh
+ *
+ * \brief Declaration of a workload item a.k.a. task.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_TASK_HH
+#define BCL2FASTQ_CONVERSION_TASK_HH
+
+
+#include <memory>
+#include <atomic>
+#include <mutex>
+#include <condition_variable>
+#include <limits>
+#include <boost/noncopyable.hpp>
+
+namespace bcl2fastq {
+namespace conversion {
+
+class Stage;
+
+class TaskManager
+{
+};
+
+/// \brief Task.
+class Task : private boost::noncopyable
+{
+public:
+
+ enum PriorityLevel
+ {
+ Zero = 0, // Highest priority
+ One,
+ Two,
+ Three,
+ Four,
+ LOWEST_PRIORITY = Four,
+ NUM_PRIORITY_LEVELS
+ };
+
+ /// \brief (Default) constructor.
+ Task(std::shared_ptr<TaskManager> taskManager);
+
+ /// \brief Virtual destructor.
+ virtual ~Task();
+
+ /// \brief Execute the task.
+ /// \retval true Continue with execution on current thread.
+ /// \retval false Finish current execution iteration on current thread.
+ /// \note This function should be overriden in order for task to actually
+ /// do something.
+ virtual bool execute(int32_t threadNum) = 0;
+
+ virtual PriorityLevel getPriority() const { return PriorityLevel::LOWEST_PRIORITY; }
+
+private:
+ std::shared_ptr<TaskManager> taskManager_;
+};
+
+typedef std::shared_ptr<Task> TaskPtr;
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_CONVERSION_TASK_HH
+
+
diff --git a/src/cxx/include/conversion/TaskQueue.hh b/src/cxx/include/conversion/TaskQueue.hh
new file mode 100644
index 0000000..6f3ee12
--- /dev/null
+++ b/src/cxx/include/conversion/TaskQueue.hh
@@ -0,0 +1,56 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file TaskQueue.hh
+ *
+ * \brief Declaration of task queue.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_TASKQUEUE_HH
+#define BCL2FASTQ_CONVERSION_TASKQUEUE_HH
+
+
+#include "conversion/Task.hh"
+#include "conversion/ThreadSafeQueue.hh"
+#include <boost/noncopyable.hpp>
+#include <array>
+#include <deque>
+
+namespace bcl2fastq {
+namespace conversion {
+
+template<typename T, uint8_t NumPriorityLevels>
+class PriorityQueue : private boost::noncopyable
+{
+public:
+ PriorityQueue() : queue_() { }
+
+ size_t size();
+ bool empty();
+ T& front();
+
+ void push_back(T& data);
+ void pop_front();
+
+ void clear();
+
+private:
+ std::array<typename std::deque<T>, NumPriorityLevels> queue_;
+};
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+#include "conversion/TaskQueue.hpp"
+
+#endif // BCL2FASTQ_CONVERSION_TASKQUEUE_HH
+
+
diff --git a/src/cxx/include/conversion/TaskQueue.hpp b/src/cxx/include/conversion/TaskQueue.hpp
new file mode 100644
index 0000000..55c7b89
--- /dev/null
+++ b/src/cxx/include/conversion/TaskQueue.hpp
@@ -0,0 +1,89 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file TaskQueue.hpp
+ *
+ * \brief Declaration of task queue.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_TASKQUEUE_HPP
+#define BCL2FASTQ_CONVERSION_TASKQUEUE_HPP
+
+
+#include "conversion/TaskQueue.hh"
+#include <numeric>
+
+namespace bcl2fastq {
+namespace conversion {
+
+template<typename T, uint8_t NumPriorityLevels>
+size_t PriorityQueue<T, NumPriorityLevels>::size()
+{
+ return std::accumulate(queue_.begin(), queue_.end(), 0, [](size_t total, std::deque<T>& queue) { return total + queue.size(); });
+}
+
+template<typename T, uint8_t NumPriorityLevels>
+bool PriorityQueue<T, NumPriorityLevels>::empty()
+{
+ for (auto& queue : queue_)
+ {
+ if (!queue.empty()) return false;
+ }
+
+ return true;
+}
+
+template<typename T, uint8_t NumPriorityLevels>
+T& PriorityQueue<T, NumPriorityLevels>::front()
+{
+ for (auto& queue : queue_)
+ {
+ if (!queue.empty()) return queue.front();
+ }
+}
+
+template<typename T, uint8_t NumPriorityLevels>
+void PriorityQueue<T, NumPriorityLevels>::push_back(T& data)
+{
+ queue_.at(data->getPriority()).push_back(data);
+}
+
+template<typename T, uint8_t NumPriorityLevels>
+void PriorityQueue<T, NumPriorityLevels>::pop_front()
+{
+ for (auto& queue : queue_)
+ {
+ if (!queue.empty())
+ {
+ queue.pop_front();
+ return;
+ }
+ }
+}
+
+template<typename T, uint8_t NumPriorityLevels>
+void PriorityQueue<T, NumPriorityLevels>::clear()
+{
+ for (auto& queue : queue_)
+ {
+ queue.clear();
+ }
+}
+
+typedef ThreadSafeQueue<TaskPtr, PriorityQueue<TaskPtr, Task::PriorityLevel::NUM_PRIORITY_LEVELS>> TaskQueue;
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_CONVERSION_TASKQUEUE_HPP
+
+
diff --git a/src/cxx/include/conversion/ThreadPool.hh b/src/cxx/include/conversion/ThreadPool.hh
new file mode 100644
index 0000000..9ee2ce2
--- /dev/null
+++ b/src/cxx/include/conversion/ThreadPool.hh
@@ -0,0 +1,80 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ThreadPool.hh
+ *
+ * \brief Definition of helpers for thread pool.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_CONVERSION_THREAD_POOL_HH
+#define BCL2FASTQ_CONVERSION_THREAD_POOL_HH
+
+#include "conversion/Stage.hh"
+#include "conversion/TaskQueue.hh"
+#include "common/SystemCompatibility.hh"
+#include <boost/noncopyable.hpp>
+#include <vector>
+#include <thread>
+#include <mutex>
+#include <boost/exception_ptr.hpp>
+
+namespace bcl2fastq
+{
+namespace conversion
+{
+
+/*
+ * Generic thread pool. Specifiy number of threads on construction. The worker
+ * threads will pull tasks off of the TaskQueue and exectue them.
+ */
+
+class ThreadPool : boost::noncopyable
+{
+public:
+ ThreadPool(uint32_t numThreads,
+ TaskQueue& dataQueue,
+ Terminator& terminator);
+
+ ~ThreadPool() { }
+
+private:
+class JoinThreads : boost::noncopyable
+{
+public:
+ JoinThreads(std::vector<std::thread>& threads,
+ boost::exception_ptr& firstThreadException);
+
+ ~JoinThreads();
+
+private:
+ boost::exception_ptr& firstThreadException_;
+ std::vector<std::thread>& threads_;
+};
+
+ void workerThread(uint32_t threadNum);
+
+ TaskQueue& dataQueue_;
+
+ Terminator& terminator_;
+
+ /// \brief Thread synchronisation mutex.
+ std::mutex mut_;
+
+ /// \brief Exception thrown by worker thread.
+ boost::exception_ptr firstThreadException_;
+
+ std::vector<std::thread> threads_;
+ JoinThreads joiner_;
+};
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_CONVERSION_THREAD_POOL_HH
diff --git a/src/cxx/include/conversion/ThreadSafeQueue.hh b/src/cxx/include/conversion/ThreadSafeQueue.hh
new file mode 100644
index 0000000..bf3e17b
--- /dev/null
+++ b/src/cxx/include/conversion/ThreadSafeQueue.hh
@@ -0,0 +1,101 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ThreadSafeQueue.hh
+ *
+ * \brief Declaration of task queue.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_THREAD_SAFE_QUEUE_HH
+#define BCL2FASTQ_CONVERSION_THREAD_SAFE_QUEUE_HH
+
+
+#include <boost/noncopyable.hpp>
+#include <deque>
+#include <atomic>
+#include <numeric>
+#include <map>
+
+namespace bcl2fastq {
+namespace conversion {
+
+
+/// \brief Thread safe queue.
+template<typename T, class Queue=std::deque<T>>
+class ThreadSafeQueue : boost::noncopyable
+{
+public:
+
+ /// \brief Default constructor.
+ ThreadSafeQueue(uint32_t maxQueueDepth = std::numeric_limits<uint32_t>::max());
+
+ /// \brief Add task.
+ /// \param task Task to be added.
+ /// \note Task queue assumes ownership of the task instance.
+ void addData(T data);
+ void addData(T data, const std::string &description);
+
+ /// \brief Get task.
+ /// \return Task at the top of the queue.
+ /// \note Ownership of the task is retained by the task queue throughout
+ /// whole task's lifetime.
+ bool tryGetData(T& data);
+ bool tryGetData(T& data, const std::string &description);
+
+ bool tryGetDataCheckEmpty(T& data);
+ bool tryGetDataCheckEmpty(T& data, const std::string &description);
+
+ bool isTerminated() const { return terminated_; }
+
+ void setFinished();
+ void terminate();
+
+private:
+ const uint32_t maxQueueDepth_;
+ std::atomic<bool> done_;
+ std::atomic<bool> terminated_;
+ mutable std::mutex mut_;
+ std::condition_variable cvEmptyQueue_;
+ std::condition_variable cvFullQueue_;
+ Queue queue_;
+};
+
+template<typename KeyType, typename DataType>
+class ThreadSafeMap : boost::noncopyable
+{
+public:
+ ThreadSafeMap();
+
+ void addData(const KeyType& key, DataType& data);
+
+ bool tryGetData(const KeyType& key, DataType& data);
+
+ void setFinished(uint32_t lastGroupNumber);
+ void terminate();
+
+private:
+ std::atomic<uint32_t> lastGroupNumber_;
+ std::atomic<bool> done_;
+ std::atomic<bool> terminated_;
+ mutable std::mutex mut_;
+ std::condition_variable cvEmpty_;
+ std::map<KeyType, DataType> map_;
+};
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
+#include "conversion/ThreadSafeQueue.hpp"
+
+#endif // BCL2FASTQ_CONVERSION_THREAD_SAFE_QUEUE_HH
+
+
diff --git a/src/cxx/include/conversion/ThreadSafeQueue.hpp b/src/cxx/include/conversion/ThreadSafeQueue.hpp
new file mode 100644
index 0000000..683fdec
--- /dev/null
+++ b/src/cxx/include/conversion/ThreadSafeQueue.hpp
@@ -0,0 +1,216 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ThreadSafeQueue.hpp
+ *
+ * \brief Implementation of thread safe queue queue.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_CONVERSION_THREAD_SAFE_QUEUE_HPP
+#define BCL2FASTQ_CONVERSION_THREAD_SAFE_QUEUE_HPP
+
+#include "common/Debug.hh"
+#include "conversion/ThreadSafeQueue.hh"
+
+
+namespace bcl2fastq {
+namespace conversion {
+
+
+template<typename T, class Queue>
+ThreadSafeQueue<T, Queue>::ThreadSafeQueue(uint32_t maxQueueDepth /*= std::numceric_limits<uint32_t>::max()*/)
+: maxQueueDepth_(maxQueueDepth)
+, done_(false)
+, terminated_(false)
+, mut_()
+, cvEmptyQueue_()
+, cvFullQueue_()
+, queue_()
+{
+}
+
+template<typename T, class Queue>
+void ThreadSafeQueue<T, Queue>::addData(T data, const std::string &description)
+{
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << description << std::endl;
+ return addData(data);
+}
+
+template<typename T, class Queue>
+void ThreadSafeQueue<T, Queue>::addData(T data)
+{
+ {
+ std::unique_lock<std::mutex> lock(mut_);
+ if (terminated_)
+ {
+ return;
+ }
+
+ if (queue_.size() >= maxQueueDepth_)
+ {
+ cvFullQueue_.wait(lock, [this] { return queue_.size() < maxQueueDepth_ || terminated_; });
+ }
+
+ if (terminated_)
+ {
+ return;
+ }
+
+ queue_.push_back(data);
+ }
+
+ cvEmptyQueue_.notify_one();
+}
+
+template<typename T, class Queue>
+bool ThreadSafeQueue<T, Queue>::tryGetData(T& data, const std::string &description)
+{
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << description << std::endl;
+ return tryGetData(data);
+}
+
+template<typename T, class Queue>
+bool ThreadSafeQueue<T, Queue>::tryGetData(T& data)
+{
+ {
+ std::unique_lock<std::mutex> lock(mut_);
+ cvEmptyQueue_.wait(lock, [this] { return !queue_.empty() || done_ || terminated_; });
+
+ if (queue_.empty() || terminated_) return false;
+
+ data = queue_.front();
+ queue_.pop_front();
+ }
+
+ cvFullQueue_.notify_one();
+
+ return true;
+}
+
+template<typename T, class Queue>
+bool ThreadSafeQueue<T, Queue>::tryGetDataCheckEmpty(T& data, const std::string &description)
+{
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << description << std::endl;
+ return tryGetDataCheckEmpty(data);
+}
+
+template<typename T, class Queue>
+bool ThreadSafeQueue<T, Queue>::tryGetDataCheckEmpty(T& data)
+{
+ {
+ std::unique_lock<std::mutex> lock(mut_);
+
+ if (queue_.empty()) return false;
+
+ data = queue_.front();
+ queue_.pop_front();
+ }
+
+ cvFullQueue_.notify_one();
+
+ return true;
+}
+
+template<typename T, class Queue>
+void ThreadSafeQueue<T, Queue>::setFinished()
+{
+ done_ = true;
+ cvEmptyQueue_.notify_all();
+}
+
+template<typename T, class Queue>
+void ThreadSafeQueue<T, Queue>::terminate()
+{
+ {
+ std::unique_lock<std::mutex> lock(mut_);
+
+ queue_.clear();
+ terminated_ = true;
+ }
+
+ cvEmptyQueue_.notify_all();
+ cvFullQueue_.notify_all();
+}
+
+template<typename KeyType, typename DataType>
+ThreadSafeMap<KeyType, DataType>::ThreadSafeMap()
+: lastGroupNumber_(std::numeric_limits<uint32_t>::max())
+, done_(false)
+, terminated_(false)
+, mut_()
+, cvEmpty_()
+, map_()
+{
+}
+
+template<typename KeyType, typename DataType>
+void ThreadSafeMap<KeyType, DataType>::addData(const KeyType& key, DataType& data)
+{
+ {
+ std::lock_guard<std::mutex> lock(mut_);
+ if (!terminated_)
+ {
+ map_.insert(std::make_pair(key, data));
+ }
+ }
+
+ cvEmpty_.notify_all();
+}
+
+template<typename KeyType, typename DataType>
+bool ThreadSafeMap<KeyType, DataType>::tryGetData(const KeyType& key, DataType& data)
+{
+ {
+ std::unique_lock<std::mutex> lock(mut_);
+
+ typename std::map<KeyType, DataType>::iterator found = map_.end();
+
+ cvEmpty_.wait(lock, [this, key, &found] { found = map_.find(key); return done_ || terminated_ || (found != map_.end() || key > lastGroupNumber_); });
+
+ if (found == map_.end() || terminated_)
+ {
+ return false;
+ }
+
+ data = found->second;
+ map_.erase(found);
+ }
+
+ cvEmpty_.notify_all();
+
+ return true;
+}
+
+template<typename KeyType, typename DataType>
+void ThreadSafeMap<KeyType, DataType>::setFinished(uint32_t lastGroupNumber)
+{
+ lastGroupNumber_ = lastGroupNumber;
+ done_ = true;
+ cvEmpty_.notify_all();
+}
+
+template<typename KeyType, typename DataType>
+void ThreadSafeMap<KeyType, DataType>::terminate()
+{
+ {
+ std::unique_lock<std::mutex> lock(mut_);
+
+ map_.clear();
+ terminated_ = true;
+ }
+
+ cvEmpty_.notify_all();
+}
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_CONVERSION_THREAD_SAFE_QUEUE_HPP
+
diff --git a/src/cxx/include/data/AggregatedBclFileReader.hh b/src/cxx/include/data/AggregatedBclFileReader.hh
new file mode 100644
index 0000000..c742c1f
--- /dev/null
+++ b/src/cxx/include/data/AggregatedBclFileReader.hh
@@ -0,0 +1,56 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file TileBclFileReader.hh
+ *
+ * \brief Declaration of BCL reader for a single tile.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_DATA_AGGREGATED_BCL_FILE_READER_HH
+#define BCL2FASTQ_DATA_AGGREGATED_BCL_FILE_READER_HH
+
+#include "data/BclFileReader.hh"
+#include "data/RawBclBuffer.hh"
+#include "data/CycleBCIFile.hh"
+#include "io/SyncFile.hh"
+
+#include <boost/filesystem/path.hpp>
+#include <boost/noncopyable.hpp>
+
+#include <memory>
+
+namespace bcl2fastq {
+namespace data {
+
+
+class AggregatedBclFileReader : public BclFileReaderT<io::SyncFile>
+{
+public:
+ AggregatedBclFileReader(const::boost::filesystem::path& inputDir,
+ const layout::LaneInfo& laneInfo,
+ common::CycleNumber cycleNumber,
+ size_t cycleIndex,
+ bool ignoreMissingBcls,
+ std::shared_ptr<io::SyncFile> bclFile,
+ std::shared_ptr<data::CycleBCIFile> cycleBciFile);
+
+ virtual bool read(RawBclBufferGroup& outputBuffer);
+
+private:
+ bool openAggFileIfNeeded();
+
+ std::shared_ptr<data::CycleBCIFile> cycleBciFile_;
+};
+
+} // namespace data
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_DATA_AGGREGATED_BCL_FILE_READER_HH
+
diff --git a/src/cxx/include/data/BclBuffer.hh b/src/cxx/include/data/BclBuffer.hh
new file mode 100644
index 0000000..f9fb1d6
--- /dev/null
+++ b/src/cxx/include/data/BclBuffer.hh
@@ -0,0 +1,125 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BclBuffer.hh
+ *
+ * \brief Declaration of BCL buffer.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_DATA_BCLBUFFER_HH
+#define BCL2FASTQ_DATA_BCLBUFFER_HH
+
+
+#include <vector>
+
+#include "data/PositionsFile.hh"
+#include "data/FilterFile.hh"
+#include "data/ControlFile.hh"
+#include "data/RawBclBuffer.hh"
+#include "layout/Layout.hh"
+#include "layout/LaneInfo.hh"
+#include "layout/BarcodeTranslationTable.hh"
+
+namespace bcl2fastq {
+namespace data {
+
+// Read 5 million clusters from a bcl file at once.
+static const size_t ClustersPerReadBuffer = 5000000;
+
+static const size_t ClustersPerTask = 16384;
+
+/// \brief BCL buffer.
+struct BclBuffer
+{
+public:
+
+ /// \brief BCL cycles data container type definition.
+ typedef std::vector<PerCycleData> BclCycleContainer;
+
+ /// \brief Cluster to sample assignment data container type definition.
+ typedef std::vector< std::pair<layout::BarcodeTranslationTable::SampleMetadata, layout::Barcode> > SamplesContainer;
+
+ /// \brief Position data container type definition.
+ typedef std::vector<data::PositionsFile::Record> PositionsContainer;
+
+ /// \brief Filter data container type definition.
+ typedef std::vector<data::FilterFile::Record> FiltersContainer;
+
+ /// \brief Control data container type definition.
+ typedef std::vector<data::ControlFile::Record> ControlsContainer;
+
+ class PatternedPositionsContainer : public PositionsContainer
+ {
+ public:
+ PatternedPositionsContainer() : PositionsContainer(), mut_(), cv_(), isReady_(false) { }
+
+ virtual ~PatternedPositionsContainer() { }
+
+ void setReady() { isReady_ = true; cv_.notify_all(); }
+
+ void waitForReady() { if (!isReady_) { std::unique_lock<std::mutex> lock(mut_); cv_.wait(lock, [this] { return isReady_.load(); }); } }
+
+ private:
+ std::mutex mut_;
+ std::condition_variable cv_;
+ std::atomic<bool> isReady_;
+ };
+
+ bool hasCycleData() const {
+ for (const PerCycleData & perCycleData : bcls_) {
+ if (!(perCycleData.bcls_.empty())) return true;
+ }
+ return false;
+ }
+
+public:
+
+ /// \brief Buffers for cycles data.
+ BclCycleContainer bcls_;
+
+ /// \brief Cluster to sample assignments.
+ SamplesContainer samples_;
+
+ /// \brief CBCL filter offsets. Used for the offsets for the last cycles which don't include non-pf data.
+ std::vector<PerCycleData::BclsContainer::size_type> cbclFilterOffsets_;
+
+ /// \brief Tile meta data.
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfo_;
+
+ /// \brief Buffer for position data.
+ std::shared_ptr<PositionsContainer> positions_;
+
+ /// \brief Buffer for filter data.
+ FiltersContainer filters_;
+
+ /// \brief Buffer for control data.
+ ControlsContainer controls_;
+
+public:
+
+ /// \brief Number of BCL clusters to be processed in single execution iteration.
+ /// \todo Refactoring: Put this into CMake configuration.
+ static const size_t clustersPerChunk_;
+};
+
+typedef std::vector<BclBuffer> BclBufferVec;
+
+/// \brief Swap function for BCL buffer.
+/// \param lhs Left-hand-side parameter.
+/// \param rhs Right-hand-side parameter.
+void swap(BclBuffer &lhs, BclBuffer &rhs);
+
+
+} // namespace data
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_DATA_BCLBUFFER_HH
+
diff --git a/src/cxx/include/data/BclFile.hh b/src/cxx/include/data/BclFile.hh
new file mode 100644
index 0000000..0bdb03b
--- /dev/null
+++ b/src/cxx/include/data/BclFile.hh
@@ -0,0 +1,113 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BclFile.hh
+ *
+ * \brief Declaration of BCL file.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_DATA_BCLFILE_HH
+#define BCL2FASTQ_DATA_BCLFILE_HH
+
+
+#include <memory>
+
+#include <boost/filesystem/path.hpp>
+
+#include "common/Types.hh"
+#include "data/FileReader.hh"
+
+
+namespace bcl2fastq {
+
+namespace io {
+class GzipDecompressor;
+}
+
+namespace data {
+
+
+/// \brief BCL file.
+class BclFile : public BinaryFileReader
+{
+ typedef boost::error_info<struct tag_errmsg, std::string> errmsg_info;
+public:
+
+ /// \brief Default constructor.
+ BclFile(const common::RawDataBuffer& data,
+ io::GzipDecompressor& decompressor,
+ bool ignoreErrors,
+ bool parseHeader,
+ bool resetDecompressor = true,
+ common::ClustersCount defaultClustersCount = 0);
+
+ virtual ~BclFile() { }
+
+public:
+ /// \brief Verify the bcl file exists and return the name. (single tile).
+ /// \param inputDir Directory to look for the bcl file.
+ /// \param laneNumber The lane number.
+ /// \param tileNumber The tile number.
+ /// \param cycleNumber The cycle number.
+ /// \param ignoreErrors Supress errors opening file and/or reading its header.
+ /// \param fileName Output parameter for the bcl file name.
+ static bool getAndVerifyFileName(const boost::filesystem::path& inputDir,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ common::CycleNumber cycleNumber,
+ bool ignoreErrors,
+ boost::filesystem::path& fileName);
+
+ /// \brief Verify the bcl file exists and return the name. (aggregate tiles).
+ /// \param inputDir Directory to look for the bcl file.
+ /// \param laneNumber The lane number.
+ /// \param cycleNumber The cycle number.
+ /// \param ignoreErrors Supress errors opening file and/or reading its header.
+ /// \param fileName Output parameter for the bcl file name.
+ static bool getAndVerifyFileName(const boost::filesystem::path& inputDir,
+ common::LaneNumber laneNumber,
+ common::CycleNumber cycleNumber,
+ bool ignoreErrors,
+ boost::filesystem::path& fileName);
+
+ /// \brief Read bytes from file to buffer.
+ /// \param targetBuffer Target buffer to read to.
+ /// \param targetSize Maximum number of bytes to be read.
+ /// \pre <tt>this->isOpen() == true</tt>
+ /// \return Number of bytes read.
+ std::streamsize read(char *targetBuffer, std::streamsize targetSize);
+
+ /// \brief Seek to specified position.
+ /// \param compressedOffset Offset in compressed data denoting start of BGZF block.
+ /// \param uncompressedOffset Offset in uncompressed data of the BGZF block.
+ /// \return true if seek was successful.
+ bool seek(std::streamsize compressedOffset, std::streamsize uncompressedOffset);
+
+private:
+ std::streamsize read(std::istream &is, char *targetBuffer, std::streamsize targetSize);
+
+ /// \brief Gets the file type string used for error messages.
+ /// \return File type string
+ virtual std::string getFileTypeStr() const { return "BCL"; }
+
+ /// \brief GZip decompression filter.
+ io::GzipDecompressor* gzipDecompressor_;
+
+ /// \brief Whether we need to use the decompressor for reading or not.
+ bool isCompressed_;
+};
+
+} // namespace data
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_DATA_BCLFILE_HH
+
diff --git a/src/cxx/include/data/BclFileReader.hh b/src/cxx/include/data/BclFileReader.hh
new file mode 100644
index 0000000..574ea99
--- /dev/null
+++ b/src/cxx/include/data/BclFileReader.hh
@@ -0,0 +1,88 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BclReader.hh
+ *
+ * \brief Declaration of BCL reader.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_DATA_BCL_FILE_READER_HH
+#define BCL2FASTQ_DATA_BCL_FILE_READER_HH
+
+#include "data/RawBclBuffer.hh"
+#include "io/SyncFile.hh"
+#include "layout/LaneInfo.hh"
+#include "common/Types.hh"
+
+#include <boost/filesystem/path.hpp>
+#include <boost/noncopyable.hpp>
+
+#include <memory>
+
+namespace bcl2fastq {
+namespace data {
+
+class BclFileReader : private boost::noncopyable
+{
+public:
+ BclFileReader(const::boost::filesystem::path& inputDir,
+ const layout::LaneInfo& laneInfo,
+ common::CycleNumber cycleNumber,
+ size_t cycleIndex,
+ bool ignoreMissingBcls)
+ : inputDir_(inputDir)
+ , laneInfo_(laneInfo)
+ , cycleNumber_(cycleNumber)
+ , cycleIndex_(cycleIndex)
+ , ignoreMissingBcls_(ignoreMissingBcls)
+ {
+ }
+
+ virtual ~BclFileReader() { }
+ virtual bool read(RawBclBufferGroup& outputBuffer) = 0;
+
+protected:
+ const boost::filesystem::path inputDir_;
+ const layout::LaneInfo& laneInfo_;
+ common::CycleNumber cycleNumber_;
+ size_t cycleIndex_;
+ bool ignoreMissingBcls_;
+};
+
+template<class BclFileType>
+class BclFileReaderT : public BclFileReader
+{
+public:
+ BclFileReaderT(const::boost::filesystem::path& inputDir,
+ const layout::LaneInfo& laneInfo,
+ common::CycleNumber cycleNumber,
+ size_t cycleIndex,
+ bool ignoreMissingBcls,
+ std::shared_ptr<BclFileType> bclFile)
+ : BclFileReader(inputDir,
+ laneInfo,
+ cycleNumber,
+ cycleIndex,
+ ignoreMissingBcls)
+ , bclFile_(bclFile)
+ {
+ }
+
+ virtual ~BclFileReaderT() { }
+
+protected:
+ std::shared_ptr<BclFileType> bclFile_;
+};
+
+} // namespace data
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_DATA_BCL_FILE_READER_HH
+
diff --git a/src/cxx/include/data/CbclFile.hh b/src/cxx/include/data/CbclFile.hh
new file mode 100644
index 0000000..7e778e7
--- /dev/null
+++ b/src/cxx/include/data/CbclFile.hh
@@ -0,0 +1,192 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file CbclFile.hh
+ *
+ * \brief Declaration of CBCL file.
+ *
+ * \author Aaron Day
+ * \author Eunho Noh
+ */
+
+
+#ifndef BCL2FASTQ_DATA_CBCLFILE_HH
+#define BCL2FASTQ_DATA_CBCLFILE_HH
+
+#include "data/BclFileReader.hh"
+#include "data/BclFile.hh"
+#include "data/BclBuffer.hh"
+
+#include <boost/noncopyable.hpp>
+#include <istream>
+
+namespace bcl2fastq {
+
+namespace layout {
+class TileInfo;
+}
+
+namespace data {
+
+class CbclFileReader : public BclFileReaderT<io::SyncFile>
+{
+public:
+ CbclFileReader(const::boost::filesystem::path& inputDir,
+ const layout::LaneInfo& laneInfo,
+ common::CycleNumber cycleNumber,
+ size_t cycleIndex,
+ bool ignoreMissingBcls,
+ const common::TileFileMap& tileFileMap,
+ std::shared_ptr<io::SyncFile> bclFile);
+
+ virtual ~CbclFileReader() { }
+
+ virtual bool read(RawBclBufferGroup& outputBuffer);
+
+ static bool doesFileExist(const boost::filesystem::path& inputDir,
+ common::LaneNumber laneNumber,
+ common::CycleNumber cycleNumber,
+ common::TileNumber tileNumber,
+ const common::TileFileMap& tileFileMap,
+ bool ignoreErrors);
+
+ static boost::filesystem::path getFileName(const boost::filesystem::path& inputDir,
+ common::LaneNumber laneNumber,
+ common::CycleNumber cycleNumber,
+ common::TileNumber tileNumber,
+ const common::TileFileMap& tileFileMap,
+ bool ignoreErrors);
+
+class Header : private boost::noncopyable
+{
+private:
+ typedef uint16_t header_version_t;
+ typedef uint8_t header_byte_t;
+ typedef uint32_t header_value_t;
+ typedef std::vector<header_value_t> header_value_vector_t;
+ typedef char stream_byte_t;
+ typedef std::vector<stream_byte_t> header_t;
+
+public:
+ Header();
+
+ header_byte_t getNumberOfBitsPerBasecall() const { return m_number_of_bits_per_basecall; }
+ header_byte_t getNumberOfBitsPerQscore() const { return m_number_of_bits_per_qscore; }
+ header_value_t getNumberOfTiles() const { return m_number_of_tiles; }
+ const header_value_vector_t& getTileIds() const { return m_tile_ids; }
+
+ // Used for invalid cbcl file.
+ void reset();
+ bool load(io::SyncFile::SyncFileReader& cbclFile, bool ignoreErrors);
+
+ /** Returns the index of the tile number of interest
+ *
+ * @param tile_id the tile to find
+ */
+ size_t getTileIndex(size_t tile_id) const
+ {
+ auto pos = std::find(m_tile_ids.begin(), m_tile_ids.end(), tile_id);
+ if(pos == m_tile_ids.end())
+ {
+ throw std::out_of_range("Invalid tile id");
+ }
+ return pos - m_tile_ids.begin();
+ }
+
+ bool containsTile(size_t tile_id) const
+ {
+ return std::find(m_tile_ids.begin(), m_tile_ids.end(), tile_id) != m_tile_ids.end();
+ }
+
+ /** Returns the tile number
+ *
+ * @param tile_index the index to the m_tileNumbers vector
+ */
+ size_t getTileId(size_t tile_index) const
+ {
+ if(tile_index >= static_cast<size_t>(m_tile_ids.size()))
+ {
+ throw std::out_of_range("Invalid tile index");
+ }
+ return m_tile_ids[tile_index];
+ }
+
+ const header_value_vector_t& getRemappedQscores() const { return m_remapped_qscores; }
+ // header_value_t getRemappedQscore(header_value_t q_score_bin) const { return m_remapped_qscores[q_score_bin]; }
+
+ size_t getBlockStartPosition(size_t tile_index) const { return m_start_positions[tile_index]; }
+ size_t getNumberOfClusters(size_t tile_index) const { return m_number_of_clusters[tile_index]; }
+ uint32_t getUncompressedBlockSize(size_t tile_index) const { return m_uncompressed_block_size[tile_index]; }
+ size_t getCompressedBlockSize(size_t tile_index) const { return m_compressed_block_size[tile_index]; }
+ uint8_t isNonPfFiltered() const { return m_non_pf_filtered; }
+ bool isQscoreDeflated() const { return m_number_of_qscore_bins > 0; }
+
+ header_value_t getHeaderSize() { return m_header_size; }
+
+private:
+ void readHeader(io::SyncFile::SyncFileReader& in);
+ header_value_t calculateHeaderSize(size_t number_of_qscore_bins, size_t number_of_tiles) const;
+ std::string validateHeaderInfo() const;
+
+ /** Read an array of data from an input stream
+ *
+ * @param in input stream
+ * @param buffer array of data
+ * @param n number of elements in the array
+ */
+ template<class T>
+ void read_binary(io::SyncFile::SyncFileReader& in, T* buffer, const size_t n)
+ {
+ bool validRead = in.read(reinterpret_cast<char *>(buffer), n*sizeof(T));
+ if (!validRead) {
+ BOOST_THROW_EXCEPTION(std::ios_base::failure("Corrupt CBCL header cannot be loaded. "));
+ }
+
+ }
+ /** Read an value of binary data from an input stream
+ *
+ * @param in input stream
+ * @param buffer value
+ */
+ template<class T>
+ void read_binary(io::SyncFile::SyncFileReader& in, T& buffer)
+ {
+ read_binary(in, &buffer, 1);
+ }
+
+ const size_t BITS_PER_BYTE = 8;
+ const size_t NUM_BITS_BCL_STORAGE = 4;
+ const size_t BASECALLS_PER_BYTE = BITS_PER_BYTE / NUM_BITS_BCL_STORAGE;
+
+ header_version_t m_version;
+ header_value_t m_header_size;
+ header_byte_t m_number_of_bits_per_basecall;
+ header_byte_t m_number_of_bits_per_qscore;
+ header_value_t m_number_of_qscore_bins;
+ header_value_vector_t m_qscore_bins;
+ header_value_vector_t m_remapped_qscores;
+ header_value_t m_number_of_tiles;
+ header_value_vector_t m_tile_ids;
+ header_value_vector_t m_number_of_clusters;
+ header_value_vector_t m_uncompressed_block_size;
+ header_value_vector_t m_compressed_block_size;
+ header_byte_t m_non_pf_filtered;
+ std::vector<size_t> m_start_positions;
+}; // end class Header
+
+ void markCycleDataInvalid(PerCycleData &cycleData);
+
+ const common::TileFileMap& tileFileMap_;
+ std::unique_ptr<Header> header_;
+};
+
+} // namespace data
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_DATA_CBCLFILE_HH
+
diff --git a/src/cxx/include/data/ControlFile.hh b/src/cxx/include/data/ControlFile.hh
new file mode 100644
index 0000000..99bb3cd
--- /dev/null
+++ b/src/cxx/include/data/ControlFile.hh
@@ -0,0 +1,109 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ControlFile.hh
+ *
+ * \brief Declaration of control file.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_DATA_CONTROLFILE_HH
+#define BCL2FASTQ_DATA_CONTROLFILE_HH
+
+
+#include <boost/filesystem/path.hpp>
+
+#include "data/FileReader.hh"
+#include "common/Types.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace data {
+
+
+/// \brief Control file.
+class ControlFile : public BinaryAllClustersFileReader
+{
+public:
+
+ /// \brief Control file record type definition.
+ struct Record
+ {
+ public:
+ Record() : data_(0) { }
+
+ Record(uint16_t data) : data_(data) { }
+
+ /// \brief Control data.
+ uint16_t data_;
+ };
+
+public:
+
+ /// \brief Construct and open control file.
+ /// \param filePath Path to control file.
+ /// \param ignoreErrors Supress errors opening file and/or reading it.
+ ControlFile(
+ const common::RawDataBuffer& data,
+ bool ignoreErrors = false
+ );
+
+ /// \brief Read records from the file to buffer.
+ /// \param targetBuffer Target buffer to read to.
+ /// \param targetSize Maximum number of records to be read.
+ /// \return Number of records read.
+ /// \pre <tt>this->isOpen() == true</tt>
+ std::size_t read(
+ ControlFile::Record *targetBuffer,
+ std::size_t targetSize
+ );
+
+ static bool doesFileExist(const boost::filesystem::path& intensitiesDir,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ boost::filesystem::path& fileName);
+
+private:
+ /// \brief Control V2 header type definition.
+#pragma pack(push, 1)
+ struct Header
+ {
+ /// \brief Value of the version field.
+ static const uint32_t VERSION;
+
+ /// \brief Version field.
+ uint32_t version_;
+
+ /// \brief Clusters count.
+ uint32_t clustersCount_;
+ };
+#pragma pack(pop)
+
+ /// \brief Read the header.
+ virtual bool readHeader();
+
+ /// \brief Validate the header. Throw an exception on failure.
+ virtual bool validateHeader(const Header& header);
+
+ /// \brief Return the number of bytes in a record.
+ virtual std::size_t getRecordBytes() const { return sizeof(Record); }
+
+ /// \brief Return the file type string. Used for logging message purposes.
+ virtual std::string getFileTypeStr() const { return "Control"; }
+};
+
+
+} // namespace data
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_DATA_CONTROLFILE_HH
+
diff --git a/src/cxx/include/data/CycleBCIFile.hh b/src/cxx/include/data/CycleBCIFile.hh
new file mode 100644
index 0000000..558a9bf
--- /dev/null
+++ b/src/cxx/include/data/CycleBCIFile.hh
@@ -0,0 +1,155 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file CycleBCIFile.hh
+ *
+ * \brief Declaration of Cycle BCI file.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_DATA_CYCLEBCIFILE_HH
+#define BCL2FASTQ_DATA_CYCLEBCIFILE_HH
+
+
+#include <boost/filesystem/path.hpp>
+
+#include "common/Types.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace data {
+
+
+/// \brief Cycle BCI file.
+class CycleBCIFile
+{
+public:
+
+ /// \brief Cycle BCI file record type definition.
+#pragma pack(push, 1)
+ struct Record
+ {
+ public:
+
+ /// \brief Uncompressed offset.
+ uint64_t uncompressedOffset : 16;
+
+ /// \brief Compressed offset.
+ uint64_t compressedOffset : 48;
+ };
+#pragma pack(pop)
+
+public:
+
+ /// \brief Construction
+ CycleBCIFile(bool ignoreMissingBcls);
+
+ /// \brief Open filter file.
+ /// \param inputDir Path to input directory.
+ /// \param laneNumber Lane number.
+ /// \param cycleNumber Cycle number.
+ bool openFile(
+ const boost::filesystem::path &inputDir,
+ common::LaneNumber laneNumber,
+ common::CycleNumber cycleNumber
+ );
+
+ static bool getAndVerifyFileName(
+ const boost::filesystem::path &inputDir,
+ common::LaneNumber laneNumber,
+ common::CycleNumber cycleNumber,
+ bool ignoreErrors,
+ boost::filesystem::path &fileName
+ );
+
+ /// \brief Check whether the file is open.
+ /// \retval true File is open.
+ /// \retval false File is not open.
+ bool isOpen() const;
+
+ /// \brief Get file path.
+ /// \return File path.
+ /// \pre <tt>this->isOpen() == true</tt>
+ boost::filesystem::path getPath() const;
+
+ /// \brief Get number of tiles.
+ /// \return Number of tiles in the file.
+ /// \pre <tt>this->isOpen() == true</tt>
+ common::TileNumber getTilesCount() const;
+
+ /// \brief Read records from the file to buffer.
+ /// \param targetBuffer Target buffer to read to.
+ /// \param targetSize Maximum number of records to be read.
+ /// \return Number of records read.
+ /// \pre <tt>this->isOpen() == true</tt>
+ std::size_t read(
+ CycleBCIFile::Record *targetBuffer,
+ std::size_t targetSize
+ );
+
+ /// \brief Get record for given tile index.
+ /// \param tileIndex Tile index.
+ /// \return Record for given tile index.
+ /// \pre <tt>this->isOpen() == true</tt>
+ Record getRecord(std::size_t tileIndex) const;
+
+private:
+
+ /// \brief Cycle BCIndex V0 header type definition.
+#pragma pack(push, 1)
+ struct Header
+ {
+ public:
+
+ /// \brief Value of the version field.
+ static const uint32_t VERSION;
+
+ public:
+
+ /// \brief Version field.
+ uint32_t version_;
+
+ /// \brief Clusters count.
+ uint32_t tilesCount_;
+
+ };
+#pragma pack(pop)
+
+private:
+
+ /// \brief Ignore missing bcl/bci data
+ bool ignoreMissingBcls_ = false;
+
+ /// \brief File path.
+ boost::filesystem::path path_;
+
+ /// \brief Number of clusters.
+ common::TileNumber tilesCount_;
+
+ /// \brief Internal buffer.
+ std::vector<char> buffer_;
+
+ /// \brief Current position in internal buffer.
+ std::vector<char>::const_iterator bufferPosition_;
+
+};
+
+
+} // namespace data
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_BCINDEX_HH
+
+
diff --git a/src/cxx/include/data/FastqFile.hh b/src/cxx/include/data/FastqFile.hh
new file mode 100644
index 0000000..2822c14
--- /dev/null
+++ b/src/cxx/include/data/FastqFile.hh
@@ -0,0 +1,109 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FastqFile.hh
+ *
+ * \brief Declaration of FASTQ file.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_DATA_FASTQFILE_HH
+#define BCL2FASTQ_DATA_FASTQFILE_HH
+
+
+#include <boost/shared_ptr.hpp>
+#include <boost/filesystem/path.hpp>
+
+#include "io/FileBufWithReopen.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace data {
+
+
+class FastqFileState;
+
+
+/// \brief FASTQ file.
+class FastqFile
+{
+public:
+
+ /// \brief FASTQ file state smart pointer type definition.
+ typedef boost::shared_ptr<FastqFileState> FastqFileStatePtr;
+
+public:
+
+ /// \brief Open FASTQ file.
+ /// \param path Path to the FASTQ file to be opened.
+ /// \param mode Access mode to be used.
+ /// \return External state of opened FASTQ file.
+ static FastqFile::FastqFileStatePtr openFile(const boost::filesystem::path &path,
+ std::ios_base::openmode mode);
+
+public:
+
+ /// \brief Get file path.
+ /// \param fileState FASTQ file external state.
+ /// \return File path.
+ static const boost::filesystem::path& getPath(const FastqFileState &fileState);
+
+ /// \brief Write bytes from buffer to file.
+ /// \param fileState FASTQ file external state.
+ /// \param sourceBuffer Source buffer to read to.
+ /// \param sourceSize Number of bytes to be written.
+ static void write(FastqFileState &fileState, const char *sourceBuffer, std::streamsize sourceSize);
+};
+
+
+/// \brief FASTQ file internal state.
+class FastqFileState
+{
+protected:
+
+ /// \brief Default constructor.
+ FastqFileState();
+
+ /// \brief Virtual destructor.
+ virtual ~FastqFileState() = 0;
+
+protected:
+
+ /// \brief Write bytes from buffer to file.
+ /// \param os Output stream to write to.
+ /// \param sourceBuffer Source buffer to read to.
+ /// \param sourceSize Number of bytes to be written.
+ virtual void write(std::ostream &os, const char *sourceBuffer, std::streamsize sourceSize) = 0;
+
+private:
+
+ /// \brief File path.
+ boost::filesystem::path path_;
+
+ /// \brief Filebuf.
+ io::FileBufWithReopen fileBuf_;
+
+private:
+
+ friend class FastqFile;
+};
+
+
+} // namespace data
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_DATA_FASTQFILE_HH
+
+
diff --git a/src/cxx/include/data/FileReader.hh b/src/cxx/include/data/FileReader.hh
new file mode 100644
index 0000000..924d5d4
--- /dev/null
+++ b/src/cxx/include/data/FileReader.hh
@@ -0,0 +1,191 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FileReader.hh
+ *
+ * \brief Declaration of FileReader file.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_DATA_FILEREADER_HH
+#define BCL2FASTQ_DATA_FILEREADER_HH
+
+#include "common/Types.hh"
+#include "common/Exceptions.hh"
+#include "io/SyncFile.hh"
+
+#include <boost/noncopyable.hpp>
+#include <boost/filesystem/path.hpp>
+#include <boost/format.hpp>
+#include <boost/iostreams/stream.hpp>
+#include <boost/iostreams/device/array.hpp>
+
+namespace bcl2fastq {
+
+namespace data {
+
+
+/// \brief Base class for all file readers
+class FileReaderBase
+{
+public:
+ /// \brief Constructor
+ /// \param data Raw data from file.
+ /// \param ignoreErrors
+ /// \param defaultClustersCount Default number of clusters
+ FileReaderBase(const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ common::ClustersCount defaultClustersCount = 0);
+
+ /// \brief Destructor
+ virtual ~FileReaderBase() = 0;
+
+ /// \brief Get file path.
+ /// \return File path.
+ /// \pre <tt>this->isOpen() == true</tt>
+ virtual const boost::filesystem::path& getPath() const;
+
+ /// \brief Get number of clusters.
+ /// \return Number of clusters in the file.
+ /// \pre <tt>this->isOpen() == true</tt>
+ virtual common::ClustersCount getClustersCount() const;
+
+ /// \brief Check whether the file is open.
+ /// \retval true File is open.
+ /// \retval false File is not open.
+ virtual bool isOpen() const { return !data_->path_.empty(); }
+
+protected:
+
+ /// \brief Validate the condition is true.
+ /// \param warningMsg The warning message to use if condition is false
+ /// \retval condition
+ bool validateCondition(bool condition, const std::string& warningMsg) const;
+
+ /// \brief Raw data
+ const common::RawDataBuffer* data_;
+
+ /// \brief Ignore errors opening file and/or reading its header.
+ bool ignoreErrors_;
+
+ /// \brief Number of clusters.
+ common::ClustersCount clustersCount_;
+
+ boost::iostreams::basic_array_source<char> inputSrc_;
+ boost::iostreams::stream<boost::iostreams::basic_array_source<char>> istr_;
+};
+
+/// \brief File reading class. Virtual inheritance because of diamond inheritance.
+class FileReader : public virtual FileReaderBase, private boost::noncopyable
+{
+public:
+
+ /// \brief Constructor. Resource acquisition is initialization.
+ /// \param data Raw data from file.
+ /// \param ignoreErrors Supress errors opening file and/or reading its header.
+ /// \param defaultClustersCount Number of clusters to assume in case of error opening the file
+ /// and/or reading its header.
+ FileReader(const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ common::ClustersCount defaultClustersCount = 0);
+
+ /// \brief Destructor
+ virtual ~FileReader() = 0;
+
+protected:
+
+ /// \brief Return the file type string. Used for logging message purposes.
+ virtual std::string getFileTypeStr() const = 0;
+
+ /// \brief Log an error message on file io failure.
+ /// \param bytesRead Number of bytes read from file.
+ /// \param bytesExpected Number of bytes expected to be read from file.
+ virtual void logError(std::streamsize bytesRead,
+ std::streamsize bytesExpected) const;
+
+ std::streamsize readBytes(char* buffer,
+ uint32_t bytes);
+};
+
+/// \brief Binary File Reader. Base class for reading binary files.
+class BinaryFileReader : public FileReader
+{
+public:
+
+ /// \brief Constructor. Resource acquisition is initialization.
+ /// \param data Raw data from file.
+ /// \param ignoreErrors Supress errors opening file and/or reading its header.
+ /// \param defaultClustersCount Number of clusters to assume in case of error opening the file
+ /// and/or reading its header.
+ BinaryFileReader(const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ common::ClustersCount defaultClustersCount =0);
+
+ /// \brief Destructor
+ virtual ~BinaryFileReader() = 0;
+
+protected:
+ /// \brief Read bytes from file to buffer.
+ /// \param targetBuffer Target buffer to read to.
+ /// \param targetSize Maximum number of bytes to be read.
+ /// \return Number of bytes read.
+ virtual std::streamsize read(char* targetBuffer,
+ std::streamsize targetSize);
+
+ /// \brief Read the header. Template allows derived classes to implement their own header
+ /// class without inheritance, which would increase the size of an instance.
+ /// \param header Header to read.
+ template<typename HEADER>
+ bool readHeader(HEADER& header);
+
+ /// \brief Read the header. The template readHeader method only loads the data from file
+ /// into the HEADER instance. This method is implemented by derived classes to validate
+ /// and do something with the data.
+ virtual bool readHeader() { return true; }
+};
+
+/// \brief Binary file class that reads all clusters from the file on initialization.
+class BinaryAllClustersFileReader : public BinaryFileReader
+{
+public:
+
+ /// \brief Constructor
+ /// \param data Raw data from file
+ /// \param ignoreErrors Supress errors opening file and/or reading its header.
+ BinaryAllClustersFileReader(const common::RawDataBuffer& data,
+ bool ignoreErrors);
+
+protected:
+
+ /// \brief Read the clusters from file
+ /// \param buffer Buffer to read data into
+ /// \param clustersCount Number of clusters to read (size of buffer)
+ /// \retval true on success, false on failure.
+ virtual bool readClusters(std::vector<char>& buffer,
+ uint32_t clustersCount);
+
+ /// \brief Validate the header. Throw an exception on failure.
+ virtual void validateHeader();
+
+ /// \brief Return the number of bytes in a record.
+ virtual std::size_t getRecordBytes() const = 0;
+
+ /// \brief Read all records in the file.
+ virtual void readRecords() { }
+};
+
+} // namespace data
+
+} // namespace bcl2fastq
+
+
+#include "data/FileReader.hpp"
+
+
+#endif // BCL2FASTQ_DATA_FILEREADER_HH
diff --git a/src/cxx/include/data/FileReader.hpp b/src/cxx/include/data/FileReader.hpp
new file mode 100644
index 0000000..1d3a106
--- /dev/null
+++ b/src/cxx/include/data/FileReader.hpp
@@ -0,0 +1,61 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FileReader.hh
+ *
+ * \brief Declaration of FileReader file.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_DATA_FILEREADER_HPP
+#define BCL2FASTQ_DATA_FILEREADER_HPP
+
+#include <boost/function.hpp>
+
+namespace bcl2fastq
+{
+namespace data
+{
+
+template<typename HEADER>
+bool BinaryFileReader::readHeader(HEADER& header)
+{
+ std::streamsize headerLength = this->read(reinterpret_cast<char *>(&header), sizeof(header));
+
+ int errnum = errno;
+ if (headerLength != sizeof(header))
+ {
+ boost::format errFormat(boost::format(
+ "Unable to read header of '%s' file '%s' : bytes_read=%d bytes_expected=%d") %
+ this->getFileTypeStr() % this->getPath().string() % headerLength % sizeof(header));
+
+ if (ignoreErrors_)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING)
+ << errFormat
+ << ":" << std::strerror(errnum) << " (" << errnum << ")" << std::endl;
+
+ return false;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(
+ common::InputDataError(errnum,
+ errFormat.str()));
+ }
+ }
+
+ return true;
+}
+
+} // namespace data
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_DATA_FILEREADER_HPP
+
diff --git a/src/cxx/include/data/FilterFile.hh b/src/cxx/include/data/FilterFile.hh
new file mode 100644
index 0000000..dfc0b3f
--- /dev/null
+++ b/src/cxx/include/data/FilterFile.hh
@@ -0,0 +1,126 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FilterFile.hh
+ *
+ * \brief Declaration of filter file.
+ *
+ * \author Marek Balint
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_DATA_FILTERFILE_HH
+#define BCL2FASTQ_DATA_FILTERFILE_HH
+
+
+#include "data/FileReader.hh"
+#include "data/ControlFile.hh"
+#include "common/Types.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace data {
+
+
+/// \brief Filter file.
+class FilterFileBase : public BinaryAllClustersFileReader
+{
+public:
+ /// \brief Filter file record type definition.
+ struct Record
+ {
+ public:
+ /// \brief Default constructor
+ Record() : data_(0) { }
+
+ /// \brief Constructor
+ /// \param data Data to store
+ Record(char data) : data_(data) { }
+
+ /// \brief Filter data.
+ char data_;
+ };
+
+ /// \brief Construct and open filter file.
+ /// \param inputDir RaW data from the filter file.
+ /// \param ignoreErrors Supress errors opening file and/or reading it.
+ FilterFileBase(
+ const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ bool skipHeader
+ );
+
+ /// \brief Return the file type string. Used for logging message purposes.
+ virtual std::string getFileTypeStr() const { return "Filter"; }
+
+protected:
+ /// \brief Filter V3 header type definition.
+#pragma pack(push, 1)
+ struct Header
+ {
+ public:
+ /// \brief Version field.
+ uint32_t version_;
+
+ /// \brief Clusters count.
+ uint32_t clustersCount_;
+ };
+#pragma pack(pop)
+
+ /// \brief Return the expected header version.
+ virtual uint32_t getHeaderVersion() const { return 3; }
+
+ /// \brief Read the header.
+ virtual bool readHeader();
+
+ /// \brief Validate the header. Throw an exception on failure.
+ virtual bool validateHeader(const Header& header);
+
+ bool skipHeader_;
+};
+
+class FilterFile : public FilterFileBase
+{
+public:
+ FilterFile(const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ bool skipHeader);
+
+ /// \brief Read records from the file to buffer.
+ /// \param targetBuffer Target buffer to read to.
+ /// \param targetSize Maximum number of records to be read.
+ /// \return Number of records read.
+ /// \pre <tt>this->isOpen() == true</tt>
+ virtual std::size_t read(
+ Record *targetBuffer,
+ std::size_t targetSize
+ );
+
+ static bool doesFileExist(const boost::filesystem::path& inputDir,
+ bool aggregateTilesFlag,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ size_t& headerSize,
+ boost::filesystem::path& filePath);
+
+private:
+ /// \brief Return the number of bytes in a record.
+ virtual std::size_t getRecordBytes() const { return sizeof(Record); }
+};
+
+
+} // namespace data
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_DATA_FILTERFILE_HH
+
+
diff --git a/src/cxx/include/data/Instrument.hh b/src/cxx/include/data/Instrument.hh
new file mode 100644
index 0000000..abe1050
--- /dev/null
+++ b/src/cxx/include/data/Instrument.hh
@@ -0,0 +1,75 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Instrument.hh
+ *
+ * \brief Declaration of Instrument file.
+ *
+ * \author Daniel Berard
+ */
+
+#ifndef BCL2FASTQ_DATA_INSTRUMENT_HH
+#define BCL2FASTQ_DATA_INSTRUMENT_HH
+
+#include "data/BclFileReader.hh"
+#include "common/Types.hh"
+
+// include/common/Types.hh:enum class TileAggregationMode { NON_AGGREGATED, AGGREGATED, CBCL };
+
+namespace bcl2fastq {
+
+namespace layout {
+class TileInfo;
+}
+
+namespace data {
+
+class Instrument
+{
+public:
+
+ Instrument(common::TileAggregationMode tileAggregationMode)
+ : tileAggregationMode_(tileAggregationMode)
+ {
+ }
+
+ virtual ~Instrument() { }
+
+ static bool isFilterFileAggregated(
+ common::TileAggregationMode mode)
+ {
+ return (mode == common::TileAggregationMode::AGGREGATED);
+ }
+
+ bool isFilterFileAggregated() const
+ {
+ return isFilterFileAggregated(tileAggregationMode_);
+ }
+
+ static bool isPositionFileAggregated(common::TileAggregationMode mode)
+ {
+ return (mode != common::TileAggregationMode::NON_AGGREGATED);
+ }
+
+ bool isPositionFileAggregated() const
+ {
+ return isPositionFileAggregated(tileAggregationMode_);
+ }
+
+private:
+
+ common::TileAggregationMode tileAggregationMode_;
+
+}; // end class Instrument
+
+} // namespace data
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_DATA_INSTRUMENT_HH
+
+
diff --git a/src/cxx/include/data/InteropFile.hh b/src/cxx/include/data/InteropFile.hh
new file mode 100644
index 0000000..009b960
--- /dev/null
+++ b/src/cxx/include/data/InteropFile.hh
@@ -0,0 +1,135 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file InteropFile.hh
+ *
+ * \brief Declaration of InterOp file.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_DATA_INTEROPFILE_HH
+#define BCL2FASTQ_DATA_INTEROPFILE_HH
+
+
+#include <boost/shared_ptr.hpp>
+#include <boost/filesystem/path.hpp>
+
+#include "io/FileBufWithReopen.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace data {
+
+
+class InteropFileState;
+
+
+/// \brief InterOp file.
+class InteropFile
+{
+public:
+
+ /// \brief Lane number type definition.
+ typedef uint16_t LaneNumber;
+
+ /// \brief Tile number type definition.
+ typedef uint16_t TileNumber;
+
+ /// \brief Read number type definition.
+ typedef uint16_t ReadNumber;
+
+ /// \brief Index name length type definition.
+ typedef uint16_t IndexNameLength;
+
+ /// \brief Clusters count type definition.
+ typedef uint32_t ClustersCount;
+
+ /// \brief Sample id length type definition.
+ typedef uint16_t SampleIdLength;
+
+ /// \brief Project name length type definition.
+ typedef uint16_t ProjectNameLength;
+
+ /// \brief InterOp file state smart pointer type definition.
+ typedef boost::shared_ptr<InteropFileState> InteropFileStatePtr;
+
+public:
+
+ /// \brief Default constructor.
+ InteropFile();
+
+public:
+
+ /// \brief Open InterOp file.
+ /// \param path Path to the InterOp file to be opened.
+ /// \return External state of opened InterOp file.
+ InteropFile::InteropFileStatePtr open(const boost::filesystem::path &path);
+
+public:
+
+ /// \brief Get file path.
+ /// \param fileState InterOp file external state.
+ /// \return File path.
+ boost::filesystem::path getPath(const InteropFileState &fileState) const;
+
+ /// \brief Write bytes from buffer to file.
+ /// \param fileState InterOp file external state.
+ /// \param sourceBuffer Source buffer to read to.
+ /// \param sourceSize Number of bytes to be written.
+ void write(InteropFileState &fileState, const char *sourceBuffer, std::streamsize sourceSize);
+
+private:
+
+ /// \brief Filebuf.
+ io::FileBufWithReopen fileBuf_;
+};
+
+
+/// \brief InterOp file internal state.
+class InteropFileState
+{
+protected:
+
+ /// \brief Constructor.
+ InteropFileState(const boost::filesystem::path& filePath);
+
+ /// \brief Virtual destructor.
+ virtual ~InteropFileState() = 0;
+
+protected:
+
+ /// \brief Write bytes from buffer to file.
+ /// \param os Output stream to write to.
+ /// \param sourceBuffer Source buffer to read to.
+ /// \param sourceSize Number of bytes to be written.
+ virtual void write(std::ostream &os, const char *sourceBuffer, std::streamsize sourceSize) = 0;
+
+private:
+
+ /// \brief File path.
+ boost::filesystem::path path_;
+
+private:
+
+ friend class InteropFile;
+};
+
+
+} // namespace data
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2INTEROP_DATA_INTEROPFILE_HH
+
+
diff --git a/src/cxx/include/data/PositionsFile.hh b/src/cxx/include/data/PositionsFile.hh
new file mode 100644
index 0000000..d617f88
--- /dev/null
+++ b/src/cxx/include/data/PositionsFile.hh
@@ -0,0 +1,136 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file PositionsFile.hh
+ *
+ * \brief Declaration of positions file.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_DATA_POSITIONSFILE_HH
+#define BCL2FASTQ_DATA_POSITIONSFILE_HH
+
+
+#include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+#include <boost/filesystem/path.hpp>
+
+#include "data/FileReader.hh"
+#include "common/Types.hh"
+
+
+namespace bcl2fastq {
+namespace data {
+
+
+/// \brief Positions file.
+class PositionsFile : public virtual FileReaderBase
+{
+public:
+
+ /// \brief Positions file record type definition.
+ struct Record
+ {
+ public:
+
+ /// \brief Cluster coordinate type definition.
+ typedef unsigned int ClusterCoordinate;
+
+ /// \brief X-coordinate.
+ ClusterCoordinate x_;
+
+ /// \brief y-coordinate.
+ ClusterCoordinate y_;
+ };
+
+ /// \brief Read records from the file to buffer.
+ /// \param targetBuffer Target buffer to read to.
+ /// \param targetSize Maximum number of records to be read.
+ /// \return Number of records read.
+ /// \pre <tt>this->isOpen() == true</tt>
+ virtual std::size_t read(
+ std::vector<data::PositionsFile::Record>& targetBuffer,
+ std::size_t targetSize
+ ) = 0;
+
+protected:
+ PositionsFile(const common::RawDataBuffer& data,
+ bool ignoreErrors = false
+ );
+};
+
+
+
+
+/// \brief Factory class to instantiate the appropriate PositionsFile type
+class PositionsFileFactory : private boost::noncopyable
+{
+public:
+ /// \brief Return true if the positions file exists.
+ /// \param intensitiesDir Path to intensities directory.
+ /// \param aggregateTilesFlag All tiles aggregated into single file flag.
+ /// \param isPatternedFlowcell True if a patterned flowcell was used.
+ /// \param laneNumber Lane number.
+ /// \param tileNumber Tile number.
+ static bool doesFileExist(const boost::filesystem::path& intensitiesDir,
+ bool aggregateTilesFlag,
+ bool isPatternedFlowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ size_t& headerSize,
+ boost::filesystem::path& posFilePath);
+
+ static std::streamsize read(const common::RawDataBuffer& inputData,
+ bool ignoreErrors,
+ bool skipHeader,
+ std::vector<data::PositionsFile::Record>& targetBuffer,
+ std::size_t targetSize);
+
+private:
+ /// \brief Check if the binary file path exists for the specified parameters.
+ /// \param intensitiesDir Path to intensities directory.
+ /// \param aggregateTilesFlag All tiles aggregated into single file flag.
+ /// \param isPatternedFlowcell True if a patterned flowcell was used.
+ /// \param laneNumber Lane number.
+ /// \param tileNumber Tile number.
+ /// \param fileExtension File extension.
+ /// \param positionsFilePath Output param for the file path.
+ /// \return true if the file exists.
+ static bool binaryFilePathExists(
+ const boost::filesystem::path& intensitiesDir,
+ bool aggregateTilesFlag,
+ bool isPatternedFlowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ const boost::filesystem::path& fileExtension,
+ boost::filesystem::path& positionsFilePath
+ );
+
+ /// \brief Check if the *_pos.txt file path exists for the specified parameters
+ /// \param intensitiesDir Path to intensities directory.
+ /// \param laneNumber Lane number.
+ /// \param tileNumber Tile number.
+ /// \param positionsFilePath Output param for the file path.
+ /// \return true if the file exists.
+ static bool posFilePathExists(
+ const boost::filesystem::path& intensitiesDir,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ boost::filesystem::path& positionsFilePath
+ );
+};
+
+} // namespace data
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_DATA_POSITIONSFILE_HH
+
diff --git a/src/cxx/include/data/RawBclBuffer.hh b/src/cxx/include/data/RawBclBuffer.hh
new file mode 100644
index 0000000..668757a
--- /dev/null
+++ b/src/cxx/include/data/RawBclBuffer.hh
@@ -0,0 +1,208 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BclReader.hh
+ *
+ * \brief Declaration of raw BCL buffer.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_DATA_RAW_BCL_BUFFER_HH
+#define BCL2FASTQ_DATA_RAW_BCL_BUFFER_HH
+
+
+#include "common/Types.hh"
+#include "layout/LaneInfo.hh"
+#include "io/GzipDecompressor.hh"
+#include "data/BclFile.hh"
+
+#include <utility>
+#include <memory>
+#include <vector>
+#include <atomic>
+#include <mutex>
+#include <condition_variable>
+
+namespace bcl2fastq {
+namespace data {
+
+struct PerCycleData
+{
+ /// \brief Return the number of clusters per byte
+ common::ClustersCount getNumClustersPerByte() const { return (8 / (numBitsPerQscore_ + 2)); }
+
+ /// \brief Return the number of clusters
+ common::ClustersCount getNumClusters() const { return bcls_.size() * getNumClustersPerByte(); }
+
+ /// \brief BCL cycles data container type definition.
+ typedef std::vector<char> BclsContainer;
+
+ PerCycleData() : bcls_(), includeNonPf_(true), numBitsPerQscore_(6), remappedQscores_() { }
+
+ /// \brief Buffers for cycles data.
+ common::RawDataBuffer bcls_;
+
+ /// \brief Indicates which cycles include non-pf clusters.
+ bool includeNonPf_;
+
+ /// \brief Number of bits per qscore for each cycle.
+ uint8_t numBitsPerQscore_;
+
+ /// \brief Remapped qscores for each cycle.
+ std::vector<uint32_t> remappedQscores_;
+
+ /// \brief Uncompressed block size.
+ uint32_t uncompressedBlockSize_;
+};
+
+struct RawBclBuffer
+{
+ RawBclBuffer()
+ : cycleData_()
+ , gzipDecompressors_()
+ , controls_()
+ , filters_()
+ , positions_()
+ , tileOffsets_()
+ , tileInfo_()
+ , groupNumber_(0)
+ , uncompressedBclOffset_(0)
+ {
+ }
+
+ uint32_t getGroupNumber() const { return groupNumber_; }
+ void setGroupNumber(uint32_t groupNumber) { groupNumber_ = groupNumber; }
+
+ bool hasCycleData() const {
+ for (PerCycleData perCycleData : cycleData_) {
+ if (!(perCycleData.bcls_.empty())) return true;
+ }
+ return false;
+ }
+
+ // One for each cycle
+ std::vector<PerCycleData> cycleData_;
+ std::vector<io::GzipDecompressor> gzipDecompressors_;
+
+ common::RawDataBuffer controls_;
+ common::RawDataBuffer filters_;
+
+ // Could be shared by all tiles if it is a patterned flowcell
+ std::shared_ptr<common::RawDataBuffer> positions_;
+
+ std::vector<std::pair<size_t, common::TileNumber>> tileOffsets_;
+
+ /// \brief Tile meta data.
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfo_;
+
+ uint32_t groupNumber_;
+
+ uint32_t uncompressedBclOffset_;
+};
+
+typedef std::vector<RawBclBuffer> RawBclBufferVec;
+
+class RawBclBufferGroup : private boost::noncopyable
+{
+public:
+ RawBclBufferGroup(size_t numTilesPerBuffer, size_t numCycles) : outputBuffer_(numTilesPerBuffer), groupNumber_(0), prevBuffer_(), prevBclFiles_(numCycles), isFinished_(false), mut_(), cv_() { }
+
+ uint32_t getGroupNumber() const { return groupNumber_; }
+ void setGroupNumber(uint32_t groupNumber) { isFinished_ = false; groupNumber_ = groupNumber; }
+
+ size_t size() const { return outputBuffer_.size(); }
+
+ RawBclBufferVec::iterator begin() { return outputBuffer_.begin(); }
+ RawBclBufferVec::iterator end() { return outputBuffer_.end(); }
+
+ RawBclBufferVec::const_iterator begin() const { return outputBuffer_.begin(); }
+ RawBclBufferVec::const_iterator end() const { return outputBuffer_.end(); }
+
+ bool hasCycleData() {
+ for (RawBclBuffer &rawBclBuffer : outputBuffer_) {
+ if (rawBclBuffer.hasCycleData()) return true;
+ }
+ return false;
+ }
+
+ bool hasCycleData(common::CycleNumber cycleIndex) {
+ for (RawBclBuffer &rawBclBuffer : outputBuffer_) {
+ PerCycleData &perCycleData = rawBclBuffer.cycleData_[cycleIndex];
+ if (!(perCycleData.bcls_.empty())) return true;
+ }
+ return false;
+ }
+
+ // This will clear BCL cluster data for the specified cycle and current tile.
+ template<class FileReaderType>
+ void clearClusters(common::CycleNumber cycleIndex,
+ std::shared_ptr<FileReaderType> bclFile)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::TRACE)
+ << "Clearing BCL data for cycle " << (1+cycleIndex) << std::endl;
+
+ for (data::RawBclBuffer& rawBclBuffer : outputBuffer_)
+ {
+ rawBclBuffer.cycleData_[cycleIndex].uncompressedBlockSize_ = 0;
+ rawBclBuffer.cycleData_[cycleIndex].bcls_.resize(0);
+ if (bclFile)
+ {
+ rawBclBuffer.cycleData_[cycleIndex].bcls_.path_ = bclFile->getPath();
+ }
+ }
+ }
+
+ void resize(size_t newSize) { outputBuffer_.resize(newSize); }
+
+ void reset()
+ {
+ for (auto& buffer : outputBuffer_)
+ {
+ for (auto& decompressor : buffer.gzipDecompressors_)
+ {
+ decompressor.clear();
+ }
+ }
+
+ prevBuffer_ = std::shared_ptr<RawBclBufferGroup>();
+
+ for (auto& bclFile : prevBclFiles_)
+ {
+ bclFile = std::shared_ptr<data::BclFile>();
+ }
+ }
+
+ void setPrevBuffer(std::shared_ptr<RawBclBufferGroup>& prevBuffer) { prevBuffer_ = prevBuffer; }
+ std::shared_ptr<RawBclBufferGroup> getPrevBuffer() { return prevBuffer_; }
+
+ std::shared_ptr<data::BclFile> getPrevBclFileForCycle(common::CycleNumber cycleNumber) { return prevBclFiles_[cycleNumber]; }
+ void setPrevBclFileForCycle(common::CycleNumber cycleNumber, std::shared_ptr<data::BclFile>& bclFile) { prevBclFiles_[cycleNumber] = bclFile; }
+
+ void setFinished() { isFinished_ = true; cv_.notify_one(); }
+ void waitForFinished() { std::unique_lock<std::mutex> lock(mut_); cv_.wait(lock, [this] { return isFinished_ == true; }); }
+
+private:
+ RawBclBufferVec outputBuffer_;
+ uint32_t groupNumber_;
+
+ std::shared_ptr<RawBclBufferGroup> prevBuffer_;
+ std::vector<std::shared_ptr<data::BclFile>> prevBclFiles_;
+
+ std::atomic<bool> isFinished_;
+ std::mutex mut_;
+ std::condition_variable cv_;
+};
+
+typedef std::shared_ptr<RawBclBufferGroup> RawBclBufferGroupPtr;
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_DATA_RAW_BCL_BUFFER_HH
+
diff --git a/src/cxx/include/data/TileBclFileReader.hh b/src/cxx/include/data/TileBclFileReader.hh
new file mode 100644
index 0000000..0bbc61e
--- /dev/null
+++ b/src/cxx/include/data/TileBclFileReader.hh
@@ -0,0 +1,49 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file TileBclFileReader.hh
+ *
+ * \brief Declaration of BCL reader for a single tile.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_DATA_TILE_BCL_FILE_READER_HH
+#define BCL2FASTQ_DATA_TILE_BCL_FILE_READER_HH
+
+#include "data/BclFileReader.hh"
+#include "data/RawBclBuffer.hh"
+#include "io/SyncFile.hh"
+
+#include <boost/filesystem/path.hpp>
+#include <boost/noncopyable.hpp>
+
+#include <memory>
+
+namespace bcl2fastq {
+namespace data {
+
+
+class TileBclFileReader : public BclFileReaderT<io::FileReaderBase>
+{
+public:
+ TileBclFileReader(const::boost::filesystem::path& inputDir,
+ const layout::LaneInfo& laneInfo,
+ common::CycleNumber cycleNumber,
+ size_t cycleIndex,
+ bool ignoreMissingBcls,
+ std::shared_ptr<io::FileReaderBase> bclFile);
+
+ virtual bool read(RawBclBufferGroup& outputBuffer);
+};
+
+} // namespace data
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_DATA_TILE_BCL_FILE_READER_HH
+
diff --git a/src/cxx/include/io/FileBufWithReopen.hh b/src/cxx/include/io/FileBufWithReopen.hh
new file mode 100644
index 0000000..b0c3ecf
--- /dev/null
+++ b/src/cxx/include/io/FileBufWithReopen.hh
@@ -0,0 +1,170 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FileBufWithReopen.hh
+ *
+ * \brief Declaration of the file buffer with reopen.
+ *
+ * \author Roman Petorvski
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_IO_FILEBUFWITHREOPEN_HH
+#define BCL2FASTQ_IO_FILEBUFWITHREOPEN_HH
+
+
+#include <ios>
+#include <fstream>
+
+#include <cstdio>
+
+#include <boost/filesystem/path.hpp>
+#include <boost/noncopyable.hpp>
+
+
+namespace bcl2fastq {
+
+
+namespace io {
+
+
+/// \brief Same as std::filebuf but has reopen which is useful in parts of the code that are not allowed to do resource allocations.
+template<typename CharT, typename Traits = std::char_traits<CharT> >
+class BasicFileBufWithReopen : public std::basic_filebuf<CharT, Traits>, private boost::noncopyable
+{
+public:
+
+ /// \brief Fadvise flags.
+ struct FadviseFlags
+ {
+ enum value_type
+ {
+ NORMAL = 0, ///< No advice (default). Sets readahead window to default size.
+ SEQUENTIAL = 1, ///< Sequential access. Sets readahead widnow to double the default size.
+ RANDOM = 2, ///< Random access. Disables readahead window.
+ NOREUSE = 4, ///< Same as @c WILLNEED until kernel 2.6.18. No-op since then.
+ WILLNEED = 8, ///< Data will be acessed in near future.
+ DONTNEED = 16, ///< Data will not be accessed in near future.
+ SEQUENTIAL_ONCE = SEQUENTIAL|DONTNEED, ///< Sequential access with caching.
+ SEQUENTIAL_OFTEN = SEQUENTIAL|WILLNEED ///< Sequential access without caching.
+ };
+ };
+
+public:
+
+ /// \brief Constructor.
+ /// \param mode Access mode to be used.
+ explicit BasicFileBufWithReopen(std::ios_base::openmode mode);
+
+private:
+
+ /// \brief Copy assignment operator.
+ /// \param rhs Right-hand-side parameter.
+ BasicFileBufWithReopen & operator=(const BasicFileBufWithReopen &rhs);
+
+public:
+
+ /// \brief (Re)opens specified file.
+ /// \param path File to be opened.
+ /// \param fadvise Fadvise flags.
+ /// \return Pointer to this.
+ /// \note Access mode specified during construction of the object
+ /// will be used.
+ /// \note Call to this method supposedly does not fail due to memory
+ /// allocation failure as it does not do any.
+ BasicFileBufWithReopen* reopen(const boost::filesystem::path& path, typename FadviseFlags::value_type fadvise = FadviseFlags::NORMAL);
+
+ /// \brief Set access mode.
+ /// \param mode Access mode to be set.
+ void setMode(std::ios_base::openmode mode);
+
+ /// \brief Get access mode.
+ /// \return Access mode.
+ std::ios_base::openmode getMode() const;
+
+ /// \brief Flush pending data.
+ void flush();
+
+ /// \brief Reset.
+ void reset();
+
+private:
+
+ /// \brief Reserve a file handle.
+ /// \retval true Success.
+ /// \retval false Failure.
+ bool reserve();
+
+private:
+
+#ifndef WIN32
+ /// \brief Abstraction of implementation dependent external buffer of underlying file stream.
+ class File
+ {
+ public:
+
+ /// \brief File type definition.
+ typedef typename BasicFileBufWithReopen<CharT, Traits>::__file_type file_type;
+
+ public:
+
+ /// \brief Constructor.
+ /// \param file Implementation dependent buffer of underlying file stream.
+ explicit File(file_type &file);
+
+ public:
+
+ /// \brief Get C-file.
+ /// \return Extracted stdio's @c FILE structure.
+ std::FILE * getCFile();
+
+ private:
+
+ /// \brief Reference to implementation dependent buffer of underlying file stream.
+ file_type &file_;
+ };
+#endif
+
+private:
+
+ /// \brief File access mode.
+ const std::ios_base::openmode mode_;
+
+#ifndef WIN32
+ /// \brief Implementation dependent buffer of underlying file stream.
+ File file_;
+#endif
+};
+
+
+/// \brief File buffer with reopen using 1-byte characters.
+typedef BasicFileBufWithReopen<char> FileBufWithReopen;
+
+
+/// \brief Read bytes from file buffer.
+/// \param fileBuf File buffer to read from.
+/// \param targetBuffer Target buffer to read to.
+/// \param targetSize Maximum number of bytes to be read.
+/// \return Number of bytes read.
+template<typename CharT>
+std::streamsize read(BasicFileBufWithReopen<CharT> &fileBuf, char *targetBuffer, std::streamsize targetSize);
+
+
+} // namespace io
+
+
+} // namespace bcl2fastq
+
+
+#include "io/FileBufWithReopen.hpp"
+
+
+#endif // BCL2FASTQ_IO_FILEBUFWITHREOPEN_HH
+
+
diff --git a/src/cxx/include/io/FileBufWithReopen.hpp b/src/cxx/include/io/FileBufWithReopen.hpp
new file mode 100644
index 0000000..ccfda77
--- /dev/null
+++ b/src/cxx/include/io/FileBufWithReopen.hpp
@@ -0,0 +1,258 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FileBufWithReopen.hpp
+ *
+ * \brief Implementation of the file buffer with reopen.
+ *
+ * \author Roman Petorvski
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_IO_FILEBUFWITHREOPEN_HPP
+#define BCL2FASTQ_IO_FILEBUFWITHREOPEN_HPP
+
+
+#include <iostream>
+#include <fcntl.h>
+
+#include "common/Exceptions.hh"
+#include "common/Debug.hh"
+#include "common/Logger.hh"
+#include "io/Utility.hh"
+#include "io/FileBufWithReopen.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace io {
+
+
+template<typename CharT, typename Traits>
+BasicFileBufWithReopen<CharT, Traits>::BasicFileBufWithReopen(std::ios_base::openmode mode)
+: std::basic_filebuf<CharT,Traits>()
+, mode_(mode)
+#ifndef WIN32
+, file_(std::basic_filebuf<CharT, Traits>::_M_file) // implementation dependent
+#endif
+{
+ if (!this->reserve())
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errno, "Failed to allocate a file handle"));
+ }
+}
+
+template<typename CharT, typename Traits>
+BasicFileBufWithReopen<CharT, Traits>* BasicFileBufWithReopen<CharT, Traits>::reopen(const boost::filesystem::path& path, typename FadviseFlags::value_type fadvise)
+{
+ const char * const openMode = iosFlagsToStdioMode(mode_);
+ BCL2FASTQ_ASSERT_MSG(openMode, "Combination of open mode flags is invalid");
+ //BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Reopening '" << path << "' with mode '" << openMode << "'" << std::endl;
+
+#ifndef WIN32
+ // I haven't figured out a way to reopen a file buffer on windows...
+ BCL2FASTQ_ASSERT_MSG(this->is_open(), "The file must be already open for reopen to be possible");
+#endif
+
+ if ((mode_ & std::ios_base::out))
+ {
+ //flush any pending data before reopening a (usually) different file, reset eof flag
+ int syncResult = this->pubsync();
+ BCL2FASTQ_ASSERT_MSG(syncResult == 0, "pubsync failed: " << errno << ": " << strerror(errno));
+ }
+#ifndef WIN32
+ /// \todo Refactoring: Spaghetti design: Move repeating code that calls posix_fadvise to separate function.
+
+ if (
+ (fadvise & FadviseFlags::NOREUSE)
+ &&
+ ::posix_fadvise(::fileno(file_.getCFile()), 0, 0, POSIX_FADV_NOREUSE)
+ &&
+ errno // reqired since POSIX_FADV_NOREUSE fails with errno 0 on /dev/null
+ ) {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING)
+ << "posix_fadvise failed for POSIX_FADV_NOREUSE: "
+ << errno << ": " << strerror(errno) << ": "
+ << path << std::endl
+ ;
+ }
+
+ if (
+ (fadvise & FadviseFlags::WILLNEED)
+ &&
+ posix_fadvise(::fileno(file_.getCFile()), 0, 0, POSIX_FADV_WILLNEED)
+ &&
+ errno // reqired since POSIX_FADV_WILLNEED fails with errno 0 on /dev/null
+ ) {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING)
+ << "posix_fadvise failed for POSIX_FADV_WILLNEED: "
+ << errno << ": " << strerror(errno) << ": "
+ << path << std::endl
+ ;
+ }
+
+ if (
+ (fadvise & FadviseFlags::DONTNEED)
+ &&
+ posix_fadvise(::fileno(file_.getCFile()), 0, 0, POSIX_FADV_DONTNEED)
+ &&
+ errno // reqired since POSIX_FADV_DONTNEED fails with errno 0 on /dev/null
+ ) {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING)
+ << "posix_fadvise failed for POSIX_FADV_DONTNEED: "
+ << errno << ": " << strerror(errno) << ": "
+ << path << std::endl
+ ;
+ }
+
+ errno = 0;
+ std::FILE *result = std::freopen(path.string().c_str(), openMode, file_.getCFile());
+ int errnum = errno;
+ if (result == NULL)
+ {
+ this->close();
+ errno = errnum;
+ return NULL;
+ }
+ BCL2FASTQ_ASSERT_MSG(file_.getCFile() == result, "According to specs, returned pointer must be the same as the one passed to freopen");
+#else
+ errno = 0;
+ if (this->is_open())
+ {
+ close();
+ }
+ open(path.string().c_str(), mode_);
+#endif
+
+ if (!(mode_ & std::ios_base::app))
+ {
+ if (this->seekoff(0, std::ios_base::beg, mode_) != std::streampos(0))
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errno, path.string()));
+ }
+ }
+#ifndef WIN32
+ if ((fadvise & FadviseFlags::SEQUENTIAL) && posix_fadvise(::fileno(result), 0, 0, POSIX_FADV_SEQUENTIAL))
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errno, path.string()));
+ }
+
+ if ((fadvise & FadviseFlags::RANDOM) && posix_fadvise(::fileno(result), 0, 0, POSIX_FADV_RANDOM))
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errno, path.string()));
+ }
+#endif
+ return this;
+}
+
+template<typename CharT, typename Traits>
+std::ios_base::openmode BasicFileBufWithReopen<CharT, Traits>::getMode() const
+{
+ return mode_;
+}
+
+template<typename CharT, typename Traits>
+void BasicFileBufWithReopen<CharT, Traits>::setMode(std::ios_base::openmode mode)
+{
+ this->reset();
+ mode_ = mode;
+}
+
+template<typename CharT, typename Traits>
+void BasicFileBufWithReopen<CharT, Traits>::flush()
+{
+ /// \todo Refactoring: Make some sensible error reporting.
+
+ //flush any pending data before reopening a (usually) different file
+ int result = this->pubsync();
+ BCL2FASTQ_ASSERT_MSG(result == 0, "TODO: make some sensible error reporting...");
+}
+
+
+namespace detail {
+
+
+/// \brief Get file that will exist on all systems.
+/// \return String containing path to a file which always exists.
+/// \todo Portability: Make this function Windows-compatible.
+static const char * fileThatAlwaysExists()
+{
+#ifdef WIN32
+ return "NUL";
+#else
+ return "/dev/null";
+#endif
+}
+
+
+} // namespace detail
+
+
+template<typename CharT, typename Traits>
+void BasicFileBufWithReopen<CharT, Traits>::reset()
+{
+ if (this->is_open())
+ {
+ this->reopen(detail::fileThatAlwaysExists());
+ }
+ else
+ {
+ this->reserve();
+ }
+ BCL2FASTQ_ASSERT_MSG(this->is_open(), "Failed to reset file buffer");
+}
+
+template<typename CharT, typename Traits>
+bool BasicFileBufWithReopen<CharT, Traits>::reserve()
+{
+ return !!this->open(detail::fileThatAlwaysExists(), mode_);
+}
+
+
+#ifndef WIN32
+template<typename CharT, typename Traits>
+BasicFileBufWithReopen<CharT, Traits>::File::File(file_type &file)
+: file_(file)
+{
+}
+
+template<typename CharT, typename Traits>
+std::FILE * BasicFileBufWithReopen<CharT, Traits>::File::getCFile()
+{
+ return file_.file();
+}
+#endif
+
+template<typename CharT>
+std::streamsize read(BasicFileBufWithReopen<CharT> &fileBuf, char *targetBuffer, std::streamsize targetSize)
+{
+ std::basic_istream<CharT> is(&fileBuf);
+
+ is.read(targetBuffer, targetSize);
+ if (is)
+ {
+ return targetSize;
+ }
+ else
+ {
+ return is.gcount();
+ }
+}
+
+
+} // namespace io
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_IO_FILEBUFWITHREOPEN_HPP
+
+
diff --git a/src/cxx/include/io/GzipCompressor.hh b/src/cxx/include/io/GzipCompressor.hh
new file mode 100644
index 0000000..46ad9cf
--- /dev/null
+++ b/src/cxx/include/io/GzipCompressor.hh
@@ -0,0 +1,116 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file GzipCompressor.hh
+ *
+ * \brief Declaration of Gzip Compressor.
+ *
+ * \author Roman Petrovski
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_IO_GZIP_COMPRESSOR_HH
+#define BCL2FASTQ_IO_GZIP_COMPRESSOR_HH
+
+#include <vector>
+#include <boost/noncopyable.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+
+namespace bcl2fastq
+{
+namespace io
+{
+
+namespace bios=boost::iostreams;
+
+class GzipCompressor : private boost::noncopyable
+{
+public:
+ GzipCompressor(std::vector<char>& outputBuffer,
+ bool useBgzf,
+ const bios::gzip_params& gzipParams = bios::gzip::default_compression);
+
+ ~GzipCompressor();
+
+ std::streamsize write(const char* s, std::streamsize n);
+
+ void flush();
+
+ static const int SUBFIELD_LENGTH = 2; // this needs to be 2 to comply with the BGZF specs
+
+ static const unsigned MAX_BGZF_BLOCK_SIZE = 0xFFFF;
+
+private:
+#pragma pack(push, 1)
+ struct Header
+ {
+ struct XFIELD
+ {
+ XFIELD(size_t compressedSize);
+
+ unsigned char XLEN[2];
+
+ unsigned char SI1;
+ unsigned char SI2;
+ unsigned char SLEN[2];
+ unsigned char BSIZE[SUBFIELD_LENGTH];
+
+ unsigned getXLEN() const {return unsigned(XLEN[0]) + XLEN[1] * 256;}
+ unsigned getBSIZE() const {return unsigned(BSIZE[0]) + BSIZE[1] * 256;}
+
+ };
+
+ unsigned char ID1;
+ unsigned char ID2;
+ unsigned char CM;
+ unsigned char FLG;
+ unsigned char MTIME[4];
+ unsigned char XFL;
+ unsigned char OS;
+
+ XFIELD xfield;
+ };
+#pragma pack(pop)
+
+public:
+ static size_t getHeaderSize() { return sizeof(Header); }
+
+ static unsigned getBlockSize(char* buffer, bool failQuietly = false);
+
+private:
+ void prepareHeader();
+
+ void rewriteHeader();
+
+ // unverified number of bytes the compressor would add to data with compression level 0
+ static const size_t gzip_junk = 41;
+ // bgzf cannot handle blocks over 0xFFFF bytes long. assume 1/1 compression ratio for input data
+ static const unsigned bgzf_buffer_size_ = MAX_BGZF_BLOCK_SIZE;
+ static const unsigned short max_uncompressed_per_block_ = bgzf_buffer_size_ - gzip_junk;
+
+ std::vector<char>& outputBuffer_;
+ std::vector<char> internalBuffer_;
+ bios::back_insert_device< std::vector<char> > sink_;
+ boost::iostreams::gzip_compressor compressor_;
+
+ // as it is difficult to predict the size of compressed data, the buffering is based on
+ // the amount of uncompressed data consumed. The assumption is that compressed data
+ // will always require no more space than the uncompressed.
+ size_t uncompressed_in_;
+
+ size_t currentBlockBegin_;
+
+ bool useBgzf_;
+};
+
+} // namespace io
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_IO_GZIP_COMPRESSOR_HH
diff --git a/src/cxx/include/io/GzipDecompressor.hh b/src/cxx/include/io/GzipDecompressor.hh
new file mode 100644
index 0000000..ef35d00
--- /dev/null
+++ b/src/cxx/include/io/GzipDecompressor.hh
@@ -0,0 +1,191 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file GzipDecompressor.hh
+ *
+ * \brief Declaration of GZip decompression filter.
+ *
+ * \author Roman Petrovski
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_IO_GZIPDECOMPRESSOR_HH
+#define BCL2FASTQ_IO_GZIPDECOMPRESSOR_HH
+
+
+#include <iosfwd>
+#include <vector>
+
+#include <cstddef>
+
+#include <boost/iostreams/concepts.hpp>
+
+#include <zlib.h>
+
+#include "common/Exceptions.hh"
+#include "common/StaticMemPool.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace io {
+
+
+struct z_stream_serialization : public z_stream
+{
+ z_stream_serialization(const z_stream &that) : z_stream(that){}
+};
+
+
+/// \brief Exception for ZLib errors.
+class ZlibError : public common::Exception, public std::runtime_error
+{
+public:
+
+ /// \brief Constructor.
+ /// \param errorNumber Error number.
+ /// \param zstream ZLib stream.
+ ZlibError(int errorNumber, z_stream &zstream, const char *msg = 0);
+
+ /// \brief Copy constructor.
+ /// \param that Other exception object to copy from.
+ ZlibError(const ZlibError &that);
+
+public:
+
+ /// \brief Get error message.
+ /// \par Exception guarantee:
+ /// no-throw
+ virtual const char * what() const throw();
+};
+
+
+/// \brief Same as boost::gzip_decompressor, but ensures zlib buffer is allocated during construction.
+class GzipDecompressor : public boost::iostreams::multichar_input_filter
+{
+public:
+
+ /// \brief Memory pool page count.
+ static const std::size_t PageCount = 3;
+
+ /// \brief Memory pool page size.
+ static const std::size_t PageSize = 65536;
+
+public:
+
+ /// \brief Intermediate buffer type definition.
+ typedef std::vector<char> buffer_type;
+
+public:
+
+ /// \brief Constructor.
+ /// \param bufferSize Intermediate buffer size.
+ explicit GzipDecompressor(buffer_type::size_type bufferSize = PageSize);
+
+ /// \brief Copy constructor.
+ /// \param that Other instance of GZip decompressor to copy from.
+ GzipDecompressor(const GzipDecompressor &that);
+
+ /// \brief Copy assignment operator.
+ /// \param rhs Right-hand-side parameter.
+ GzipDecompressor & operator=(const GzipDecompressor &rhs);
+
+ /// \brief Destructor.
+ ~GzipDecompressor();
+
+public:
+
+ /// \brief Process closure notification.
+ template<typename Source>
+ void close(Source &);
+
+ /// \brief Completely clears stream state. Resets input and output buffer information.
+ void clear();
+
+ /// \brief Reset the stream state so that new compressed block can be processed. Does not touch input or output positions.
+ void reset();
+
+ /// \brief Clear internal buffer.
+ void flush();
+
+public:
+
+ /// \brief Read filtered characters.
+ /// \param compressedStream Source to read from.
+ /// \param targetBuffer Target buffer to read to.
+ /// \param targetSize Maximum number of characters to be read.
+ /// \return Number of characters read.
+ template<typename Source>
+ std::streamsize read(Source &compressedStream, char *targetBuffer, std::streamsize targetSize);
+
+private:
+
+ /// \brief Read filtered characters.
+ /// \param compressedStream Source to read from.
+ /// \param targetBuffer Target buffer to read to.
+ /// \param targetSize Maximum number of characters to be read.
+ /// \return Number of characters read.
+ template<typename Source>
+ std::streamsize readInternal(Source &compressedStream, char *targetBuffer, std::streamsize targetSize);
+
+ /// \brief Initializes decompression stream.
+ void init();
+
+ /// \brief Process pending bytes from intermediate buffer.
+ void processPendingBytes(bool endOfData);
+
+private:
+
+ /// \brief Memory pool type definition.
+ typedef common::StaticMemPool<PageSize, PageCount> mempool_type;
+
+private:
+
+ /// \brief Meomory allocation routine for ZLib.
+ /// \param opaque Private data object.
+ /// \param items Number of items.
+ /// \param size Size of an item.
+ /// \return Pointer to allocated memory, or @c Z_NULL in case of failure.
+ static voidpf zalloc OF((voidpf opaque, uInt items, uInt size));
+
+ /// \brief Memory deallocation routine for ZLib.
+ /// \param opaque Private data object.
+ /// \param address Pointer to memory previously allocated by @c zalloc.
+ static void zfree OF((voidpf opaque, voidpf address));
+
+private:
+
+ /// \brief ZLib stream.
+ z_stream zstream_;
+
+ /// \brief Memory pool for ZLib.
+ mempool_type memPool_;
+
+ /// \brief Number of bytes in intermediate buffer.
+ std::streamsize pendingBytes_;
+
+ /// \brief Intermediate buffer.
+ buffer_type buffer_;
+};
+
+
+} // namespace io
+
+
+} // namespace bcl2fastq
+
+
+#include "io/GzipDecompressor.hpp"
+
+
+#endif // BCL2FASTQ_IO_GZIPDECOMPRESSOR_HH
+
+
diff --git a/src/cxx/include/io/GzipDecompressor.hpp b/src/cxx/include/io/GzipDecompressor.hpp
new file mode 100644
index 0000000..1f66298
--- /dev/null
+++ b/src/cxx/include/io/GzipDecompressor.hpp
@@ -0,0 +1,129 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file GzipDecompressor.hpp
+ *
+ * \brief Implementation of GZip decompression filter.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_IO_GZIPDECOMPRESSOR_HPP
+#define BCL2FASTQ_IO_GZIPDECOMPRESSOR_HPP
+
+
+#include <boost/iostreams/read.hpp>
+#include <boost/foreach.hpp>
+
+#include "common/Debug.hh"
+#include "common/Logger.hh"
+#include "io/GzipDecompressor.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace io {
+
+
+template<typename Source>
+void GzipDecompressor::close(Source &)
+{
+ this->reset();
+}
+
+template<typename Source>
+std::streamsize GzipDecompressor::read(Source &compressedStream, char *targetBuffer, std::streamsize targetSize)
+{
+ std::streamsize bytesRead = readInternal(compressedStream, targetBuffer, targetSize);
+ while ((bytesRead < targetSize) && !compressedStream.eof() && bytesRead != -1)
+ {
+ std::streamsize bytesRead2 = readInternal(compressedStream, targetBuffer+bytesRead, targetSize-bytesRead);
+ if (bytesRead2 != -1)
+ {
+ bytesRead += bytesRead2;
+ } else {
+ break;
+ }
+ }
+
+ return bytesRead;
+}
+
+template<typename Source>
+std::streamsize GzipDecompressor::readInternal(Source &compressedStream, char *targetBuffer, std::streamsize targetSize)
+{
+ if (targetSize == 0)
+ {
+ return 0;
+ }
+
+ zstream_.next_out = reinterpret_cast<Bytef *>(targetBuffer);
+ zstream_.avail_out = static_cast<uInt>(targetSize);
+ this->processPendingBytes(!compressedStream.good());
+
+ if (zstream_.avail_out)
+ {
+ if (compressedStream.good())
+ {
+ std::streamsize bytesToRead = (buffer_.end() - buffer_.begin()) - pendingBytes_;
+ std::streamsize result = 0;
+ if (bytesToRead > 0)
+ {
+ result = boost::iostreams::read(compressedStream, &buffer_.begin()[pendingBytes_], bytesToRead);
+ }
+
+ if (result != -1 /* EOF */)
+ {
+ pendingBytes_ += result;
+ }
+ }
+ if (!compressedStream.good() && !compressedStream.eof())
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errno, "Failed to read compressed data"));
+ }
+
+ this->processPendingBytes(!compressedStream.good());
+ }
+
+ std::streamsize ret = targetSize - zstream_.avail_out;
+
+ if (ret == 0)
+ {
+ if (pendingBytes_ > 0)
+ {
+ this->reset();
+ this->processPendingBytes(!compressedStream.good());
+ ret = targetSize - zstream_.avail_out;
+ } else if (!compressedStream.good()) {
+ BOOST_THROW_EXCEPTION(common::IoError(errno, "Unexpected end of file"));
+ }
+
+ //BCL2FASTQ_ASSERT_MSG( pendingBytes_ == 0,
+ // "When no bytes come out of decompressor and the input stream is all finished,"
+ // " expecting the pendingBytes_ to be 0. Actual: " << pendingBytes_ );
+
+ //BCL2FASTQ_LOG(common::LogLevel::TRACE) << "GzipDecompressor::read: finished" << std::endl;
+ //return -1;
+ }
+
+ return ret ? ret : -1;
+}
+
+
+} // namespace io
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_IO_GZIPDECOMPRESSOR_HPP
+
+
diff --git a/src/cxx/include/io/SyncFile.hh b/src/cxx/include/io/SyncFile.hh
new file mode 100644
index 0000000..f218085
--- /dev/null
+++ b/src/cxx/include/io/SyncFile.hh
@@ -0,0 +1,265 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file SyncFile.hh
+ *
+ * \brief Declaration of Synchronized file.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_IO_SYNC_FILE_HH
+#define BCL2FASTQ_IO_SYNC_FILE_HH
+
+#include "io/FileBufWithReopen.hh"
+#include "io/GzipCompressor.hh"
+#include "common/Types.hh"
+
+#include <boost/format.hpp>
+#include <boost/filesystem/path.hpp>
+#include <boost/noncopyable.hpp>
+#include <condition_variable>
+#include <atomic>
+#include <mutex>
+#include <string>
+#include <vector>
+
+namespace bcl2fastq {
+namespace io {
+
+// This interface class is here so that most code can use a pointer to this
+// base class rather than use a class that touches the file system.
+// This makes writing unit tests easier.
+class FileReaderBase : private boost::noncopyable
+{
+public:
+ FileReaderBase();
+
+ FileReaderBase(const boost::filesystem::path& path,
+ bool ignoreErrors);
+
+ virtual ~FileReaderBase() { }
+
+ /// \brief Return the file type string. Used for logging message purposes.
+ virtual std::string getFileTypeStr() const { return "Unprocessed"; }
+
+ /// \brief Get file path.
+ /// \return File path.
+ /// \pre <tt>this->isOpen() == true</tt>
+ virtual const boost::filesystem::path& getPath() const { return path_; }
+
+ virtual void openFile(const boost::filesystem::path& path,
+ bool ignoreErrors,
+ size_t tileIdx) = 0;
+
+ virtual bool isOpen() const = 0;
+
+ virtual bool readEntireFile(common::RawDataBuffer& buffer) = 0;
+
+protected:
+ boost::filesystem::path path_;
+
+ bool ignoreErrors_;
+}; // end class FileReaderBase
+
+
+class UnprocessedFile : public FileReaderBase
+{
+public:
+ UnprocessedFile(std::ios_base::openmode openFlags);
+
+ UnprocessedFile(const boost::filesystem::path& path,
+ bool ignoreErrors,
+ std::ios_base::openmode openFlags = std::ios_base::in | std::ios_base::binary);
+
+ virtual ~UnprocessedFile();
+
+ virtual void openFile(const boost::filesystem::path& path,
+ bool ignoreErrors,
+ size_t tileIdx);
+
+ virtual bool isOpen() const { return !path_.empty(); }
+
+ /// \brief Return the file type string. Used for logging message purposes.
+ virtual std::string getFileTypeStr() const { return "Unprocessed"; }
+
+ virtual bool readEntireFile(common::RawDataBuffer& buffer);
+
+protected:
+
+ virtual bool openFileBuf();
+
+ virtual void openFileImpl(const boost::filesystem::path& path,
+ bool ignoreErrors);
+
+ virtual bool readBytes(std::vector<char>& buffer,
+ uint32_t bytes);
+
+ virtual void logError(std::streamsize bytesRead,
+ std::streamsize bytesExpected) const;
+
+ /// \brief Filebuf.
+ io::FileBufWithReopen fileBuf_;
+
+ size_t fileSize_;
+};
+
+// Note: It is unlikely that this synchronization is even necessary, but I'm
+// doing it to ensure correctness. In general, 100s of files are read for each
+// tile (1 BCL file for each cycle). Read tasks are popped off the queue in
+// order. The only way we would need synchronization is if somehow 1 read
+// task took long enough for all of the other files to be read and the next
+// tile to begin processing.
+class SyncFile : public UnprocessedFile
+{
+public:
+
+ // SyncFileReader must be used when reading the file in order to ensure
+ // that the tile number is incremented and that the lock is held.
+ class SyncFileReader
+ {
+ public:
+ SyncFileReader(SyncFile& syncFile,
+ size_t tileIdx)
+ : syncFile_(syncFile),
+ lock_(syncFile.mut_)
+ {
+ syncFile_.cv_.wait(lock_, [this, tileIdx] { return tileIdx == syncFile_.currentTile_; });
+ }
+
+ ~SyncFileReader()
+ {
+ ++syncFile_.currentTile_;
+ lock_.unlock();
+ syncFile_.cv_.notify_all();
+ }
+
+ const boost::filesystem::path& getPath() const
+ {
+ return syncFile_.getPath();
+ }
+
+ void openFile(const boost::filesystem::path& path,
+ bool ignoreErrors)
+ {
+ syncFile_.openFile(path, ignoreErrors, syncFile_.currentTile_);
+ }
+
+ bool readEntireFile(common::RawDataBuffer& buffer)
+ {
+ buffer.path_ = syncFile_.path_;
+ return syncFile_.readEntireFile(buffer);
+ }
+
+ bool read(common::RawDataBuffer& buffer,
+ std::streamsize bytes)
+ {
+ buffer.path_ = syncFile_.path_;
+ return syncFile_.read(buffer, bytes);
+ }
+
+ bool read(char* buffer,
+ std::streamsize bytes)
+ {
+ return syncFile_.read(buffer, bytes);
+ }
+
+ bool readGzipBlocks(common::RawDataBuffer& buffer,
+ std::streamsize startPos,
+ std::streamsize nextStartPos)
+ {
+ buffer.path_ = syncFile_.path_;
+ return syncFile_.readGzipBlocks(buffer, startPos, nextStartPos);
+ }
+
+ bool seek(std::streamsize offset,
+ std::ios_base::seekdir way = std::ios_base::cur)
+ {
+ return syncFile_.seek(offset, way);
+ }
+
+ private:
+ SyncFile& syncFile_;
+ std::unique_lock<std::mutex> lock_;
+ };
+
+ SyncFile(std::ios_base::openmode openFlags);
+
+ SyncFile(const boost::filesystem::path& path,
+ bool ignoreErrors,
+ std::ios_base::openmode openFlags);
+
+ virtual ~SyncFile();
+
+ /// \brief Return the file type string. Used for logging message purposes.
+ virtual std::string getFileTypeStr() const { return "Synchronized"; }
+
+ private:
+
+ bool appendBgzfBlock(std::vector<char> &gzipData);
+
+private:
+
+ virtual bool read(char* buffer,
+ std::streamsize bytes);
+
+ virtual bool read(std::vector<char>& buffer,
+ std::streamsize bytes);
+
+ virtual bool seek(std::streamsize offset,
+ std::ios_base::seekdir way = std::ios_base::cur);
+
+ /// \brief Read enough bgzf gzip blocks to get the tile data.
+ /// \param buffer Holds the gzip char data read from file.
+ /// \param startPos The file position to the start of the gzip block containing the tile data.
+ /// \param nextStartPos The file position to the start of the gzip block containing the next tile data. If this value is zero, read to the end of the file.
+ virtual bool readGzipBlocks(std::vector<char>& buffer,
+ std::streamsize startPos,
+ std::streamsize nextStartPos);
+
+ std::mutex mut_;
+ std::condition_variable cv_;
+ std::atomic<size_t> currentTile_;
+ std::vector<char> buffer_;
+};
+
+
+#define CATCH_AND_IGNORE(EXCEPTION_TYPE, MSG_BEGIN)\
+catch (EXCEPTION_TYPE& e)\
+{\
+ if (!ignoreErrors_)\
+ {\
+ BCL2FASTQ_LOG(common::LogLevel::ERROR_TYPE)\
+ << " File: " << getPath().string() << "'." << std::endl;\
+\
+ BOOST_THROW_EXCEPTION(e);\
+ }\
+\
+ BCL2FASTQ_LOG(common::LogLevel::WARNING)\
+ << MSG_BEGIN << getPath().string() << "'." << std::endl\
+ << e.what() << std::endl;\
+}
+
+#define CATCH_AND_IGNORE_ALL(MSG_BEGIN)\
+catch (...)\
+{\
+ if (!ignoreErrors_)\
+ {\
+ throw;\
+ }\
+\
+ BCL2FASTQ_LOG(common::LogLevel::WARNING)\
+ << MSG_BEGIN << getPath().string() << ".'" << std::endl\
+ << "Unexpected exception." << std::endl;\
+}
+
+} // namespace io
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_IO_SYNC_FILE_HH
+
diff --git a/src/cxx/include/io/Utility.hh b/src/cxx/include/io/Utility.hh
new file mode 100644
index 0000000..2bf8baa
--- /dev/null
+++ b/src/cxx/include/io/Utility.hh
@@ -0,0 +1,45 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Utility.hh
+ *
+ * \brief Declaration of various I/O related utilities.
+ *
+ * \author Roman Petorvski
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_IO_UTILITY_HH
+#define BCL2FASTQ_IO_UTILITY_HH
+
+
+#include <ios>
+
+
+namespace bcl2fastq {
+
+
+namespace io {
+
+
+/// \brief Translate ios access flags to stdio mode.
+/// \param mode Access flags used by ios.
+/// \return Mode string used by stdio functions.
+const char * iosFlagsToStdioMode(std::ios_base::openmode mode);
+
+
+} // namespace io
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_IO_UTILITY_HH
+
+
diff --git a/src/cxx/include/io/Xml.hh b/src/cxx/include/io/Xml.hh
new file mode 100644
index 0000000..ef1a275
--- /dev/null
+++ b/src/cxx/include/io/Xml.hh
@@ -0,0 +1,58 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Xml.hh
+ *
+ * \brief Basic XML parser.
+ *
+ * \author Marek Balint
+ * \author Roman Petrovski
+ */
+
+
+#ifndef BCL2FASTQ_IO_XML_HH
+#define BCL2FASTQ_IO_XML_HH
+
+
+#include <boost/filesystem/path.hpp>
+#include <boost/property_tree/ptree.hpp>
+
+
+namespace bcl2fastq {
+namespace io {
+
+void index(std::vector<std::string> indexAttributes, boost::property_tree::ptree &tree);
+
+std::ostream &serializeAsXml(std::ostream &os, const boost::property_tree::ptree &tree);
+
+std::istream &parseAsXml(std::istream &is, boost::property_tree::ptree &tree);
+
+
+/// \brief Parse XML data.
+/// \param xmlDataBegin Begin iterator to XML data to be parsed.
+/// \param xmlDataEnd End iterator to XML data to be parsed.
+/// \return Data strucutre containing data parsed out of the XML data.
+template<typename Iterator>
+boost::property_tree::ptree parseXmlData(Iterator xmlDataBegin, Iterator xmlDataEnd);
+
+/// \brief Parse XML file.
+/// \param xmlFile Path to XML file to be parsed.
+/// \return Data strucutre containing data parsed out of the XML file.
+boost::property_tree::ptree parseXmlFile(const boost::filesystem::path& xmlFile);
+
+
+} // namespace io
+} // namespace bcl2fastq
+
+
+#include "io/Xml.hpp"
+
+
+#endif //BCL2FASTQ_IO_XML_HH
+
+
diff --git a/src/cxx/include/io/Xml.hpp b/src/cxx/include/io/Xml.hpp
new file mode 100644
index 0000000..4df7b6b
--- /dev/null
+++ b/src/cxx/include/io/Xml.hpp
@@ -0,0 +1,47 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file XmlParser.hpp
+ *
+ * \brief Basic XML parser.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_IO_XML_HPP
+#define BCL2FASTQ_IO_XML_HPP
+
+
+#include <sstream>
+#include <string>
+
+#include <boost/property_tree/xml_parser.hpp>
+
+
+namespace bcl2fastq {
+namespace io {
+
+
+template<typename Iterator>
+boost::property_tree::ptree parseXmlData(Iterator xmlDataBegin, Iterator xmlDataEnd)
+{
+ boost::property_tree::ptree ret;
+ std::istringstream istream(std::string(xmlDataBegin, xmlDataEnd));
+ boost::property_tree::read_xml<boost::property_tree::ptree>(static_cast<std::basic_istream<char> &>(istream), ret);
+ return ret;
+}
+
+
+} // namespace io
+} // namespace bcl2fastq
+
+
+#endif //BCL2FASTQ_IO_XML_HPP
+
+
diff --git a/src/cxx/include/layout/BCIndex.hh b/src/cxx/include/layout/BCIndex.hh
new file mode 100644
index 0000000..bb0a7ff
--- /dev/null
+++ b/src/cxx/include/layout/BCIndex.hh
@@ -0,0 +1,111 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BCIndex.hh
+ *
+ * \brief Declaration of BCI helper.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_BCINDEX_HH
+#define BCL2FASTQ_LAYOUT_BCINDEX_HH
+
+
+#include <boost/filesystem/path.hpp>
+
+#include "common/Types.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+/// \brief BCI file helper.
+class BCIndex
+{
+public:
+
+ /// \brief Raw data type definition.
+ typedef std::vector<char> RawData;
+
+ /// \brief Tile meta data.
+ struct TileMetadata
+ {
+ public:
+
+ /// \brief Tile number.
+ common::TileNumber tileNumber_;
+
+ /// \brief Clusters count.
+ common::ClustersCount clustersCount_;
+ };
+
+ /// \brief Tile meta data container type definition.
+ typedef std::vector<TileMetadata> TileMetadataContainer;
+
+public:
+
+ /// \brief Constructor.
+ /// \param bciData BCI data.
+ explicit BCIndex(const RawData &bciData);
+
+public:
+
+ /// \brief Get beginning of tile meta data.
+ /// \return Iterator to beginning of tile meta data.
+ BCIndex::TileMetadataContainer::const_iterator tileMetadataBegin() const;
+
+ /// \brief Get end of tile meta data.
+ /// \return Iterator to end of tile meta data.
+ BCIndex::TileMetadataContainer::const_iterator tileMetadataEnd() const;
+
+private:
+
+ /// \brief Initialize tile meta data container.
+ /// \param bciData Source BCI data.
+ void initTileMetadata(const RawData &bciData);
+
+private:
+
+ /// \brief Tile meta data container.
+ TileMetadataContainer tiles_;
+};
+
+
+/// \brief Parse BCI file.
+/// \param bciFile Path to BCI file to be parsed.
+/// \return Data strucutre containing data parsed out of the BCI file.
+BCIndex::RawData parseBciFile(const boost::filesystem::path& bciFile);
+
+/// \brief Check presence of BCI file.
+/// \param inputDir Path to input directory.
+/// \param laneNumber Lane number.
+/// \retval true The BCI file is present.
+/// \retval false There is no BCI file.
+bool checkBciFile(boost::filesystem::path inputDir, common::LaneNumber laneNumber);
+
+/// \brief Create BCIndex object.
+/// \param inputDir Path to input directory.
+/// \param laneNumber Lane number.
+/// \return BCIndex object constructed with data from given input directory.
+BCIndex createBCIndex(boost::filesystem::path inputDir, common::LaneNumber laneNumber);
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_BCINDEX_HH
+
+
diff --git a/src/cxx/include/layout/Barcode.hh b/src/cxx/include/layout/Barcode.hh
new file mode 100644
index 0000000..9dd1287
--- /dev/null
+++ b/src/cxx/include/layout/Barcode.hh
@@ -0,0 +1,209 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Barcode.hh
+ *
+ * \brief Declaration of barcode.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_BARCODE_HH
+#define BCL2FASTQ_LAYOUT_BARCODE_HH
+
+
+#include <vector>
+#include <string>
+
+#include <stdint.h>
+
+#include "common/Types.hh"
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+/// \brief Barcode type definition.
+class Barcode
+{
+public:
+
+ /// \brief Barcode component type definition.
+ class Component
+ {
+ public:
+
+ /// \brief Size type definition.
+ /// \note Size of the integer type limits maximum number of component's bases (1 base per 3 bits).
+ typedef uint8_t SizeType;
+
+ public:
+
+ /// \brief Invalid base character.
+ static const char INVALID_BASE;
+
+ public:
+
+ /// \brief Constructor.
+ /// \param value Component value.
+ /// \pre <code>value.empty() == false</code>
+ /// \pre <code>value.size() <= 21</code>
+ explicit Component(const std::string &value);
+
+ Component() : value_() { }
+
+ public:
+
+ /// \brief Less-than comparison.
+ /// \param rhs Other instance to compare with.
+ /// \retval true Other instance is less than this instance.
+ /// \retval false Other instance is not less than this instance.
+ bool less(const Component &rhs) const { return value_ < rhs.value_; }
+
+ bool operator!=(const Component &rhs) const { return value_ != rhs.value_; }
+ public:
+
+ /// \brief Get barcode component length.
+ /// \return Number of bases in barcode component.
+ Barcode::Component::SizeType getLength() const { return value_.size(); }
+
+ /// \brief Get the barcode component length.
+ /// \return Number of bases in barcode component.
+ size_t size() const { return value_.size(); }
+
+ /// \brief Get the barcode string.
+ /// \return the string.
+ const std::string& getString() const { return value_; }
+
+ /// \brief Get the barcode string.
+ /// \return the string.
+ std::string& getString() { return value_; }
+
+ /// \brief Reset the memory, without deallocating.
+ void reset() { value_.clear(); }
+
+ public:
+
+ /// \brief Generate components with mismatches.
+ /// \param position Mismatching position.
+ /// \param componentInserter Insert iterator for mismatching barcode components.
+ /// \pre <code>position < this->getLength()</code>
+ template<typename Inserter>
+ void generateMismatches(
+ Component::SizeType position,
+ Inserter componentInserter
+ ) const;
+
+ private:
+
+ /// \brief Value.
+ std::string value_;
+ };
+
+ /// \brief Barcodes container type definition.
+ typedef std::vector<Component> ComponentsContainer;
+
+public:
+
+ /// \brief Barcode components separator.
+ static const char COMPONENT_SEPARATOR;
+
+ /// \brief Label to use when no Barcode is given.
+ static const char DEFAULT_BARCODE[];
+
+public:
+
+ /// \brief Constructor.
+ /// \param componentsBegin Beginning of barcode components.
+ /// \param componentsEnd End of barcode components.
+ /// \pre Iterator @c componentsEnd is reachable from @c componentsBegin.
+ /// \pre <code>componentsBegin != componentsEnd</code>
+ template<typename Iterator>
+ Barcode(Iterator componentsBegin, Iterator componentsEnd);
+
+ Barcode() : components_() { }
+
+ void reset(size_t numComponents);
+
+ std::string toString() const;
+
+public:
+
+ /// \brief Less-than comparison.
+ /// \param rhs Other instance to compare with.
+ /// \retval true Other instance is less than this instance.
+ /// \retval false Other instance is not less than this instance.
+ bool operator<(const Barcode &rhs) const;
+
+ bool operator==(const Barcode &rhs) const;
+ bool operator!=(const Barcode &rhs) const;
+
+public:
+
+ /// \brief Get beginning of barcode components.
+ /// \return Iterator to beginning of barcode segmetns.
+ Barcode::ComponentsContainer::const_iterator componentsBegin() const { return components_.begin(); }
+
+ /// \brief Get end of barcode components.
+ /// \return Iterator to end of barcode components.
+ Barcode::ComponentsContainer::const_iterator componentsEnd() const { return components_.end(); }
+
+ /// \brief Get the barcode components
+ /// \return Barcode components
+ const ComponentsContainer& getComponents() const { return components_; }
+
+public:
+
+ /// \brief Generate barcodes with mismatches.
+ /// \param component Mismatching component.
+ /// \param position Mismatching position of component.
+ /// \param barcodeInserter Insert iterator for mismatching barcodes.
+ /// \pre Iterator @c component must be reachable from <code>this->componentsBegin()</code>
+ /// \pre Iterator @c component must be dereferencable.
+ /// \pre <code>position < component->getLength()</code>
+ template<typename Inserter>
+ void generateMismatches(
+ ComponentsContainer::const_iterator component,
+ Component::SizeType position,
+ Inserter barcodeInserter
+ ) const;
+
+ /// \brief Ouput the barcode to the buffer
+ /// \param buffer to output to
+ void output(std::vector<char>& buffer) const;
+
+ /// \brief Mask the barcode for the given index number
+ /// \param index number.
+ void mask(common::ReadNumber indexNumber);
+
+//private:
+
+ friend std::ostream& operator<<(std::ostream& os, const Barcode& barcode);
+
+ /// \brief Barcode components.
+ ComponentsContainer components_;
+};
+
+/// \brief Barcodes container type definition.
+typedef std::vector<Barcode> BarcodesContainer;
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
+#include "layout/Barcode.hpp"
+
+
+#endif // BCL2FASTQ_LAYOUT_BARCODE_HH
+
+
diff --git a/src/cxx/include/layout/Barcode.hpp b/src/cxx/include/layout/Barcode.hpp
new file mode 100644
index 0000000..bfb3129
--- /dev/null
+++ b/src/cxx/include/layout/Barcode.hpp
@@ -0,0 +1,129 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Barcode.hpp
+ *
+ * \brief Implementation of barcode.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_BARCODE_HPP
+#define BCL2FASTQ_LAYOUT_BARCODE_HPP
+
+
+#include <vector>
+#include <iterator>
+
+#include <boost/foreach.hpp>
+
+#include "common/Debug.hh"
+#include "common/Exceptions.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+template<typename Inserter>
+void Barcode::Component::generateMismatches(
+ Component::SizeType position,
+ Inserter componentInserter
+) const
+{
+ BCL2FASTQ_ASSERT_MSG(position < value_.size(), "Position " << int(position) << " out of range [0," << value_.size() << ")");
+
+ static const char *mismatchesTable[] = {
+ "CGTN", // (A)
+ "AGTN", // (C)
+ "ACGN", // (T)
+ "ACGT", // (N)
+ "ACTN", // (G)
+ };
+
+ size_t mismatchIndex = 0;
+ switch (value_[position])
+ {
+ case 'A':
+ mismatchIndex = 0;
+ break;
+ case 'C':
+ mismatchIndex = 1;
+ break;
+ case 'T':
+ mismatchIndex = 2;
+ break;
+ case 'N':
+ mismatchIndex = 3;
+ break;
+ case 'G':
+ mismatchIndex = 4;
+ break;
+ default:
+ BOOST_ASSERT(false);
+ };
+
+ // magic number 3: 3 bits per base
+ // magic number 0x07: 00000111
+ BOOST_FOREACH (const char mismatchedBase, mismatchesTable[mismatchIndex])
+ {
+ std::string mismatchedValue(value_);
+ mismatchedValue.at(position) = mismatchedBase;
+ const Component component(mismatchedValue);
+ *(*componentInserter) = component;
+ ++(*componentInserter);
+ }
+}
+
+
+template<typename Iterator>
+Barcode::Barcode(Iterator componentsBegin, Iterator componentsEnd)
+: components_(componentsBegin, componentsEnd)
+{
+ if (0) //(componentsBegin == componentsEnd)
+ {
+ BOOST_THROW_EXCEPTION(common::LogicError("Barcode must have at least one component"));
+ }
+}
+
+template<typename Inserter>
+void Barcode::generateMismatches(
+ ComponentsContainer::const_iterator component,
+ Component::SizeType position,
+ Inserter barcodeInserter
+) const
+{
+ BCL2FASTQ_ASSERT_MSG(component >= components_.begin(), "Invalid component");
+ BCL2FASTQ_ASSERT_MSG(component < components_.end(), "Invalid component");
+
+ // magic number 4: There are 5 distinct values ('A', 'C', 'G', 'T', 'N'), therefore there are 4 distinct mismatches
+ std::vector<Component> mismatchedComponents;
+ mismatchedComponents.reserve(4);
+ component->generateMismatches(position, std::back_inserter(mismatchedComponents));
+
+ BOOST_FOREACH (const Component &mismatchedComponent, mismatchedComponents)
+ {
+ Barcode barcode(*this);
+ barcode.components_.at(component-components_.begin()) = mismatchedComponent;
+ *(*barcodeInserter) = barcode;
+ ++(*barcodeInserter);
+ }
+}
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_BARCODE_HPP
+
+
diff --git a/src/cxx/include/layout/BarcodeCollisionDetector.hh b/src/cxx/include/layout/BarcodeCollisionDetector.hh
new file mode 100644
index 0000000..78f7d15
--- /dev/null
+++ b/src/cxx/include/layout/BarcodeCollisionDetector.hh
@@ -0,0 +1,73 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BarcodeCollisionDetector.hh
+ *
+ * \brief Declaration of BarcodeCollisionDetector
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_BARCODE_COLLISION_DETECTOR_HH
+#define BCL2FASTQ_LAYOUT_BARCODE_COLLISION_DETECTOR_HH
+
+#include "layout/Barcode.hh"
+#include "common/Exceptions.hh"
+#include "config/Bcl2FastqOptions.hh"
+
+#include <boost/noncopyable.hpp>
+#include <vector>
+
+namespace bcl2fastq
+{
+namespace layout
+{
+
+class BarcodeCollisionError : public common::RuntimeError
+{
+public:
+ BarcodeCollisionError(const std::string& message);
+ BarcodeCollisionError(const BarcodeCollisionError& other);
+};
+
+class BarcodeCollisionDetector : private boost::noncopyable
+{
+public:
+ BarcodeCollisionDetector(config::BarcodeMismatchCountsContainer& componentMaxMismatches,
+ bool autoSetToZeroMismatches = false);
+
+ void validateBarcode(BarcodesContainer barcodes);
+
+private:
+ void validateNewBarcodeSizes(const BarcodesContainer& barcodes,
+ const std::vector<std::size_t>& barcodeComponentSizes) const;
+
+ void validateNewBarcodeSizesAgainstExisting(const std::vector<std::size_t>& barcodeComponentSizes) const;
+
+ void validateBarcodeSizes(const BarcodesContainer& barcodes);
+
+ void validateBarcodes(const BarcodesContainer::value_type& barcode1,
+ const BarcodesContainer::value_type& barcode2);
+
+ void handleCollision(const BarcodesContainer::value_type& barcode1,
+ const BarcodesContainer::value_type& barcode2);
+
+ size_t getNumMismatches(const Barcode::Component& barcode1,
+ const Barcode::Component& barcode2) const;
+
+ bool autoSetToZeroMismatches_;
+ bool triggerZeroMismatches_;
+ config::BarcodeMismatchCountsContainer& componentMaxMismatches_;
+ BarcodesContainer barcodes_;
+};
+
+} // namespace layout
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_LAYOUT_BARCODE_COLLISION_DETECTOR_HH
diff --git a/src/cxx/include/layout/BarcodeTranslationTable.hh b/src/cxx/include/layout/BarcodeTranslationTable.hh
new file mode 100644
index 0000000..6ee4889
--- /dev/null
+++ b/src/cxx/include/layout/BarcodeTranslationTable.hh
@@ -0,0 +1,181 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BarcodeTranslationTable.hh
+ *
+ * \brief Declaration of barcode translation table.
+ *
+ * \author Roman Petrovski
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_BARCODETRANSLATIONTABLE_HH
+#define BCL2FASTQ_LAYOUT_BARCODETRANSLATIONTABLE_HH
+
+
+#include <vector>
+#include <string>
+
+#include <boost/filesystem.hpp>
+#include <boost/iterator/transform_iterator.hpp>
+
+#include "common/Exceptions.hh"
+#include "config/Bcl2FastqOptions.hh"
+#include "layout/Barcode.hh"
+#include "layout/SampleInfo.hh"
+#include "layout/LaneInfo.hh"
+
+
+namespace bcl2fastq
+{
+namespace layout
+{
+
+
+/// \brief Barcode translation table.
+/// \todo Refactoring: This class is just a pretty-printed copy-paste form old converter, forced into this code.
+/// \todo Features: Make number of allowed mismatches per component configurable (currently hardcoded to 2).
+class BarcodeTranslationTable : private boost::noncopyable
+{
+public:
+
+ /// \brief Sample meta data.
+ struct SampleMetadata
+ {
+ /// \brief Sample index.
+ layout::LaneInfo::SampleInfosContainer::size_type sampleIndex_;
+
+ /// \brief Barcode index.
+ BarcodesContainer::size_type barcodeIndex_;
+
+ SampleMetadata( layout::LaneInfo::SampleInfosContainer::size_type sampleIndex,
+ BarcodesContainer::size_type barcodeIndex = 0)
+ : sampleIndex_(sampleIndex), barcodeIndex_(barcodeIndex) {}
+
+ SampleMetadata() ///< \brief Default sample
+ : sampleIndex_(0), barcodeIndex_(0) {}
+
+ /// \param lhs Left-hand-side argument.
+ /// \param rhs Right-hand-side argument.
+ /// \retval true Left-hand-side argument is less than right-hand-side argument.
+ /// \retval false Left-hand-side argument is not less than right-hand-side argument.
+ bool operator<( const SampleMetadata &rhs ) const
+ {
+ if (this->sampleIndex_ != rhs.sampleIndex_)
+ {
+ return this->sampleIndex_ < rhs.sampleIndex_;
+ }
+ return this->barcodeIndex_ < rhs.barcodeIndex_;
+ }
+
+ bool operator==( const SampleMetadata &rhs ) const
+ {
+ return (this->sampleIndex_ == rhs.sampleIndex_) && (this->barcodeIndex_ == rhs.barcodeIndex_);
+ }
+
+ bool operator!=( const SampleMetadata &rhs ) const { return !(*this == rhs); }
+ };
+
+public:
+
+ /// \brief Constructor.
+ /// \param samplesBegin Beginning of samples.
+ /// \param samplesEnd End of samples.
+ /// \param componentMaxMismatches Maximum number of allowed mismatches per barcode component.
+ BarcodeTranslationTable(
+ const layout::LaneInfo::SampleInfosContainer::const_iterator samplesBegin,
+ const layout::LaneInfo::SampleInfosContainer::const_iterator samplesEnd,
+ const config::BarcodeMismatchCountsContainer& componentMaxMismatches
+ );
+
+public:
+
+ /// \brief Translate barcode to sample.
+ /// \param barcode Barcode to be translated.
+ /// \param mismatches Return number of mismatches w.r.t. perfect barcode
+ /// \return Sample associated with the barcode.
+ BarcodeTranslationTable::SampleMetadata translateBarcode(
+ const layout::Barcode &barcode,
+ unsigned int &mismatches
+ ) const;
+
+private:
+
+ /// \brief Barcode to sample mapping type definition.
+ struct BarcodeSampleMapping
+ {
+ layout::Barcode barcode;
+ SampleMetadata sample;
+ unsigned int mismatches;
+
+ BarcodeSampleMapping(const layout::Barcode &b, const SampleMetadata &s, unsigned int m=0)
+ : barcode(b), sample(s), mismatches(m) {}
+ };
+
+ /// \brief Barcode to sample mappings container type definition.
+ typedef std::vector<BarcodeSampleMapping> BarcodeSampleMappingsContainer;
+
+ /// \brief Barcode to sample mappings range definition.
+ typedef std::pair<BarcodeSampleMappingsContainer::const_iterator,
+ BarcodeSampleMappingsContainer::const_iterator> BarcodeSampleMappingsRange;
+private:
+
+ /// \brief Initialize barcode to sample mappings.
+ /// \param samplesBegin Beginning of samples.
+ /// \param samplesEnd End of samples.
+ /// \param componentMaxMismatches Maximum number of allowed mismatches per barcode component.
+ /// \return Initialized barcode to sample mappings container.
+ static const BarcodeTranslationTable::BarcodeSampleMappingsContainer initBarcodeSampleMappings(
+ const layout::LaneInfo::SampleInfosContainer::const_iterator samplesBegin,
+ const layout::LaneInfo::SampleInfosContainer::const_iterator samplesEnd,
+ const config::BarcodeMismatchCountsContainer& componentMaxMismatches
+ );
+
+ /// \brief Insert barcode to sample mapping into container.
+ /// \param barcodeMapping Barcode to sample mapping.
+ /// \param component Component being currently processed.
+ /// \param position Position of component, where mismatches are to be simulated.
+ /// \param allowedMismatchesLeft Number of mismatches left for current component.
+ /// \param componentMaxMismatchesIter Iterator to maximum number of allowed mismatches for current component.
+ /// \param componentMaxMismatchesEnd End iterator to maximum number of allowed mismatches per barcode component.
+ /// \param container Container to insert to.
+ static void insertBarcodeMapping(
+ const BarcodeSampleMapping &barcodeMapping,
+ layout::Barcode::ComponentsContainer::const_iterator component,
+ layout::Barcode::Component::SizeType position,
+ unsigned int allowedMismatchesLeft,
+ std::vector<std::size_t>::const_iterator componentMaxMismatchesIter,
+ std::vector<std::size_t>::const_iterator componentMaxMismatchesEnd,
+ BarcodeSampleMappingsContainer &container
+ );
+
+ /// \brief Compare barcode to sample mappings.
+ /// \param lhs Left-hand-side argument.
+ /// \param rhs Right-hand-side argument.
+ /// \retval true Left-hand-side argument is less than right-hand-side argument.
+ /// \retval false Left-hand-side argument is not less than right-hand-side argument.
+ static bool compareBarcodeSampleMappings(const BarcodeSampleMapping &lhs, const BarcodeSampleMapping &rhs);
+
+private:
+
+ /// \brief Barcode to sample mappings.
+ const BarcodeSampleMappingsContainer barcodeSampleMappings_;
+
+ friend std::ostream& operator<<(std::ostream& os, const BarcodeTranslationTable::BarcodeSampleMapping &bsm);
+};
+
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_BARCODETRANSLATIONTABLE_HH
+
+
diff --git a/src/cxx/include/layout/ConfigXml.hh b/src/cxx/include/layout/ConfigXml.hh
new file mode 100644
index 0000000..74fa269
--- /dev/null
+++ b/src/cxx/include/layout/ConfigXml.hh
@@ -0,0 +1,111 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ConfigXml.hh
+ *
+ * \brief Declaration of config.xml helper.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_CONFIG_HH
+#define BCL2FASTQ_LAYOUT_CONFIG_HH
+
+
+#include <boost/filesystem/path.hpp>
+#include <boost/property_tree/ptree.hpp>
+
+#include "common/Types.hh"
+#include "layout/LaneInfo.hh"
+#include "config/SampleSheetCsv.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+/// \brief Confing.xml file helper.
+class ConfigXml
+{
+public:
+
+ /// \brief Tile meta data.
+ struct TileMetadata
+ {
+ public:
+
+ /// \brief Tile number.
+ common::TileNumber tileNumber_;
+ };
+
+ /// \brief Tile meta data container type definition.
+ typedef std::vector<TileMetadata> TileMetadataContainer;
+
+public:
+
+ /// \brief Constructor.
+ /// \param configData Config XML data.
+ explicit ConfigXml(boost::property_tree::ptree configData);
+
+public:
+
+ /// \brief Check if config.xml contains the lane number
+ /// \param laneNumber Lane number
+ /// \return True if config.xml contains the lane number.
+ bool containsLaneNumber(common::LaneNumber laneNumber) const { return (laneNumber > 0 && laneNumber <= tiles_.size()); }
+
+ /// \brief Get beginning of tile meta data.
+ /// \param laneNumber Lane number.
+ /// \return Iterator to beginning of tile meta data.
+ ConfigXml::TileMetadataContainer::const_iterator tileMetadataBegin(common::LaneNumber laneNumber) const;
+
+ /// \brief Get end of tile meta data.
+ /// \param laneNumber Lane number.
+ /// \return Iterator to end of tile meta data.
+ ConfigXml::TileMetadataContainer::const_iterator tileMetadataEnd(common::LaneNumber laneNumber) const;
+
+private:
+
+ /// \brief Initialize tile meta data container.
+ void initTileMetadata();
+
+private:
+
+ /// \brief Property tree holding the actual data.
+ boost::property_tree::ptree ptree_;
+
+ /// \brief Tile meta data container.
+ std::vector<TileMetadataContainer> tiles_;
+};
+
+
+/// \brief Check presence of config.xml.
+/// \param inputDir Path to input directory.
+/// \retval true The config.xml file is present.
+/// \retval false There is no config.xml.
+bool checkConfigXml(const boost::filesystem::path& inputDir);
+
+
+/// \brief Create ConfigXml object.
+/// \param inputDir Path to input directory.
+/// \return ConfigXml object constructed with data from given input directory.
+ConfigXml createConfigXml(const boost::filesystem::path& inputDir);
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_CONFIG_HH
+
+
diff --git a/src/cxx/include/layout/CycleInfo.hh b/src/cxx/include/layout/CycleInfo.hh
new file mode 100644
index 0000000..324acb0
--- /dev/null
+++ b/src/cxx/include/layout/CycleInfo.hh
@@ -0,0 +1,78 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file CycleInfo.hh
+ *
+ * \brief Declaration of cycle metadata.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_CYCLEINFO_HH
+#define BCL2FASTQ_LAYOUT_CYCLEINFO_HH
+
+
+#include <vector>
+
+#include <boost/filesystem/path.hpp>
+
+#include "common/Types.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+/// \brief Cycle metadata.
+class CycleInfo
+{
+public:
+
+ /// \brief BCL file metadata container type definition.
+ typedef std::vector<boost::filesystem::path> BclFilesContainer;
+
+public:
+
+ /// \brief Constructor.
+ /// \param number Cycle number.
+ explicit CycleInfo(common::CycleNumber number);
+
+public:
+
+ /// \brief Get cycle number.
+ /// \return Cycle number.
+ common::CycleNumber getNumber() const;
+
+ /// \brief Equality operator.
+ /// \return true if equal.
+ bool operator==(const CycleInfo& other) const { return number_ == other.number_; }
+
+ /// \brief Less than operator
+ /// \return true if less.
+ bool operator<(const CycleInfo& other) const { return number_ < other.number_; }
+
+private:
+
+ /// \brief Cycle number.
+ /// Source: RunInfo.xml (FirstCycle..LastCycle or NumCycles)
+ common::CycleNumber number_;
+};
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_CYCLEINFO_HH
+
+
diff --git a/src/cxx/include/layout/FileExistenceVerifier.hh b/src/cxx/include/layout/FileExistenceVerifier.hh
new file mode 100644
index 0000000..f64bb54
--- /dev/null
+++ b/src/cxx/include/layout/FileExistenceVerifier.hh
@@ -0,0 +1,85 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FileExistenceVerifier.hh
+ *
+ * \brief Declaration of FileExistenceVerifier.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_LAYOUT_FILE_EXISTENCE_VERIFIER_HH
+#define BCL2FASTQ_LAYOUT_FILE_EXISTENCE_VERIFIER_HH
+
+#include "common/Types.hh"
+
+#include <boost/noncopyable.hpp>
+#include <boost/filesystem/path.hpp>
+
+namespace bcl2fastq
+{
+namespace layout
+{
+
+class LaneInfo;
+
+class FileExistenceVerifier : private boost::noncopyable
+{
+public:
+
+ static void verifyAllFilesExist(const boost::filesystem::path& inputDir,
+ const boost::filesystem::path& intensitiesDir,
+ const LaneInfo& laneInfo,
+ const common::TileFileMap& tileFileNameMap,
+ common::TileAggregationMode aggregateTilesMode,
+ bool isPatternedFlowcell,
+ bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions);
+
+private:
+ static void verifyFilesExist(const boost::filesystem::path& inputDir,
+ const boost::filesystem::path& intensitiesDir,
+ const LaneInfo& laneInfo,
+ const common::TileFileMap& tileFileNameMap,
+ common::TileAggregationMode aggregateTilesMode,
+ bool isPatternedFlowcell,
+ bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions,
+ const common::TileNumber tileNumber = 0);
+
+ static void verifyFilterFileExists(const boost::filesystem::path& inputDir,
+ common::TileAggregationMode aggregateTilesMode,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ common::ReadNumber readNumber);
+
+ static void verifyBclFileExists(const boost::filesystem::path& inputDir,
+ common::TileAggregationMode aggregateTilesMode,
+ common::LaneNumber laneNumber,
+ common::CycleNumber cycleNumber,
+ common::TileNumber tileNumber,
+ const common::TileFileMap& tileFileNameMap);
+
+ static void verifyPositionsFileExists(const boost::filesystem::path& intensitiesDir,
+ common::TileAggregationMode aggregateTilesMode,
+ bool isPatternedFlowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber);
+
+ static void throwException(const std::string& fileType,
+ common::TileAggregationMode aggregateTilesMode,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber);
+};
+
+} // namespace layout
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_LAYOUT_FILE_EXISTENCE_VERIFIER_HH
diff --git a/src/cxx/include/layout/FlowcellInfo.hh b/src/cxx/include/layout/FlowcellInfo.hh
new file mode 100644
index 0000000..cd98783
--- /dev/null
+++ b/src/cxx/include/layout/FlowcellInfo.hh
@@ -0,0 +1,130 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FlowcellInfo.hh
+ *
+ * \brief Declaration of flowcell metadata.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_FLOWCELLINFO_HH
+#define BCL2FASTQ_LAYOUT_FLOWCELLINFO_HH
+
+
+#include "layout/RunInfoXml.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+/// \brief Flowcell metadata.
+class FlowcellInfo
+{
+public:
+
+ /// \brief Get instrument.
+ /// \return Instrument.
+ const std::string& getInstrument() const { return instrument_; }
+
+ /// \brief Set instrument.
+ /// \param instrument Value to be set.
+ void setInstrument(const std::string &instrument);
+
+ /// \brief Get run number.
+ /// \return Run number.
+ const std::string& getRunNumber() const { return runNumber_; }
+
+ /// \brief Set run number.
+ /// \param runNumber Value to be set.
+ void setRunNumber(const std::string &runNumber);
+
+ /// \brief Get flowcell ID.
+ /// \return Flowcell ID.
+ const std::string& getFlowcellId() const { return flowcellId_; }
+
+ /// \brief Set flowcell ID.
+ /// \param flowcellId Value to be set.
+ void setFlowcellId(const std::string &flowcellId);
+
+ /// \brief Get aggregate tiles flag.
+ /// \retval true Data for all tiles are aggregated into single file.
+ /// \retval false There is separate file for each tile.
+ common::TileAggregationMode getAggregateTilesMode() const;
+
+ /// \brief Set value of aggregate tiles flag.
+ /// \param aggregateTilesFlag Value to be set.
+ void setAggregateTilesFlag(common::TileAggregationMode aggregateTilesFlag);
+
+ /// \brief Get patterned flowcell flag.
+ /// \retval true The clusters positions is known up-front.
+ /// \retval false The clusters positions is not known up-front.
+ bool isPatternedFlowcell() const;
+
+ /// \brief Set value of patterned flowcell flag to true.
+ void setPatternedFlowcell();
+
+ /// \brief Reset value of patterned flowcell flag to false.
+ void resetPatternedFlowcell();
+
+ /// \brief Set the run id.
+ /// \param Run Id to set.
+ void setRunId(const std::string& runId) { runId_ = runId; }
+
+ /// \brief Get the run id.
+ /// \retval Run Id.
+ const std::string& getRunId() const { return runId_; }
+
+private:
+
+ /// \brief Instrument.
+ ///
+ /// Source: RunInfo.xml
+ std::string instrument_;
+
+ /// \brief Run number.
+ ///
+ /// Source: RunInfo.xml
+ std::string runNumber_;
+
+ /// \brief Flowcell ID.
+ ///
+ /// Source: RunInfo.xml
+ std::string flowcellId_;
+
+ /// \brief Aggregate tiles flag.
+ ///
+ /// Source: command line argument --aggregated-tiles
+ common::TileAggregationMode aggregateTilesFlag_;
+
+ /// \brief Patterned Flowcell flag.
+ ///
+ /// Source: whether s.locs exist or not
+ bool patternedFlowcell_;
+
+ /// \brief Run ID.
+ ///
+ /// Source: RunInfo.xml
+ std::string runId_;
+};
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_FLOWCELLINFO_HH
+
+
diff --git a/src/cxx/include/layout/LaneInfo.hh b/src/cxx/include/layout/LaneInfo.hh
new file mode 100644
index 0000000..8674a6a
--- /dev/null
+++ b/src/cxx/include/layout/LaneInfo.hh
@@ -0,0 +1,185 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file LaneInfo.hh
+ *
+ * \brief Declaration of lane metadata.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_LANEINFO_HH
+#define BCL2FASTQ_LAYOUT_LANEINFO_HH
+
+
+#include <vector>
+
+#include <boost/filesystem/path.hpp>
+#include <boost/format.hpp>
+
+#include "common/Types.hh"
+#include "layout/SampleInfo.hh"
+#include "layout/TileInfo.hh"
+#include "layout/ReadInfo.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+/// \brief Lane metadata.
+class LaneInfo
+{
+public:
+
+ /// \brief Sample metadata container type definition.
+ typedef std::vector<SampleInfo> SampleInfosContainer;
+
+ /// \brief Tile metadata container type definition.
+ typedef std::vector<TileInfo> TileInfosContainer;
+
+public:
+
+ /// \brief Constructor.
+ /// \param number Lane number.
+ explicit LaneInfo(common::LaneNumber number);
+
+public:
+
+ /// \brief Get lane number.
+ /// \return Lane number.
+ common::LaneNumber getNumber() const { return number_; }
+
+ /// \brief Get lane directory name.
+ /// \return Lane directory name.
+ boost::filesystem::path getDirName() const { return boost::filesystem::path((boost::format("L%03d") % number_).str()); }
+
+public:
+
+ /// \brief Remove the default sample.
+ void removeDefaultSample();
+
+ /// \brief Mask the barcode for the given index number
+ /// \param index number.
+ void maskBarcode(common::ReadNumber indexNumber);
+
+ /// \brief Get beginning of lanes.
+ /// \return Iterator to beginning of lanes.
+ LaneInfo::SampleInfosContainer::const_iterator sampleInfosBegin() const { return sampleInfos_.begin(); }
+
+ /// \brief Get end of lanes.
+ /// \return Iterator to end of lanes.
+ LaneInfo::SampleInfosContainer::const_iterator sampleInfosEnd() const { return sampleInfos_.end(); }
+
+ /// \brief Get the sample info container.
+ /// \return Sample info container.
+ const SampleInfosContainer& getSampleInfos() const { return sampleInfos_; }
+
+ /// \brief Sort sampleInfos by sample number.
+ void sortSampleInfos();
+
+ /// \brief Get the tile infos
+ /// \return Container of tile infos
+ const TileInfosContainer& getTileInfos() const { return tileInfos_; }
+
+ /// \brief Get the read infos
+ /// \return Read infos container
+ const ReadInfosContainer& readInfos() const { return readInfos_; }
+
+ /// \brief Get the read infos. (The functions returning iterators are silly.)
+ /// \return Read infos.
+ const ReadInfosContainer& getReadInfos() const { return readInfos_; }
+
+ /// \brief Get the minimum trimmed read length.
+ /// \return Minimum trimmed read length.
+ size_t getMinimumTrimmedReadLength() const { return minimumTrimmedReadLength_; }
+
+ /// \brief Get the maximum number of barcode mismatches.
+ /// \return a vector of the maximum number of barcode mismatches. 1 entry for each index.
+ const std::vector<size_t>& getMaxBarcodeMismatches() const { return maxBarcodeMismatches_; }
+
+ /// \brief Get number of cycles to load from BCL files.
+ /// \return Number of cycles.
+ common::CycleNumber getNumCyclesToLoad() const;
+
+public:
+
+ /// \brief Get clusters count.
+ /// \return Number of clusters on all tiles on this lane.
+ /// \pre <tt>this->haveClustersCount() == true</tt>
+ common::ClustersCount getClustersCount() const;
+
+ /// \brief Check whether clusters count is known.
+ /// \retval true Clusters count is known.
+ /// \retval false Clusters count is not known.
+ bool haveClustersCount() const;
+
+public:
+
+ /// \brief Add sample metadata.
+ /// \param sampleInfo Sample metadata to be added to read.
+ void addSample(const SampleInfo &sampleInfo) { sampleInfos_.push_back(sampleInfo); }
+
+ /// \brief Add tile metadata.
+ /// \param tileInfo Tile metadata to be added to read.
+ void addTile(const TileInfo &tileInfo) { tileInfos_.push_back(tileInfo); }
+
+ /// \brief Add read metadata.
+ // \param readInfo Read metadata to be added.
+ void addRead(const ReadInfo& readInfo) { readInfos_.push_back(readInfo); }
+
+ /// \brief Set the minimum trimmed read length.
+ /// \param minimumTrimmedReadLength The minimum trimmed read length.
+ void setMinimumTrimmedReadLength(size_t minimumTrimmedReadLength) { minimumTrimmedReadLength_ = minimumTrimmedReadLength; }
+
+ /// \brief Set the maximum number of barcode mismatches.
+ /// \param Vector of maximum number of barcode mismatches. 1 entry for each index.
+ void setMaxBarcodeMismtaches(const std::vector<std::size_t>& maxBarcodeMismatches) { maxBarcodeMismatches_ = maxBarcodeMismatches; }
+
+ /// \brief Overloaded comparison operator.
+ bool operator<(const LaneInfo &laneInfo) const { return number_ < laneInfo.number_; }
+
+private:
+
+ /// \brief Lane number.
+ ///
+ /// Source: RunInfo.xml (1..n)
+ common::LaneNumber number_;
+
+ /// \brief Samples metadata container.
+ SampleInfosContainer sampleInfos_;
+
+ /// \brief Tile metadata container.
+ TileInfosContainer tileInfos_;
+
+ /// \brief Reads metadata container.
+ ReadInfosContainer readInfos_;
+
+ /// \brief Minimum trimmed read length.
+ size_t minimumTrimmedReadLength_;
+
+ /// \brief Maximum number of mismatches allowed in a barcode.
+ std::vector<std::size_t> maxBarcodeMismatches_;
+};
+
+/// \brief Lane metadata container type definition.
+typedef std::vector<LaneInfo> LaneInfosContainer;
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_LANEINFO_HH
+
+
diff --git a/src/cxx/include/layout/Layout.hh b/src/cxx/include/layout/Layout.hh
new file mode 100644
index 0000000..09cf01d
--- /dev/null
+++ b/src/cxx/include/layout/Layout.hh
@@ -0,0 +1,417 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Layout.hh
+ *
+ * \brief Declaration of data layout.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_LAYOUT_HH
+#define BCL2FASTQ_LAYOUT_LAYOUT_HH
+
+
+#include <vector>
+
+#include <boost/filesystem/path.hpp>
+#include <boost/optional.hpp>
+#include <boost/noncopyable.hpp>
+
+#include "layout/FlowcellInfo.hh"
+#include "layout/LaneInfo.hh"
+#include "layout/ReadInfo.hh"
+#include "common/Types.hh"
+#include "config/Bcl2FastqOptions.hh"
+
+namespace bcl2fastq {
+
+namespace config {
+class SampleSheetCsv;
+}
+
+namespace layout {
+
+
+
+/// \brief Data %layout.
+///
+/// \dot
+/// digraph LAYOUT {
+/// nodesep=0.5;
+/// rankdir=LR;
+/// fontsize=12;
+///
+/// layout [ label="layout" URL="\ref Layout" shape=box ];
+///
+/// flowcell [ label="flowcell" URL="\ref FlowcellInfo" shape=box ];
+/// layout -> flowcell [ label="1:1" URL="\ref Layout::flowcellInfo_" ];
+///
+/// flowcellId [ label="ID" URL="\ref FlowcellInfo::flowcellId_" shape=ellipse ];
+/// flowcell -> flowcellId
+///
+/// runNumber [ label="run number" URL="\ref FlowcellInfo::runNumber_" shape=ellipse ];
+/// flowcell -> runNumber
+///
+/// instrument [ label="instrument" URL="\ref FlowcellInfo::instrument_" shape=ellipse ];
+/// flowcell -> instrument
+///
+/// aggregateTilesFlag [ label="aggregated tiles flag" URL="\ref FlowcellInfo::aggregateTilesFlag_" shape=ellipse ];
+/// flowcell -> aggregateTilesFlag
+///
+/// lane [ label="lane" URL="\ref LaneInfo" shape=box ];
+/// layout -> lane [ label="1:n" URL="\ref Layout::laneInfos_" ];
+///
+/// laneNumber [ label="number" URL="\ref LaneInfo::number_" shape=ellipse ];
+/// lane -> laneNumber
+///
+/// sample [ label="sample" URL="\ref SampleInfo" shape=box ];
+/// lane -> sample [ label="1:n" URL="\ref LaneInfo::sampleInfos_" ]
+///
+/// sampleNumber [ label="number" URL="\ref SampleInfo::number_" shape=ellipse ];
+/// sample -> sampleNumber
+///
+/// sampleId [ label="ID" URL="\ref SampleInfo::sampleId_" shape=ellipse ];
+/// sample -> sampleId
+///
+/// sampleName [ label="name" URL="\ref SampleInfo::sampleName_" shape=ellipse ];
+/// sample -> sampleName
+///
+/// barcode [ label="barcode" URL="\ref Barcode" shape=box ];
+/// sample -> barcode [ label="1:n" URL="\ref SampleInfo::barcodes_" ];
+///
+/// component [ label="component" URL="\ref Barcode::Component" shape=box ];
+/// barcode -> component [ label="1:n" URL="\ref Barcode::components_" ];
+///
+/// componentLength [ label="length" URL="\ref Barcode::Component::length_" shape=ellipse ];
+/// component -> componentLength;
+///
+/// componentValue [ label="value" URL="\ref Barcode::Component::value_" shape=ellipse ];
+/// component -> componentValue;
+///
+/// tile [ label="tile" URL="\ref TileInfo" shape=box ];
+/// lane -> tile [ label="1:n" URL="\ref LaneInfo::tileInfos_" ]
+///
+/// tileNumber [ label="number" URL="\ref TileInfo::number_" shape=ellipse ];
+/// tile -> tileNumber
+///
+/// tileIndex [ label="index" URL="\ref TileInfo::index_" shape=ellipse ];
+/// tile -> tileIndex
+///
+/// tileClustersCount [ label="clusters count" URL="\ref TileInfo::clustersCount_" shape=ellipse ];
+/// tile -> tileClustersCount
+///
+/// tileHaveClustersCount [ label="have clusters count flag" URL="\ref TileInfo::haveClustersCount_" shape=ellipse ];
+/// tile -> tileHaveClustersCount
+///
+/// tileSkippedTilesCount [ label="skipped tiles count" URL="\ref TileInfo::skippedTiles" shape=ellipse ];
+/// tile -> tileSkippedTilesCount
+///
+/// tileSkippedClustersCount [ label="skipped clusters count" URL="\ref TileInfo::skippedClusters_" shape=ellipse ];
+/// tile -> tileSkippedClustersCount
+///
+/// read [ label="read" URL="\ref ReadInfo" shape=box ];
+/// layout -> read [ label="1:n" URL="\ref Layout::readInfos_" ];
+///
+/// readNumber [ label="number" URL="\ref ReadInfo::number_" shape=ellipse ];
+/// read -> readNumber
+///
+/// indexReadFlag [ label="index read flag" URL="\ref ReadInfo::indexReadFlag_" shape=ellipse ];
+/// read -> indexReadFlag
+///
+/// maskAdapters [ label="mask adapters" URL="\ref ReadInfo::maskAdapters_" shape=ellipse ];
+/// read -> maskAdapters
+///
+/// trimAdapters [ label="trim adapters" URL="\ref ReadInfo::trimAdapters_" shape=ellipse ];
+/// read -> trimAdapters
+/// }
+/// \enddot
+///
+/// - %layout
+/// - flowcell
+/// - ID <- RunInfo.xml
+/// - run number <- RunInfo.xml
+/// - instrument <- RunInfo.xml
+/// - aggregated tiles flag <- aggregated-tiles cmdline argument
+/// - lanes
+/// - number <- RunInfo.xml (1..n)
+/// - samples
+/// - number <- SampleSheet.csv (0,1..n)
+/// - ID <- SampleSheet.csv
+/// - name <- SampleSheet.csv
+/// - barcodes <- SampleSheet.csv
+/// - barcode components
+/// - tiles
+/// - number <- BCI file (Nova), config.xml (HiSeq/MiSeq)
+/// - index <- index in tiles container (0..n)
+/// - clusters count <- BCI file (Nova), BCL file (HiSeq/MiSeq)
+/// - have clusters count flag <- true only in BCI file is present
+/// - number of skipped tiles immediately before this tile <- tiles cmdline argument
+/// - number of skipped clusters immediately before this tile <- tiles cmdline argument
+/// - reads
+/// - number <- RunInfo.xml (1..n)
+/// - index read flag <- RunInfo.xml
+/// - mask adapters <- SampleSheet.csv
+/// - trim adapters <- SampleSheet.csv
+/// - cycles
+/// - number <- RunInfo.xml (FirstCycle..LastCycle or NumCycles)
+class Layout : private boost::noncopyable
+{
+public:
+/// \brief Detect layout.
+/// \param inputDir Path to input directory.
+/// \param reportsDir Path to reports directory.
+/// \param statsDir Path to stats directory.
+/// \param tilesAggregationFlag All tiles in same BCL file vs. separate BCL file for each tile.
+/// \param tilesFilterList List of regexps for tile filtering.
+/// \return Initialized layout object.
+ Layout(const boost::filesystem::path& intensitiesDir,
+ const boost::filesystem::path& inputDir,
+ const boost::filesystem::path& outputDir,
+ const boost::filesystem::path& reportsDir,
+ const boost::filesystem::path& statsDir,
+ const config::SampleSheetCsv& sampleSheet,
+ const RunInfoXml& runInfoXml,
+ const std::vector<std::string> & tilesFilterList,
+ const std::vector<std::string>& useBasesMasks,
+ size_t minimumTrimmedReadLength,
+ bool autoSetToZeroMismatches,
+ config::BarcodeMismatchCountsContainer& maxMismatches,
+ bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions,
+ common::CycleNumber read1StartCycle,
+ common::CycleNumber read2StartCycle,
+ common::CycleNumber read1EndCycle,
+ common::CycleNumber read2EndCycle,
+ common::CycleNumber read1UmiLength,
+ common::CycleNumber read2UmiLength,
+ common::CycleNumber read1UmiStartFromCycle,
+ common::CycleNumber read2UmiStartFromCycle,
+ bool trimUmi,
+ bool includeNonPfClusters);
+
+ /// \brief Get flowcell metadata.
+ /// \return Flowcell metadata.
+ const FlowcellInfo & getFlowcellInfo() const;
+
+ /// \brief Get beginning of lanes.
+ /// \return Iterator to beginning of lanes.
+ LaneInfosContainer::const_iterator laneInfosBegin() const;
+
+ /// \brief Get end of lanes.
+ /// \return Iterator to end of lanes.
+ LaneInfosContainer::const_iterator laneInfosEnd() const;
+
+ /// \brief Get the map of tile number to file name.
+ /// \return Map of tile number to file name.
+ const common::TileFileMap& getTileFileMap() const { return tileFileMap_; }
+
+ common::NumBasesPerByte getNumBasesPerByte() const { return numBasesPerByte_; }
+
+private:
+
+ void detectFlowcellInfo(const boost::filesystem::path& intensitiesDir,
+ const RunInfoXml& runInfoXml,
+ bool hasBci,
+ bool hasCbcl);
+
+ /// \brief Flowcell metadata.
+ FlowcellInfo flowcellInfo_;
+
+ /// \brief Lanes metadata container.
+ LaneInfosContainer laneInfos_;
+
+ /// \brief Map of tile number to file name.
+ common::TileFileMap tileFileMap_;
+
+ /// \brief Number of bases per byte.
+ common::NumBasesPerByte numBasesPerByte_;
+};
+
+// TODO: Refactor this into a class.
+void detectReadLayout(const RunInfoXml& runInfoXml,
+ const config::SampleSheetCsv& sampleSheetCsv,
+ const std::vector<std::string>& useBasesMaskStrs,
+ size_t minimumTrimmedReadLength,
+ common::CycleNumber read1StartCycle,
+ common::CycleNumber read2StartCycle,
+ common::CycleNumber read1EndCycle,
+ common::CycleNumber read2EndCycle,
+ common::CycleNumber read1UmiLength,
+ common::CycleNumber read2UmiLength,
+ common::CycleNumber read1UmiStartFromCycle,
+ common::CycleNumber read2UmiStartFromCycle,
+ bool trimUmi,
+ bool autoSetToZeroMismatches,
+ config::BarcodeMismatchCountsContainer& componentMaxMismatches,
+ LaneInfosContainer& lanes,
+ const std::vector<std::vector<size_t>>& barcodeLengthsForLane);
+
+} // namespace conversion
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_LAYOUT_HH
+
+
+// http://www.graphviz.org/content/generate-directory-tree-dot
+// digraph LAYOUT {
+// nodesep=0.5;
+// rankdir=LR;
+// fixedsize=true;
+// node [ concentrate=true ];
+//
+// layout [ label="layout" URL="\ref Layout" shape=box ];
+//
+// flowcell_point [ shape = point ];
+// flowcell [ label="flowcell" URL="\ref FlowcellInfo" shape=box ];
+// flowcell_point -> flowcell [ label="1:1" URL="\ref Layout::flowcellInfo_" ];
+//
+// flowcellId_point [ shape = point ];
+// flowcellId [ label="ID" URL="\ref FlowcellInfo::flowcellId_" shape=ellipse ];
+// flowcellId_point -> flowcellId;
+//
+// runNumber_point [ shape = point ];
+// runNumber [ label="run number" URL="\ref FlowcellInfo::runNumber_" shape=ellipse ];
+// runNumber_point -> runNumber;
+//
+// instrument_point [ shape = point ];
+// instrument [ label="instrument" URL="\ref FlowcellInfo::instrument_" shape=ellipse ];
+// instrument_point -> instrument;
+//
+// aggregateTilesFlag_point [ shape = point ];
+// aggregateTilesFlag [ label="aggregated tiles flag" URL="\ref FlowcellInfo::aggregateTilesFlag_" shape=ellipse ];
+// aggregateTilesFlag_point -> aggregateTilesFlag;
+//
+// lane_point [ shape = point ];
+// lane [ label="lane" URL="\ref LaneInfo" shape=box ];
+// lane_point -> lane [ label="1:n" URL="\ref Layout::laneInfos_" ];
+//
+// laneNumber_point [ shape = point ];
+// laneNumber [ label="number" URL="\ref LaneInfo::number_" shape=ellipse ];
+// laneNumber_point -> laneNumber;
+//
+// sample_point [ shape = point ];
+// sample [ label="sample" URL="\ref SampleInfo" shape=box ];
+// sample_point -> sample [ label="1:n" URL="\ref LaneInfo::sampleInfos_" ];
+//
+// sampleNumber_point [ shape = point ];
+// sampleNumber [ label="number" URL="\ref SampleInfo::number_" shape=ellipse ];
+// sampleNumber_point -> sampleNumber;
+//
+// sampleId_point [ shape = point ];
+// sampleId [ label="ID" URL="\ref SampleInfo::sampleId_" shape=ellipse ];
+// sampleId_point -> sampleId;
+//
+// sampleName_point [ shape = point ];
+// sampleName [ label="name" URL="\ref SampleInfo::sampleName_" shape=ellipse ];
+// sampleName_point -> sampleName;
+//
+// barcode_point [ shape = point ];
+// barcode [ label="barcode" URL="\ref Barcode" shape=box ];
+// barcode_point -> barcode [ label="1:n" URL="\ref SampleInfo::barcodes_" ];
+//
+// component_point [ shape = point ];
+// component [ label="component" URL="\ref Barcode::Component" shape=box ];
+// component_point -> component [ label="1:n" URL="\ref Barcode::components_" ];
+//
+// componentLength_point [ shape = point ];
+// componentLength [ label="length" URL="\ref Barcode::Component::length_" shape=ellipse ];
+// componentLength_point -> componentLength;
+//
+// componentValue_point [ shape = point ];
+// componentValue [ label="value" URL="\ref Barcode::Component::value_" shape=ellipse ];
+// componentValue_point -> componentValue;
+//
+// tile_point [ shape = point ];
+// tile [ label="tile" URL="\ref TileInfo" shape=box ];
+// tile_point -> tile [ label="1:n" URL="\ref LaneInfo::tileInfos_" ];
+//
+// tileNumber_point [ shape = point ];
+// tileNumber [ label="number" URL="\ref TileInfo::number_" shape=ellipse ];
+// tileNumber_point -> tileNumber;
+//
+// tileIndex_point [ shape = point ];
+// tileIndex [ label="index" URL="\ref TileInfo::index_" shape=ellipse ];
+// tileIndex_point -> tileIndex;
+//
+// tileClustersCount_point [ shape = point ];
+// tileClustersCount [ label="clusters count" URL="\ref TileInfo::clustersCount_" shape=ellipse ];
+// tileClustersCount_point -> tileClustersCount;
+//
+// tileHaveClustersCount_point [ shape = point ];
+// tileHaveClustersCount [ label="have clusters count flag" URL="\ref TileInfo::haveClustersCount_" shape=ellipse ];
+// tileHaveClustersCount_point -> tileHaveClustersCount;
+//
+// tileSkippedTilesCount_point [ shape = point ];
+// tileSkippedTilesCount [ label="skipped tiles count" URL="\ref TileInfo::skippedTiles" shape=ellipse ];
+// tileSkippedTilesCount_point -> tileSkippedTilesCount;
+//
+// tileSkippedClustersCount_point [ shape = point ];
+// tileSkippedClustersCount [ label="skipped clusters count" URL="\ref TileInfo::skippedClusters_" shape=ellipse ];
+// tileSkippedClustersCount_point -> tileSkippedClustersCount;
+//
+// read_point [ shape = point ];
+// read [ label="read" URL="\ref ReadInfo" shape=box ];
+// read_point -> read [ label="1:n" URL="\ref Layout::readInfos_" ];
+//
+// readNumber_point [ shape = point ];
+// readNumber [ label="number" URL="\ref ReadInfo::number_" shape=ellipse ];
+// readNumber_point -> readNumber;
+//
+// indexReadFlag_point [ shape = point ];
+// indexReadFlag [ label="index read flag" URL="\ref ReadInfo::indexReadFlag_" shape=ellipse ];
+// indexReadFlag_point -> indexReadFlag;
+//
+// maskAdapters_point [ shape = point ];
+// maskAdapters [ label="mask adapters" URL="\ref ReadInfo::maskAdapters_" shape=ellipse ];
+// maskAdapters_point -> maskAdapters;
+//
+// trimAdapters_point [ shape = point ];
+// trimAdapters [ label="trim adapters" URL="\ref ReadInfo::trimAdapters_" shape=ellipse ];
+// trimAdapters_point -> trimAdapters;
+//
+// {
+// rank=same;
+// layout -> flowcell_point -> lane_point -> read_point [ arrowhead=none ];
+// }
+// {
+// rank=same;
+// flowcell -> flowcellId_point -> runNumber_point -> instrument_point -> aggregateTilesFlag_point [ arrowhead=none ];
+// }
+// {
+// rank=same;
+// lane -> laneNumber_point -> sample_point -> tile_point [ arrowhead=none ];
+// }
+// {
+// rank=same;
+// sample -> sampleNumber_point -> sampleId_point -> sampleName_point -> barcode_point [ arrowhead=none ];
+// }
+// {
+// rank=same;
+// barcode -> component_point [ arrowhead=none ];
+// }
+// {
+// rank=same;
+// component -> componentLength_point -> componentValue_point [ arrowhead=none ];
+// }
+// {
+// rank=same;
+// tile -> tileNumber_point -> tileIndex_point -> tileClustersCount_point -> tileHaveClustersCount_point -> tileSkippedTilesCount_point -> tileSkippedClustersCount_point [ arrowhead=none ];
+// }
+// {
+// rank=same;
+// read -> readNumber_point -> indexReadFlag_point -> maskAdapters_point -> trimAdapters_point [ arrowhead=none ];
+// }
+
+
diff --git a/src/cxx/include/layout/ReadInfo.hh b/src/cxx/include/layout/ReadInfo.hh
new file mode 100644
index 0000000..3845b01
--- /dev/null
+++ b/src/cxx/include/layout/ReadInfo.hh
@@ -0,0 +1,207 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ReadInfo.hh
+ *
+ * \brief Declaration of read metadata.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_READINFO_HH
+#define BCL2FASTQ_LAYOUT_READINFO_HH
+
+
+#include <vector>
+#include <string>
+#include <set>
+
+#include "common/Types.hh"
+#include "layout/CycleInfo.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+/// \brief Read metadata.
+class ReadInfo
+{
+public:
+
+ /// \brief Cycle metadata container type definition.
+ typedef std::vector<CycleInfo> CycleInfosContainer;
+
+ /// \brief Adapters container type definition.
+ typedef std::vector<std::string> AdaptersContainer;
+
+public:
+
+ /// \brief Constructor.
+ /// \param number Read number.
+ /// \param readType Type of read (DATA, INDEX).
+ /// \param cyclesToUse cycles to use (cycles to convert for data reads, index bases used in demultiplexing for index reads).
+ /// \param unmaskedCycles All cycles found in RunInfo.xml, including those that were subsequently masked.
+ /// \param umiCycles Cycles to use for the UMI, which is placed in the headers.
+ ReadInfo(common::ReadNumber number,
+ common::ReadType readType,
+ const common::CycleRange& cyclesToUse,
+ const common::CycleRange& unmaskedCycles,
+ const common::CycleRange& umiCycles);
+
+public:
+
+ /// \brief Get read number.
+ /// \return Read number.
+ common::ReadNumber getNumber() const { return number_; }
+
+ /// \brief Determine if this is an index read.
+ /// \retval true This is index read.
+ /// \retval false This is not index read.
+ bool isIndexRead() const { return readType_ == common::ReadType::INDEX; }
+
+ /// \brief Determine if this is a data read.
+ /// \retval true This is a data read.
+ /// \retval false This is not a data read.
+ bool isDataRead() const { return readType_ == common::ReadType::DATA; }
+
+ /// \brief Get the read type
+ /// \retval Read type.
+ common::ReadType getReadType() const { return readType_; }
+
+public:
+
+ /// \brief Get cycles
+ /// \return Unmasked cycles
+ const ReadInfo::CycleInfosContainer& cycleInfos() const { return cycleInfos_; }
+
+ /// \brief Get unmasked cycles
+ /// \return Unmasked cycles
+ const ReadInfo::CycleInfosContainer& unmaskedCycleInfos() const { return unmaskedCycleInfos_; }
+
+ /// \brief Get cycles to load from BCL files
+ /// \return Cycles to load
+ const std::set<CycleInfo>& cyclesToLoad() const { return cyclesToLoad_; }
+
+ /// \brief Get beginning of adapters to be masked.
+ /// \return Iterator to beginning of adapters to be masked.
+ ReadInfo::AdaptersContainer::const_iterator maskAdaptersBegin() const { return maskAdapters_.begin(); }
+
+ /// \brief Get end of adapters to be masked.
+ /// \return Iterator to end of adapters to be trimmed.
+ ReadInfo::AdaptersContainer::const_iterator maskAdaptersEnd() const { return maskAdapters_.end(); }
+
+ /// \brief Get beginning of adapters to be trimmed.
+ /// \return Iterator to beginning of adapters to be trimmed.
+ ReadInfo::AdaptersContainer::const_iterator trimAdaptersBegin() const { return trimAdapters_.begin(); }
+
+ /// \brief Get end of adapters to be trimmed.
+ /// \return Iterator to end of adapters to be trimmed.
+ ReadInfo::AdaptersContainer::const_iterator trimAdaptersEnd() const { return trimAdapters_.end(); }
+
+public:
+
+ /// \brief Get the number of cycles to load from the BCL files. Note that we skip cycles we won't use.
+ /// \return Number of cycles to load
+ size_t getNumCyclesToLoad() const { return cyclesToLoad_.size(); }
+
+ /// \brief Get the BCL buffer offset for this read
+ /// \return BCL buffer offset
+ common::CycleNumber getBclBufferOffset() const { return bclBufferOffset_; }
+
+ /// \brief Get the first cycle to load from the BCL file.
+ /// \return The cycle number of the first cycle.
+ common::CycleNumber getFirstLoadedCycle() const { return cyclesToLoad_.begin()->getNumber(); }
+
+ /// \brief Get the cycle numbers of the UMI cycles.
+ /// \return The range of cycles to use for UMI.
+ const common::CycleRange& getUmiCycles() const { return umiCycleRange_; }
+
+ /// \brief Add adapter to be masked.
+ /// \param maskAdapter Adapter to be masked.
+ void addMaskAdapter(const std::string &maskAdapter) { maskAdapters_.push_back(maskAdapter); }
+
+ /// \brief Add adapter to be trimmed.
+ /// \param trimAdapter Adapter to be trimmed.
+ void addTrimAdapter(const std::string &trimAdapter) { trimAdapters_.push_back(trimAdapter); }
+
+ /// \brief Return true if a FASTQ file should be created for this read.
+ /// \return True if a FASTQ file should be created.
+ bool shouldCreateFastqForRead(bool createFastqsForIndexReads) const { return !cycleInfos_.empty() && (isDataRead() || (createFastqsForIndexReads && isIndexRead())); }
+
+private:
+
+ /// \brief Add the UMI cycles to the read.
+ /// \param The range of UMI cycles to add.
+ void addUmiBases(common::CycleRange umiCycleRange);
+
+ /// \brief Add cycle metadata.
+ /// \param cycleInfo Cycle metadata to be added to read.
+ void addCycle(const CycleInfo &cycleInfo) { cycleInfos_.push_back(cycleInfo); cyclesToLoad_.insert(cycleInfo); }
+
+ /// \brief Add unmasked cycle metadata.
+ /// \param cycleInfo Cycle metadata to be added to read.
+ void addUnmaskedCycle(const CycleInfo &cycleInfo);
+
+ /// \brief Read number.
+ ///
+ /// Source: RunInfo.xml (1..n)
+ common::ReadNumber number_;
+
+ /// \brief Type of read (DATA, INDEX, or UMI).
+ ///
+ /// Source: RunInfo.xml, SampleSheet.csv, command line
+ common::ReadType readType_;
+
+ /// \brief Cycles metadata container. (May drop masked cycles)
+ CycleInfosContainer cycleInfos_;
+
+ /// \brief Unmasked cycles.
+ CycleInfosContainer unmaskedCycleInfos_;
+
+ /// \brief Cycle to load from BCL files.
+ ///
+ /// \Source: RunInfo.xml, --use-bases-mask, SampleSheet.csv.
+ std::set<CycleInfo> cyclesToLoad_;
+
+ /// \brief Adapters to be masked.
+ ///
+ /// Source: SampleSheet.csv
+ AdaptersContainer maskAdapters_;
+
+ /// \brief Adapters to be trimmed.
+ ///
+ /// Source: SampleSheet.csv
+ AdaptersContainer trimAdapters_;
+
+ /// \brief UMI cycle range
+ ///
+ /// Source: SampleSheet.csv
+ common::CycleRange umiCycleRange_;
+
+ /// \brief BCL buffer offset
+ ///
+ /// Source: SampleSheet.csv, --use-bases-mask, RunInfo.xml
+ common::CycleNumber bclBufferOffset_;
+};
+
+/// \brief Read metadata container type definition.
+typedef std::vector<ReadInfo> ReadInfosContainer;
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_READINFO_HH
+
+
diff --git a/src/cxx/include/layout/ReadMetadata.hh b/src/cxx/include/layout/ReadMetadata.hh
new file mode 100644
index 0000000..a428e30
--- /dev/null
+++ b/src/cxx/include/layout/ReadMetadata.hh
@@ -0,0 +1,65 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ReadMetadata.hh
+ *
+ * \brief Declaration of ReadMetadata.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_LAYOUT_READ_METADATA_HH
+#define BCL2FASTQ_LAYOUT_READ_METADATA_HH
+
+#include "common/Types.hh"
+
+#include <vector>
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+ struct ReadMetadata
+ {
+ typedef std::pair<common::CycleNumber,common::CycleNumber> Range;
+
+ ReadMetadata( Range r = Range(0, 0), common::ReadNumber readNumber = 0, common::ReadType readType = common::ReadType::DATA )
+ : readNumber_(readNumber), readType_(readType), firstCycle_(r.first), lastCycle_(r.second), firstUnmaskedCycle_(r.first), lastUnmaskedCycle_(r.second) {}
+ public:
+
+ /// \brief Read number.
+ common::ReadNumber readNumber_;
+
+ /// \brief Read type (DATA, INDEX, or UMI).
+ common::ReadType readType_;
+
+ /// \brief Number of the first cycle.
+ common::CycleNumber firstCycle_;
+
+ /// \brief Number of the last cycle.
+ common::CycleNumber lastCycle_;
+
+ /// \brief Number of first unmasked cycle.
+ common::CycleNumber firstUnmaskedCycle_;
+
+ /// \brief Number of last unmasked cycle.
+ common::CycleNumber lastUnmaskedCycle_;
+ };
+
+ /// \brief Read meta data container type definition.
+ typedef std::vector<ReadMetadata> ReadMetadataContainer;
+
+} // namespace layout
+
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_LAYOUT_READ_METADATA_HH
+
diff --git a/src/cxx/include/layout/RunInfoXml.hh b/src/cxx/include/layout/RunInfoXml.hh
new file mode 100644
index 0000000..703ded3
--- /dev/null
+++ b/src/cxx/include/layout/RunInfoXml.hh
@@ -0,0 +1,120 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file RunInfoXml.hh
+ *
+ * \brief Declaration of RunInfo.xml helper.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_RUNINFOXML_HH
+#define BCL2FASTQ_LAYOUT_RUNINFOXML_HH
+
+
+#include <boost/filesystem/path.hpp>
+#include <boost/property_tree/ptree.hpp>
+
+#include "common/Types.hh"
+#include "layout/LaneInfo.hh"
+#include "layout/ReadMetadata.hh"
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+/// \brief RunInfo.xml file helper.
+class RunInfoXml
+{
+public:
+
+ /// \brief Tile number data container type definition.
+ typedef std::vector<common::TileNumber> TileNumbersContainer;
+
+public:
+
+ /// \brief Constructor.
+ /// \param runInfoData RunInfo XML data.
+ explicit RunInfoXml(boost::property_tree::ptree runInfoData);
+
+public:
+
+ /// \brief Get instrument.
+ /// \return Instrument.
+ std::string getInstrument() const;
+
+ /// \brief Get run id.
+ /// \return Run Id.
+ std::string getRunId() const;
+
+ /// \brief Get run number.
+ /// \return Run number.
+ std::string getRunNumber() const;
+
+ /// \brief Get flowcell ID.
+ /// \return Flowcell ID.
+ std::string getFlowcellId() const;
+
+ /// \brief Get lanes count.
+ /// \return Lanes count
+ common::LaneNumber getLanesCount() const;
+
+ /// \brief Get beginning of read meta data.
+ /// \return Iterator to beginning of read meta data.
+ ReadMetadataContainer::const_iterator readMetadataBegin() const;
+
+ /// \brief Get end of read meta data.
+ /// \return Iterator to end of read meta data.
+ ReadMetadataContainer::const_iterator readMetadataEnd() const;
+
+ /// \brief Get beginning of tile numbers data.
+ /// \return Iterator to beginning of tile numbers data.
+ RunInfoXml::TileNumbersContainer::const_iterator tileNumbersBegin( common::LaneNumber lane ) const;
+
+ /// \brief Get end of tile numbers data.
+ /// \return Iterator to end of tile numbers data.
+ RunInfoXml::TileNumbersContainer::const_iterator tileNumbersEnd( common::LaneNumber lane ) const;
+private:
+
+ /// \brief Initialize read meta data container.
+ void initReadMetadata();
+
+ /// \brief Initialize tile numbers data container.
+ void initTileNumbers();
+
+private:
+
+ /// \brief Property tree holding the actual data.
+ boost::property_tree::ptree ptree_;
+
+ /// \brief Read meta data container.
+ ReadMetadataContainer reads_;
+
+ /// \brief Tile numbers, grouped by lanes
+ std::vector< TileNumbersContainer > tiles_;
+};
+
+
+/// \brief Create RunInfoXml object.
+/// \param runfolderDir Path to runfolder.
+/// \return RunInfoXml object constructed with data from given runfolder.
+RunInfoXml createRunInfoXml(boost::filesystem::path runfolderDir);
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_RUNINFOXML_HH
+
+
diff --git a/src/cxx/include/layout/SampleInfo.hh b/src/cxx/include/layout/SampleInfo.hh
new file mode 100644
index 0000000..c66fcea
--- /dev/null
+++ b/src/cxx/include/layout/SampleInfo.hh
@@ -0,0 +1,159 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file SampleInfo.hh
+ *
+ * \brief Declaration of sample metadata.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_SAMPLEINFO_HH
+#define BCL2FASTQ_LAYOUT_SAMPLEINFO_HH
+
+
+#include <vector>
+#include <string>
+
+#include "common/Types.hh"
+#include "layout/Barcode.hh"
+
+
+namespace bcl2fastq
+{
+namespace layout
+{
+
+
+/// \brief Sample metadata.
+class SampleInfo
+{
+public:
+
+ /// \brief Sample ID type definition.
+ typedef std::string SampleId;
+
+ /// \brief Sample name type definition.
+ typedef std::string SampleName;
+
+ /// \brief Project type definition.
+ typedef std::string Project;
+
+public:
+
+ /// \brief Label to use when no project name has been provided.
+ static const std::string defaultProject;
+ /// \brief Label to use when no sample id has been provided.
+ static const std::string defaultId;
+ /// \brief Label to use when no sample name has been provided.
+ static const std::string defaultName;
+
+ /// \brief Constructor.
+ /// \param number Sample number.
+ /// \param sampleId Sample ID.
+ /// \param sampleName Sample name.
+ /// \param project Project name.
+ SampleInfo(
+ common::SampleNumber number = 0,
+ SampleId sampleId = SampleInfo::defaultId,
+ SampleName sampleName = SampleInfo::defaultName,
+ Project project = SampleInfo::defaultProject
+ );
+
+public:
+
+ /// \brief Get sample number.
+ /// \return Sample number.
+ common::SampleNumber getNumber() const;
+
+ /// \brief Set sample number.
+ void setNumber(common::SampleNumber number) { number_ = number; }
+
+ /// \brief Get sample ID.
+ /// \return Sample ID.
+ const SampleInfo::SampleId& getId() const { return sampleId_; }
+
+ /// \brief Get sample name.
+ /// \return Sample name.
+ const SampleInfo::SampleName& getName() const { return sampleName_; }
+
+ /// \brief Get Project.
+ /// \return Project.
+ const SampleInfo::Project& getProject() const { return project_; }
+
+ /// \brief Get Barcode.
+ /// \return The Barcode.
+ const Barcode& getBarcode(BarcodesContainer::size_type idx) const;
+
+ /// \return whether Sample Id is defined or not.
+ bool hasId() const;
+
+ /// \return whether Sample Name is defined or not.
+ bool hasName() const;
+
+ /// \return whether Project is defined or not.
+ bool hasProject() const;
+
+ /// \return whether sample has any barcodes or not.
+ bool hasBarcodes() const;
+
+ /// \brief Mask the barcode for the given index number
+ /// \param index number.
+ void maskBarcode(common::ReadNumber indexNumber);
+
+public:
+
+ const BarcodesContainer& getBarcodes() const { return barcodes_; }
+
+public:
+
+ /// \brief Add barcode.
+ /// \param barcode Barcode to be added to sample.
+ void addBarcode(const Barcode &barcode);
+
+private:
+
+ /// \brief Sample number.
+ ///
+ /// Source: SampleSheet.csv (0,1..n)
+ common::SampleNumber number_;
+
+ /// \brief Sample ID.
+ ///
+ /// Source: SampleSheet.csv
+ SampleId sampleId_;
+
+ /// \brief Sample name.
+ ///
+ /// Source: SampleSheet.csv
+ SampleName sampleName_;
+
+ /// \brief Project name.
+ ///
+ /// Source: SampleSheet.csv
+ Project project_;
+
+ /// \brief Barcodes.
+ ///
+ /// Source: SampleSheet.csv
+ BarcodesContainer barcodes_;
+
+private:
+ friend std::ostream& operator<<(std::ostream& os, const SampleInfo &sampleInfo);
+
+};
+
+
+} // namespace layout
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_SAMPLEINFO_HH
+
+
diff --git a/src/cxx/include/layout/TileInfo.hh b/src/cxx/include/layout/TileInfo.hh
new file mode 100644
index 0000000..3b520a5
--- /dev/null
+++ b/src/cxx/include/layout/TileInfo.hh
@@ -0,0 +1,134 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file TileInfo.hh
+ *
+ * \brief Declaration of tile metadata.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_TILEINFO_HH
+#define BCL2FASTQ_LAYOUT_TILEINFO_HH
+
+
+#include "common/Types.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+/// \brief Tile metadata.
+class TileInfo
+{
+public:
+
+ /// \brief Default Constructor.
+ TileInfo();
+
+ /// \brief Constructor.
+ /// \param number Tile number.
+ /// \param index Tile index.
+ /// \param skippedTiles Number of skipped tiles before this tile.
+ explicit TileInfo(
+ common::TileNumber number,
+ size_t index,
+ size_t skippedTiles
+ );
+
+ /// \brief Constructor.
+ /// \param number Tile number.
+ /// \param index Tile index.
+ /// \param clustersCount Number of clusters.
+ /// \param skippedTiles Number of skipped tiles before this tile.
+ /// \param skippedClusters Number of skipped clusters before this tile.
+ TileInfo(
+ common::TileNumber number,
+ size_t index,
+ common::ClustersCount clustersCount,
+ size_t skippedTiles,
+ size_t skippedClusters
+ );
+
+public:
+
+ /// \brief Get tile number.
+ /// \return Tile number.
+ common::TileNumber getNumber() const { return number_; }
+
+ /// \brief Get tile index.
+ /// \return Tile index.
+ common::TileNumber getIndex() const { return index_; }
+
+ /// \brief Get clusters count.
+ /// \return Number of clusters on this tile.
+ /// \pre <tt>this->haveClustersCount() == true</tt>
+ common::ClustersCount getClustersCount() const { return clustersCount_; }
+
+ /// \brief Check whether clusters count is known.
+ /// \retval true Clusters count is known.
+ /// \retval false Clusters count is not known.
+ bool haveClustersCount() const { return haveClustersCount_; }
+
+ /// \brief Get number of skipped tiles.
+ /// \return Number of tiles skipped before this tile.
+ size_t getSkippedTilesCount() const { return skippedTiles_; }
+
+ /// \brief Get number of skipped clusters.
+ /// \return Number of clusters skipped before this tile.
+ size_t getSkippedClustersCount() const { return skippedClusters_; }
+
+ /// \brief Overloaded comparison operator.
+ bool operator<( const TileInfo &tile ) const { return index_ < tile.index_; }
+private:
+
+ /// \brief Tile number.
+ ///
+ /// Source: BCI file (Nova), config.xml (HiSeq/MiSeq)
+ common::TileNumber number_;
+
+ /// \brief Tile index.
+ ///
+ /// Source: calculated (index in container of tiles)
+ size_t index_;
+
+ /// \brief Number of clusters on the tile.
+ ///
+ /// Source: BCI file (Nova), config.xml (HiSeq/MiSeq)
+ common::ClustersCount clustersCount_;
+
+ /// \brief Clusters count known flag.
+ ///
+ /// Source: true only if BCI file is present
+ bool haveClustersCount_;
+
+ /// \brief Number of tiles skipped before this tile.
+ ///
+ /// Source: command line argument --tiles
+ size_t skippedTiles_;
+
+ /// \brief Number of clusters skipped before this tile.
+ ///
+ /// Source: command line argument --tiles
+ size_t skippedClusters_;
+};
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_LAYOUT_TILEINFO_HH
+
+
diff --git a/src/cxx/include/layout/UseBasesMask.hh b/src/cxx/include/layout/UseBasesMask.hh
new file mode 100644
index 0000000..5c689a5
--- /dev/null
+++ b/src/cxx/include/layout/UseBasesMask.hh
@@ -0,0 +1,90 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file UseBasesMask.hh
+ *
+ * \brief Declaration of UseBasesMask.
+ *
+ * \author Aaron Day
+ */
+
+#ifndef BCL2FASTQ_LAYOUT_USE_BASES_MASK_HH
+#define BCL2FASTQ_LAYOUT_USE_BASES_MASK_HH
+
+#include "common/Types.hh"
+
+#include "layout/ReadMetadata.hh"
+#include "common/Exceptions.hh"
+
+#include <string>
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+class UseBasesMaskFormatError : public common::RuntimeError
+{
+public:
+ UseBasesMaskFormatError(const std::string& message);
+ UseBasesMaskFormatError(const UseBasesMaskFormatError& other);
+};
+
+class UseBasesMask
+{
+public:
+ UseBasesMask(std::string mask,
+ ReadMetadataContainer::const_iterator readsBegin,
+ ReadMetadataContainer::const_iterator readsEnd);
+
+ bool isEmpty() const { return reads_.empty(); }
+
+ /// \brief Get beginning of read meta data.
+ /// \return Iterator to beginning of read meta data.
+ ReadMetadataContainer::const_iterator readMetadataBegin() const;
+
+ /// \brief Get end of read meta data.
+ /// \return Iterator to end of read meta data.
+ ReadMetadataContainer::const_iterator readMetadataEnd() const;
+private:
+
+ /// \brief Returns true if the character is invalid
+ bool isMaskCharInvalid(char maskChar) const;
+
+ /// \brief Throw an exception if the mask does not meet basic validation criteria.
+ void basicValidate(const std::string& mask) const;
+
+ /// \brief Expand the numbers in the mask, i.e. y3 => yyy
+ void expandNumbers(std::string& mask) const;
+
+ /// \brief Parse the asterisk from the mask
+ void parseAsteriskAndValidateSize(std::string& mask,
+ bool& doesMatchRunInfo,
+ unsigned int totalCycles) const;
+
+ /// \brief Parse the cycles to ignore.
+ /// \return pair<# ignore beginning, # ignore end>
+ std::pair<int, int> parseIgnoreCycles(std::string& mask) const;
+
+ /// \brief Add the cycles to the reads_.
+ void addCycles(const std::string& mask,
+ common::ReadType readType,
+ common::ReadNumber& dataReadsCounter,
+ common::ReadNumber& indexReadsCounter,
+ common::CycleNumber& cyclesCounter);
+
+ /// \brief Read meta data container.
+ ReadMetadataContainer reads_;
+};
+
+} // namespace layout
+
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_LAYOUT_USE_BASES_MASK_HH
+
diff --git a/src/cxx/include/stats/BarcodeHits.hh b/src/cxx/include/stats/BarcodeHits.hh
new file mode 100644
index 0000000..1d5228f
--- /dev/null
+++ b/src/cxx/include/stats/BarcodeHits.hh
@@ -0,0 +1,155 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BarcodeHits.hh
+ *
+ * \brief Declaration of Barcode Hits.
+ *
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_STATS_BARCODEHITS_HH
+#define BCL2FASTQ_STATS_BARCODEHITS_HH
+
+#include <algorithm>
+#include <numeric>
+
+#include <boost/foreach.hpp>
+
+#include "common/Types.hh"
+#include "common/Debug.hh"
+
+
+namespace bcl2fastq
+{
+namespace stats
+{
+
+
+/// \brief Keeping track of barcodes
+class BarcodeHits
+{
+ typedef std::pair< layout::Barcode, unsigned long > Hits;
+ typedef std::map< Hits::first_type, Hits::second_type > Collector;
+public:
+ typedef std::vector< Hits > Popular;
+
+public:
+ BarcodeHits() : raw_(), popular_(), numBarcodes_(0) {}
+
+ void record(const Collector::key_type &sequence)
+ {
+ static const uint64_t UNKNOWN_BARCODE_SAMPLE_FREQ = 20;
+
+ ++numBarcodes_;
+ if (numBarcodes_ % UNKNOWN_BARCODE_SAMPLE_FREQ == 0)
+ {
+ raw_[sequence] += UNKNOWN_BARCODE_SAMPLE_FREQ;
+ }
+ }
+
+ void merge( const BarcodeHits &rhs )
+ {
+ BOOST_FOREACH( const Collector::value_type &barcodeHit, rhs.raw_ )
+ {
+ std::pair< Collector::iterator, bool > ret;
+ ret = raw_.insert ( barcodeHit );
+ if (ret.second==false)
+ {
+ ret.first->second += barcodeHit.second;
+ }
+ }
+
+ numBarcodes_ += rhs.numBarcodes_;
+ }
+
+ void reset()
+ {
+ raw_.clear();
+ Collector().swap(raw_);
+ numBarcodes_ = 0;
+ }
+
+ void findPopular( size_t size, common::LaneNumber laneNumber )
+ {
+ popular_.reserve(size);
+ BOOST_FOREACH( const Collector::value_type &barcodeHit, raw_ )
+ {
+ Popular::iterator insertIt = std::lower_bound( popular_.begin(),
+ popular_.end(),
+ std::make_pair(layout::Barcode(), barcodeHit.second),
+ BarcodeHits::orderByHits );
+
+ if (popular_.size() < size)
+ {
+ popular_.insert(insertIt, barcodeHit);
+ }
+ else if (popular_.end() != insertIt)
+ {
+ popular_.insert(insertIt, barcodeHit);
+ popular_.pop_back();
+ }
+ }
+ const std::string unknownBarcodeCount( popular_.size()
+ ? ("the " + boost::lexical_cast<std::string>(popular_.size()) + " most popular")
+ : "no");
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Taking " << unknownBarcodeCount << " unknown barcodes "
+ << "(out of " << raw_.size() << ") "
+ << "in lane '" << laneNumber << "'" << std::endl;
+ // destroying raw_ once popular_ has been populated, as this takes too much space!
+ reset();
+ }
+
+ const Popular &getPopularBarcodes() const
+ {
+ //BCL2FASTQ_ASSERT_MSG(!popular_.empty(), "Cannot retrieve popular barcodes");
+ return popular_;
+ }
+
+ Hits::second_type sum() const
+ {
+ return std::accumulate( raw_.begin(), raw_.end(), Hits::second_type(0), BarcodeHits::sumHits );
+ }
+
+ static bool orderBySequence( const Hits &lhs, const Hits &rhs )
+ {
+ return lhs.first < rhs.first;
+ }
+ static bool orderByHits( const Hits &lhs, const Hits &rhs )
+ {
+ // we want the Greatest Hits on top!
+ return lhs.second > rhs.second;
+ }
+
+private:
+ static Hits::second_type sumHits( Hits::second_type lhs, const Hits &rhs )
+ {
+ // we want the Greatest Hits on top!
+ return lhs + rhs.second;
+ }
+
+private:
+ /// \brief count of all barcodes
+ Collector raw_;
+
+ /// \brief subset with the most popular barcodes
+ Popular popular_;
+
+ /// \brief Total number of barcodes recorded
+ uint64_t numBarcodes_;
+};
+
+
+} // namespace stats
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_STATS_BARCODEHITS_HH
+
+
diff --git a/src/cxx/include/stats/BarcodeStats.hh b/src/cxx/include/stats/BarcodeStats.hh
new file mode 100644
index 0000000..d647eb4
--- /dev/null
+++ b/src/cxx/include/stats/BarcodeStats.hh
@@ -0,0 +1,106 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BarcodeStats.hh
+ *
+ * \brief Declaration of Barcode Stats.
+ *
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_STATS_BARCODESTATS_HH
+#define BCL2FASTQ_STATS_BARCODESTATS_HH
+
+
+#include "common/Types.hh"
+#include "layout/BarcodeTranslationTable.hh"
+
+#include <boost/noncopyable.hpp>
+#include <numeric>
+
+namespace bcl2fastq
+{
+namespace stats
+{
+
+
+/// \brief Demultiplexing statistics for single barcode.
+class BarcodeStats
+{
+public:
+ // layout::BarcodeTranslationTable::SampleMetadata index;
+ typedef std::pair< layout::BarcodeTranslationTable::SampleMetadata,
+ layout::LaneInfo::TileInfosContainer::size_type > Index;
+
+ Index index;
+
+ common::ClustersCount getBarcodeCount() const { return std::accumulate(barcodeMismatchCounts_.begin(), barcodeMismatchCounts_.end(), 0); }
+
+ // This method is used only if there was no demultiplexing.
+ void setBarcodeCount(common::ClustersCount barcodeCount)
+ {
+ BOOST_ASSERT(getBarcodeCount() == 0);
+
+ barcodeMismatchCounts_[0] = barcodeCount;
+ }
+
+ void incrementBarcodeCount(common::ClustersCount numMismatches)
+ {
+ BOOST_ASSERT(numMismatches <= MAX_MISMATCHES);
+ ++barcodeMismatchCounts_[numMismatches];
+ }
+
+ static const size_t MAX_MISMATCHES = 5;
+
+ // The index into the array is the number of mismatches
+ const std::array<common::ClustersCount, MAX_MISMATCHES+1>& getBarcodeMismatchCounts() const { return barcodeMismatchCounts_; }
+
+private:
+ std::array<common::ClustersCount, MAX_MISMATCHES+1> barcodeMismatchCounts_;
+
+ static Index defaultIndex() { return std::make_pair( layout::BarcodeTranslationTable::SampleMetadata(), 0 ); }
+
+public:
+ BarcodeStats( const layout::BarcodeTranslationTable::SampleMetadata &sampleMetadata,
+ layout::LaneInfo::TileInfosContainer::size_type tileIndex )
+ : index( std::make_pair(sampleMetadata,tileIndex) ), barcodeMismatchCounts_() {}
+ BarcodeStats(const Index &idx)
+ : index(idx), barcodeMismatchCounts_() {}
+ BarcodeStats()
+ : index( defaultIndex() ), barcodeMismatchCounts_() {}
+
+ ~BarcodeStats() { /*BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Stats going away" << std::endl;*/ }
+ BarcodeStats &operator+=(const BarcodeStats &rhs)
+ {
+ for (size_t i = 0; i < MAX_MISMATCHES+1; ++i)
+ {
+ barcodeMismatchCounts_[i] += rhs.barcodeMismatchCounts_[i];
+ }
+ return *this;
+ }
+
+ bool operator<( const BarcodeStats &rhs ) const
+ {
+ return index < rhs.index;
+ }
+
+ void reset()
+ {
+ barcodeMismatchCounts_.fill(0);
+ }
+};
+
+
+} // namespace stats
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_STATS_BARCODESTATS_HH
+
+
diff --git a/src/cxx/include/stats/ConversionStatsXml.hh b/src/cxx/include/stats/ConversionStatsXml.hh
new file mode 100644
index 0000000..726829f
--- /dev/null
+++ b/src/cxx/include/stats/ConversionStatsXml.hh
@@ -0,0 +1,84 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ConversionStatsXml.hh
+ *
+ * \brief Xml Serialization of Conversion statistics.
+ *
+ * \author Mauricio Varea
+ */
+
+#ifndef BCL2FASTQ_STATS_CONVERSION_STATS_XML_HH
+#define BCL2FASTQ_STATS_CONVERSION_STATS_XML_HH
+
+#include "common/Types.hh"
+#include "io/Xml.hh"
+#include "layout/Layout.hh"
+
+#include "stats/TileStats.hpp"
+#include "stats/BarcodeHits.hh"
+
+namespace bcl2fastq
+{
+namespace stats
+{
+
+class ConversionStatsXml : public boost::property_tree::ptree
+{
+public:
+ ConversionStatsXml();
+
+ void addLaneTile(
+ const std::string &flowcellId,
+ const std::string &projectName,
+ const std::string &sampleName,
+ const std::string &barcodeName,
+ common::TileNumber tileNumber,
+ const unsigned lane,
+ const stats::TileBarcodeStats& tileStats);
+
+ void addLaneTileRead(
+ const std::string &flowcellId,
+ const std::string &projectName,
+ const std::string &sampleName,
+ const std::string &barcodeName,
+ common::TileNumber tileNumber,
+ common::ReadNumber readNumber,
+ const unsigned lane,
+ const stats::ReadBarcodeStats& tileStats);
+
+ void addFlowcellLane(
+ const std::string &flowcellId,
+ const unsigned lane,
+ const stats::BarcodeHits::Popular &topUnknownBarcodes);
+
+private:
+ static boost::property_tree::path getPrefix(
+ const std::string &flowcellId,
+ const std::string &projectName,
+ const std::string &sampleName,
+ const std::string &barcodeName,
+ common::TileNumber tileNumber,
+ const unsigned lane );
+};
+
+inline std::ostream &operator<< (std::ostream &os, const ConversionStatsXml &tree)
+{
+ return bcl2fastq::io::serializeAsXml(os, tree);
+}
+
+inline std::istream &operator>> (std::istream &is, ConversionStatsXml &tree)
+{
+ return bcl2fastq::io::parseAsXml(is, dynamic_cast< boost::property_tree::ptree& >(tree) );
+}
+
+
+} //namespace stats
+} //namespace bcl2fastq
+
+#endif //BCL2FASTQ_STATS_CONVERSION_STATS_XML_HH
diff --git a/src/cxx/include/stats/DemultiplexingStatsXml.hh b/src/cxx/include/stats/DemultiplexingStatsXml.hh
new file mode 100644
index 0000000..68b8658
--- /dev/null
+++ b/src/cxx/include/stats/DemultiplexingStatsXml.hh
@@ -0,0 +1,58 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file DemultiplexingStatsXml.hh
+ *
+ * \brief Xml Serialization of Demultiplexing statistics.
+ *
+ * \author Mauricio Varea
+ */
+
+#ifndef BCL2FASTQ_STATS_DEMULTIPLEXING_STATS_XML_HH
+#define BCL2FASTQ_STATS_DEMULTIPLEXING_STATS_XML_HH
+
+#include "io/Xml.hh"
+#include "layout/Layout.hh"
+
+#include "stats/BarcodeStats.hh"
+
+
+namespace bcl2fastq
+{
+namespace stats
+{
+
+class DemultiplexingStatsXml : public boost::property_tree::ptree
+{
+public:
+ DemultiplexingStatsXml();
+
+ void addLaneBarcode(
+ const std::string &flowcellId,
+ const std::string &projectName,
+ const std::string &sampleName,
+ const std::string &barcodeName,
+ const unsigned lane,
+ const stats::BarcodeStats& barcodeStats);
+};
+
+inline std::ostream &operator<< (std::ostream &os, const DemultiplexingStatsXml &tree)
+{
+ return bcl2fastq::io::serializeAsXml(os, tree);
+}
+
+inline std::istream &operator>> (std::istream &is, DemultiplexingStatsXml &tree)
+{
+ return bcl2fastq::io::parseAsXml(is, dynamic_cast< boost::property_tree::ptree& >(tree) );
+}
+
+
+} //namespace stats
+} //namespace bcl2fastq
+
+#endif //BCL2FASTQ_STATS_DEMULTIPLEXING_STATS_XML_HH
diff --git a/src/cxx/include/stats/DemuxReportGenerator.hh b/src/cxx/include/stats/DemuxReportGenerator.hh
new file mode 100644
index 0000000..d1e03d1
--- /dev/null
+++ b/src/cxx/include/stats/DemuxReportGenerator.hh
@@ -0,0 +1,75 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file DemuxReportGenerator.hh
+ *
+ * \brief Triggers the XSLT transformation
+ *
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_STATS_DEMUX_REPORT_GENERATOR_HH
+#define BCL2FASTQ_STATS_DEMUX_REPORT_GENERATOR_HH
+
+#include <boost/filesystem.hpp>
+
+#include "layout/Layout.hh"
+#include "common/Exceptions.hh"
+
+namespace bcl2fastq
+{
+namespace stats
+{
+
+/// \brief Exception for Xslt errors.
+class LibXsltError : public common::RuntimeError
+{
+public:
+ LibXsltError(int errorNumber, const std::string &message)
+ : RuntimeError(errorNumber, message)
+ {}
+ LibXsltError(const std::string &message)
+ : RuntimeError(0, message)
+ {}
+ LibXsltError()
+ : RuntimeError(EINVAL, "libxslt failure")
+ {}
+
+ LibXsltError(const LibXsltError &that)
+ : RuntimeError(that)
+ {}
+public:
+ virtual const char * what() const throw()
+ {
+ return this->getMessage().c_str();
+ }
+};
+
+
+class DemuxReportGenerator
+{
+ const boost::filesystem::path outputDirectoryHtml_;
+
+public:
+ DemuxReportGenerator(
+ const layout::Layout &flowcellLayout,
+ const boost::filesystem::path &outputDirectory
+ );
+
+ void run( const boost::filesystem::path &basecallingStatsXmlPath,
+ const boost::filesystem::path &demultiplexingStatsXmlPath );
+
+ static void makeUniqueSampleNameMap(const layout::LaneInfo& laneInfo,
+ std::map<std::string, std::string>& uniqueSampleNameMap);
+};
+
+} // namespace stats
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_STATS_DEMUX_REPORT_GENERATOR_HH
diff --git a/src/cxx/include/stats/Json.hh b/src/cxx/include/stats/Json.hh
new file mode 100644
index 0000000..602f416
--- /dev/null
+++ b/src/cxx/include/stats/Json.hh
@@ -0,0 +1,119 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Json.hh
+ *
+ * \brief Writes a json file
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_STATS_JSON_HH
+#define BCL2FASTQ_STATS_JSON_HH
+
+#include "common/Debug.hh"
+
+#include <boost/noncopyable.hpp>
+
+#include <limits>
+#include <string>
+#include <vector>
+#include <memory>
+#include <ostream>
+
+namespace bcl2fastq
+{
+namespace stats
+{
+
+class JsonType
+{
+public:
+ JsonType() { }
+ virtual ~JsonType() { }
+
+ virtual void stream(std::ostream& os, int depth = 0) const = 0;
+
+protected:
+ void streamIndent(std::ostream& os, int depth) const;
+};
+
+class JsonString : public JsonType
+{
+public:
+ JsonString() : JsonType() { }
+ JsonString(const std::string& value) : JsonType(), value_(value) { }
+
+ virtual void stream(std::ostream& os, int depth = 0) const;
+
+private:
+ const std::string value_;
+};
+
+class JsonNumber : public JsonType
+{
+public:
+ JsonNumber() : JsonType(), value_(std::numeric_limits<int64_t>::max()) { }
+ JsonNumber(int64_t number) : JsonType(), value_(number) { }
+
+ void add(int64_t number) { BCL2FASTQ_ASSERT_MSG(value_ != std::numeric_limits<int64_t>::max(), "Can't increment a null JsonNumber"); value_ += number; }
+ virtual void stream(std::ostream& os, int depth = 0) const;
+
+ int64_t getValue() const { return value_; }
+
+private:
+ int64_t value_;
+};
+
+class JsonBool : public JsonType
+{
+public:
+ JsonBool(bool value) : JsonType(), value_(value) { }
+
+ virtual void stream(std::ostream& os, int depth = 0) const;
+
+private:
+ bool value_;
+};
+
+class JsonArray : public JsonType
+{
+public:
+ JsonArray() : JsonType(), values_() { }
+ virtual ~JsonArray() { }
+
+ void add(const std::shared_ptr<JsonType>& jsonValue) { values_.push_back(jsonValue); }
+
+ void stream(std::ostream& os, int depth = 0) const;
+
+private:
+ std::vector<std::shared_ptr<JsonType>> values_;
+};
+
+class JsonObject : public JsonType
+{
+public:
+ JsonObject() : JsonType(), attributes_() { }
+ virtual ~JsonObject() { }
+
+ void add(const std::string& key, const std::shared_ptr<JsonType>& jsonValue)
+ {
+ attributes_.push_back(std::make_pair(key, jsonValue));
+ }
+
+ void stream(std::ostream& os, int depth = 0) const;
+
+private:
+ std::vector<std::pair<std::string, std::shared_ptr<JsonType>>> attributes_;
+};
+
+} // namespace stats
+} // namespace bcl2fastq
+
+#endif // BCL2FASTQ_STATS_JSON_HH
diff --git a/src/cxx/include/stats/TileStats.hpp b/src/cxx/include/stats/TileStats.hpp
new file mode 100644
index 0000000..7db10e8
--- /dev/null
+++ b/src/cxx/include/stats/TileStats.hpp
@@ -0,0 +1,237 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file TileStats.hpp
+ *
+ * \brief Declaration of Tile Stats.
+ *
+ * \author Mauricio Varea
+ */
+
+
+#ifndef BCL2FASTQ_STATS_TILESTATS_HPP
+#define BCL2FASTQ_STATS_TILESTATS_HPP
+
+
+#include "common/Types.hh"
+#include "layout/BarcodeTranslationTable.hh"
+#include "layout/Layout.hh"
+
+
+
+namespace bcl2fastq
+{
+namespace stats
+{
+
+
+/// \brief Across-all-reads statistics for a single tile.
+struct AllReadsStats
+{
+ common::ClustersCount clusterCount;
+
+ typedef std::vector<size_t> TrimLengthCount;
+ typedef std::vector<TrimLengthCount> TrimLengthCountForReads;
+
+ TrimLengthCountForReads trimLengthCountForReads;
+
+ AllReadsStats()
+ : clusterCount(0)
+ , trimLengthCountForReads()
+ {}
+
+ /*AllReadsStats(size_t readLength)
+ : clusterCount(0)
+ , trimLengthCount(readLength+1)
+ {}*/
+
+ void init(size_t readLength,
+ size_t numNonIndexReads)
+ {
+ BOOST_ASSERT(readLength != 0);
+
+ trimLengthCountForReads.resize(numNonIndexReads);
+
+ BOOST_FOREACH(TrimLengthCount& trimLengthCount, trimLengthCountForReads)
+ {
+ trimLengthCount.resize(readLength+1);
+ }
+ }
+
+ void RecordAdapterStats(size_t trimmedBases,
+ size_t readNumber)
+ {
+ BOOST_ASSERT(readNumber <= trimLengthCountForReads.size());
+ BOOST_ASSERT(trimmedBases < trimLengthCountForReads[readNumber-1].size());
+
+ trimLengthCountForReads[readNumber - 1][trimmedBases] += 1;
+ }
+
+ AllReadsStats operator+=(const AllReadsStats &rhs)
+ {
+ clusterCount += rhs.clusterCount;
+
+ size_t numReads = rhs.trimLengthCountForReads.size();
+ trimLengthCountForReads.resize(numReads);
+
+ for (size_t i = 0; i < numReads; ++i)
+ {
+ size_t readLength = rhs.trimLengthCountForReads[i].size();
+ trimLengthCountForReads[i].resize(readLength);
+
+ for (size_t j = 0; j < readLength; ++j)
+ {
+ trimLengthCountForReads[i][j] += rhs.trimLengthCountForReads[i][j];
+ }
+ }
+
+ return *this;
+ }
+
+ void reset()
+ {
+ clusterCount = 0;
+ BOOST_FOREACH(std::vector<size_t>& trimLengthCount, std::make_pair(trimLengthCountForReads.begin(), trimLengthCountForReads.end()))
+ {
+ BOOST_FOREACH (size_t& count, std::make_pair(trimLengthCount.begin(), trimLengthCount.end()))
+ {
+ count = 0;
+ }
+ }
+ }
+};
+
+/// \brief Per-read statistics for a single tile.
+struct ReadStats
+{
+ common::Yield yield;
+ common::Yield yieldQ30;
+ common::QualityScore qualityScoreSum;
+
+ ReadStats()
+ : yield(0)
+ , yieldQ30(0)
+ , qualityScoreSum(0)
+ {}
+
+ ReadStats &operator+=(const ReadStats &rhs)
+ {
+ yield += rhs.yield;
+ yieldQ30 += rhs.yieldQ30;
+ qualityScoreSum += rhs.qualityScoreSum;
+ return *this;
+ }
+
+ const ReadStats operator+(const ReadStats &rhs) const
+ {
+ return ReadStats(*this) += rhs;
+ }
+
+ void reset()
+ {
+ yield = 0;
+ yieldQ30 = 0;
+ qualityScoreSum = 0;
+ }
+
+};
+
+namespace detail
+{
+ static const size_t TileStatsCount = 2;
+} // namespace detail
+
+/// \brief Raw and PF stats for a single tile-barcode
+template< typename T >
+class TileStats : public boost::array<T, detail::TileStatsCount>
+{
+public:
+ enum type {RAW=0,PF};
+
+ static const char *str_type[detail::TileStatsCount];
+
+public:
+ TileStats( const boost::array<T, detail::TileStatsCount> &a )
+ : boost::array<T, detail::TileStatsCount>(a) {}
+ TileStats()
+ : boost::array<T, detail::TileStatsCount>() {}
+ TileStats( const T &ts )
+ : boost::array<T, detail::TileStatsCount>() { this->fill(ts); }
+
+ virtual ~TileStats() {}
+
+ TileStats<T> &operator+=( const TileStats<T> &rhs )
+ {
+ size_t length = this->size();
+ for (size_t i = 0; i < length; ++i)
+ {
+ (*this)[i] += rhs[i];
+ }
+ return *this;
+ }
+
+ void reset()
+ {
+ for( typename boost::array< T, detail::TileStatsCount>::size_type i=0;
+ i < boost::array<T, detail::TileStatsCount>::static_size;
+ i++ )
+ {
+ (*this)[i].reset();
+ }
+ }
+};
+
+
+/// \brief stats for a single tile-barcode
+struct TileBarcodeStats : public TileStats<AllReadsStats>
+{
+ TileBarcodeStats( const TileStats<AllReadsStats> &ts ) : TileStats<AllReadsStats>(ts) {}
+ TileBarcodeStats( common::ReadNumber readIndex = 0 ) : TileStats<AllReadsStats>() {}
+ TileBarcodeStats( const AllReadsStats &ts ) : TileStats<AllReadsStats>(ts) {}
+
+ TileBarcodeStats( const TileBarcodeStats &rbs ) : TileStats<AllReadsStats>(rbs) {}
+
+ TileBarcodeStats &operator+=( const TileBarcodeStats &rhs )
+ {
+ TileStats<AllReadsStats>::operator+=(rhs);
+ return *this;
+ }
+};
+
+
+/// \brief stats for a single tile-barcode, annotated by read number
+struct ReadBarcodeStats : public TileStats<ReadStats>
+{
+ layout::ReadInfosContainer::size_type read;
+
+ ReadBarcodeStats( const TileStats<ReadStats> &tbs,
+ layout::ReadInfosContainer::size_type readIndex = 0 )
+ : TileStats<ReadStats>(tbs), read(readIndex) {}
+ ReadBarcodeStats( common::ReadNumber readIndex = 0 )
+ : TileStats<ReadStats>(), read(readIndex) {}
+ ReadBarcodeStats( const ReadStats &ts,
+ layout::ReadInfosContainer::size_type readIndex = 0 )
+ : TileStats<ReadStats>(ts), read(readIndex) {}
+
+ ReadBarcodeStats(const ReadBarcodeStats &rbs) : TileStats<ReadStats>(rbs), read(rbs.read) {}
+
+ ReadBarcodeStats &operator+=( const ReadBarcodeStats &rhs )
+ {
+ TileStats<ReadStats>::operator+=(rhs);
+ return *this;
+ }
+};
+
+
+} // namespace stats
+} // namespace bcl2fastq
+
+
+#endif // BCL2FASTQ_STATS_TILESTATS_HPP
+
+
diff --git a/src/cxx/lib/CMakeLists.txt b/src/cxx/lib/CMakeLists.txt
new file mode 100644
index 0000000..60639ca
--- /dev/null
+++ b/src/cxx/lib/CMakeLists.txt
@@ -0,0 +1,48 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the cxx/lib subfolder.
+##
+## author Roman Petrovski
+##
+################################################################################
+
+
+##
+## List of BCL2FASTQ libraries
+##
+set (BCL2FASTQ_ALL_LIBRARIES
+ common
+ config
+ io
+ data
+ layout
+ stats
+ conversion
+)
+
+##
+## Build all the libraries for the project
+## BCL2FASTQ_AVAILABLE_LIBRARIES is incrementally updated
+##
+
+set (BCL2FASTQ_AVAILABLE_LIBRARIES "")
+set (BCL2FASTQ_ALL_LIBRARY_DIRS "")
+foreach (BCL2FASTQ_LIB_DIR ${BCL2FASTQ_ALL_LIBRARIES})
+ add_subdirectory(${BCL2FASTQ_LIB_DIR})
+ set(BCL2FASTQ_AVAILABLE_LIBRARIES bcl2fastq_${BCL2FASTQ_LIB_DIR} ${BCL2FASTQ_AVAILABLE_LIBRARIES} )
+endforeach (BCL2FASTQ_LIB_DIR)
+
+set (BCL2FASTQ_AVAILABLE_LIBRARIES ${BCL2FASTQ_AVAILABLE_LIBRARIES} PARENT_SCOPE)
+
+
diff --git a/src/cxx/lib/common/CMakeLists.txt b/src/cxx/lib/common/CMakeLists.txt
new file mode 100644
index 0000000..8989845
--- /dev/null
+++ b/src/cxx/lib/common/CMakeLists.txt
@@ -0,0 +1,23 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the cxx/lib/common subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CXX_LIBRARY_CMAKE})
+
+
diff --git a/src/cxx/lib/common/CsvGrammar.cpp b/src/cxx/lib/common/CsvGrammar.cpp
new file mode 100644
index 0000000..d3620f5
--- /dev/null
+++ b/src/cxx/lib/common/CsvGrammar.cpp
@@ -0,0 +1,89 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file CsvGrammar.cpp
+ *
+ * \brief Basic CSV grammar.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <algorithm>
+#include <iterator>
+
+#include <boost/format.hpp>
+#include <boost/bind.hpp>
+#include <boost/filesystem/fstream.hpp>
+
+#include "common/CsvGrammar.hh"
+#include "common/Exceptions.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+CsvGrammarAttribute parseCsvFile(const boost::filesystem::path& csvFile)
+{
+ int errnum;
+
+ boost::filesystem::ifstream csvStream(csvFile);
+ errnum = errno;
+ if (!csvStream)
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errnum, (boost::format("Unable to open '%s' file for reading") % csvFile.string()).str()));
+ }
+ csvStream.unsetf(std::ios::skipws);
+
+ typedef std::vector<char> CsvBuffer;
+ CsvBuffer csvBuffer;
+ errnum = 0;
+ std::copy(
+ std::istream_iterator<CsvBuffer::value_type>(csvStream),
+ std::istream_iterator<CsvBuffer::value_type>(),
+ std::back_inserter(csvBuffer)
+ );
+ errnum = errno;
+ if (!csvStream.eof())
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errnum, "Failed to read " + csvFile.string() + " to the end"));
+ }
+
+ return parseCsvData(csvBuffer.begin(), csvBuffer.end());
+}
+
+CsvGrammarAttribute::value_type::size_type correctCsvColumnsCount(CsvGrammarAttribute &csvData, const std::string &defaultValue)
+{
+ if (csvData.empty())
+ {
+ return 0;
+ }
+
+ const common::CsvGrammarAttribute::value_type::size_type maxCols = std::max_element(
+ csvData.begin(),
+ csvData.end(),
+ boost::bind(&common::CsvGrammarAttribute::value_type::size, _1) < boost::bind(&common::CsvGrammarAttribute::value_type::size, _2)
+ )->size();
+ std::for_each(
+ csvData.begin(),
+ csvData.end(),
+ boost::bind(&common::CsvGrammarAttribute::value_type::resize, _1, maxCols, defaultValue)
+ );
+
+ return maxCols;
+}
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/common/DirectoryValidator.cpp b/src/cxx/lib/common/DirectoryValidator.cpp
new file mode 100644
index 0000000..38a2a54
--- /dev/null
+++ b/src/cxx/lib/common/DirectoryValidator.cpp
@@ -0,0 +1,94 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file DirectoryValidator.cpp
+ *
+ * \brief Implementation of directory validator.
+ *
+ * \author Aaron Day
+ */
+
+
+#include "common/DirectoryValidator.hh"
+
+#include <boost/foreach.hpp>
+#include <boost/filesystem/operations.hpp>
+
+namespace bcl2fastq
+{
+namespace common
+{
+
+DirectoryValidationError::DirectoryValidationError(const boost::filesystem::path& uniquePath,
+ const std::string& uniquePathDescription,
+ const boost::filesystem::path& invalidPath,
+ const std::string& invalidPathDescription)
+: common::RuntimeError(0, createErrorMessage(uniquePath,
+ uniquePathDescription,
+ invalidPath,
+ invalidPathDescription))
+{
+}
+
+DirectoryValidationError::DirectoryValidationError(const DirectoryValidationError& other)
+: common::RuntimeError(other)
+{
+}
+
+std::string DirectoryValidationError::createErrorMessage(const boost::filesystem::path& uniquePath,
+ const std::string& uniquePathDescription,
+ const boost::filesystem::path& invalidPath,
+ const std::string& invalidPathDescription)
+{
+ return invalidPathDescription + " path: '" + invalidPath.string() +
+ "' clashes with " + uniquePathDescription + " path: '" + uniquePath.string() + "'";
+}
+
+
+DirectoryValidator::DirectoryValidator() : uniquePaths_()
+{
+}
+
+DirectoryValidator& DirectoryValidator::getSingleton()
+{
+ static DirectoryValidator instance;
+ return instance;
+}
+
+void DirectoryValidator::addUniquePath(const boost::filesystem::path& path,
+ const std::string& description)
+{
+ uniquePaths_.push_back(std::make_pair(boost::filesystem::absolute(path), description));
+}
+
+void DirectoryValidator::validateNoClash(const boost::filesystem::path& path,
+ const std::string& description) const
+{
+ static const auto pathSeparator = boost::filesystem::path("/").make_preferred().native();
+
+ std::string absolutePath = boost::filesystem::absolute(path / pathSeparator).string();
+
+ typedef std::pair<boost::filesystem::path, std::string> PathDescriptionPair;
+ BOOST_FOREACH(const PathDescriptionPair& uniquePath,
+ std::make_pair(uniquePaths_.begin(), uniquePaths_.end()))
+ {
+ if (absolutePath.size() >= uniquePath.first.string().size() &&
+ absolutePath.substr(0, uniquePath.first.string().size()) == uniquePath.first.string())
+ {
+ BOOST_THROW_EXCEPTION(DirectoryValidationError(uniquePath.first,
+ uniquePath.second,
+ absolutePath,
+ description));
+ }
+ }
+}
+
+} // namespace common
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/common/Exceptions.cpp b/src/cxx/lib/common/Exceptions.cpp
new file mode 100644
index 0000000..a3dd2f2
--- /dev/null
+++ b/src/cxx/lib/common/Exceptions.cpp
@@ -0,0 +1,192 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Exceptions.cpp
+ *
+ * \brief Implementation of the common exception mechanism.
+ *
+ * \author Come Raczy
+ * \author Marek Balint
+ */
+
+
+#include <cstring>
+
+#include <boost/exception/diagnostic_information.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/format.hpp>
+
+#include "common/Exceptions.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+Exception::Exception(int errorNumber, const std::string &message)
+: boost::exception()
+, errorNumber_(errorNumber)
+, message_(message)
+{
+}
+
+Exception::Exception(const Exception &that)
+: boost::exception(that)
+, errorNumber_(that.errorNumber_)
+, message_(that.message_)
+{
+}
+
+Exception::~Exception() throw()
+{
+}
+
+// Restrict assignment operator.
+#if 0
+Exception & Exception::operator=(const Exception &)
+{
+}
+#endif
+
+int Exception::getErrorNumber() const
+{
+ return errorNumber_;
+}
+
+const std::string &Exception::getMessage() const
+{
+ return message_;
+}
+
+std::string Exception::getContext() const
+{
+ const std::string now = boost::posix_time::to_simple_string(
+ boost::posix_time::second_clock::local_time()
+ );
+
+ return (boost::format("%s: %s (%d): %s")
+ % now
+ % std::string(std::strerror(errorNumber_))
+ % errorNumber_
+ % boost::diagnostic_information(*this)
+ ).str();
+}
+
+
+RuntimeError::RuntimeError(int errorNumber, const std::string &message)
+: Exception(errorNumber, message)
+, std::runtime_error(message + ": unknown error " + boost::lexical_cast<std::string>(errorNumber))
+{
+}
+
+RuntimeError::RuntimeError(const std::string &message)
+: Exception(0, message)
+, std::runtime_error(message)
+{
+}
+
+RuntimeError::RuntimeError(const RuntimeError &that)
+: Exception(that)
+, std::runtime_error(that)
+{
+}
+
+const char * RuntimeError::what() const throw()
+{
+ return this->getMessage().c_str();
+}
+
+
+LogicError::LogicError(int errorNumber, const std::string &message)
+: Exception(errorNumber, message)
+, std::logic_error(message)
+{
+}
+
+LogicError::LogicError(const std::string &message)
+: Exception(0, message)
+, std::logic_error(message)
+{
+}
+
+LogicError::LogicError(const LogicError &that)
+: Exception(that)
+, std::logic_error(that)
+{
+}
+
+const char * LogicError::what() const throw()
+{
+ return this->getMessage().c_str();
+}
+
+
+IoError::IoError(int errorNumber, const std::string &message)
+: Exception(errorNumber, message)
+, std::ios_base::failure(message)
+{
+}
+
+IoError::IoError(const std::string &message)
+: Exception(0, message)
+, std::ios_base::failure(message)
+{
+}
+
+IoError::IoError(const IoError &that)
+: Exception(that)
+, std::ios_base::failure(that)
+{
+}
+
+const char * IoError::what() const throw()
+{
+ return this->getMessage().c_str();
+}
+
+
+InputDataError::InputDataError(int errorNumber, const std::string &message)
+: RuntimeError(errorNumber, message)
+{
+}
+
+InputDataError::InputDataError(const std::string &message)
+: RuntimeError(0, message)
+{
+}
+
+InputDataError::InputDataError(const InputDataError &that)
+: RuntimeError(that)
+{
+}
+
+
+OutOfRange::OutOfRange(int errorNumber, const std::string &message)
+: LogicError(errorNumber, message)
+{
+}
+
+OutOfRange::OutOfRange(const std::string &message)
+: LogicError(message)
+{
+}
+
+OutOfRange::OutOfRange(const OutOfRange &that)
+: LogicError(that)
+{
+}
+
+
+} // namespace comomn
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/common/FileSystem.cpp b/src/cxx/lib/common/FileSystem.cpp
new file mode 100644
index 0000000..5280924
--- /dev/null
+++ b/src/cxx/lib/common/FileSystem.cpp
@@ -0,0 +1,64 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FileSystem.cpp
+ *
+ * file system helper utilities.
+ *
+ * \author Roman Petrovski
+ * \author Mauricio Varea
+ */
+#include <boost/foreach.hpp>
+#include <boost/filesystem.hpp>
+
+#include "common/Exceptions.hh"
+#include "common/FileSystem.hh"
+
+namespace bcl2fastq
+{
+namespace common
+{
+
+void createDirectories(std::vector<boost::filesystem::path> createList)
+{
+ std::sort(createList.begin(), createList.end());
+ createList.erase(std::unique(createList.begin(), createList.end()), createList.end());
+ BOOST_FOREACH(const boost::filesystem::path &directory, createList)
+ {
+ if (!boost::filesystem::exists(directory))
+ {
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "creating directory " << directory << std::endl;
+ boost::system::error_code errorCode;
+ if(!boost::filesystem::create_directories(directory, errorCode) &&!exists(directory))
+ {
+ using common::IoError;
+ BOOST_THROW_EXCEPTION(IoError(errorCode.value(), "Failed to create directory " + directory.string()));
+ }
+ }
+ }
+}
+
+
+char makeDirectorySeparatorChar()
+{
+ boost::filesystem::path slash("/");
+ slash.make_preferred();
+ BCL2FASTQ_ASSERT_MSG(1 == slash.string().size(), "Unexpected directory separator char length greater than 1: " << slash)
+ return *slash.string().begin();
+}
+
+static const char DIRECTORY_SEPARATOR_CHAR = makeDirectorySeparatorChar();
+
+char getDirectorySeparatorChar()
+{
+ return DIRECTORY_SEPARATOR_CHAR;
+}
+
+
+} // namespace common
+} // namespace bcl2fastq
diff --git a/src/cxx/lib/common/InstallationPath.cpp b/src/cxx/lib/common/InstallationPath.cpp
new file mode 100644
index 0000000..7517289
--- /dev/null
+++ b/src/cxx/lib/common/InstallationPath.cpp
@@ -0,0 +1,69 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file InstallationPath.cpp
+ *
+ * file Locates the installation path.
+ *
+ * \author Aaron Day
+ */
+
+#ifdef WIN32
+#include <windows.h>
+#endif
+
+#include "common/InstallationPath.hh"
+
+#include "common/Debug.hh"
+
+namespace bcl2fastq
+{
+namespace common
+{
+
+InstallationPath& InstallationPath::getSingleton()
+{
+ static InstallationPath instance;
+ return instance;
+}
+
+
+InstallationPath::InstallationPath()
+ : installationRoot_()
+{
+#ifndef WIN32
+ char szBuffer[10240];
+ BCL2FASTQ_ASSERT_MSG(-1 != readlink("/proc/self/exe", szBuffer, sizeof(szBuffer)), "TODO: handle the readlink error: " << errno);
+
+ installationRoot_ = boost::filesystem::path(szBuffer).parent_path().parent_path();
+#else
+ wchar_t buffer[MAX_PATH];
+ HMODULE hModule = GetModuleHandle(NULL);
+ if (hModule != NULL)
+ {
+ // When passing NULL to GetModuleHandle, it returns handle of exe itself
+ GetModuleFileName(hModule,buffer, (sizeof(buffer)));
+ }
+
+ installationRoot_ = boost::filesystem::path(buffer).parent_path().parent_path();
+#endif
+}
+
+boost::filesystem::path InstallationPath::expandPath(const boost::filesystem::path& path) const
+{
+ if (installationRoot_.empty())
+ {
+ return path;
+ }
+
+ return installationRoot_ / path;
+}
+
+} // namespace common
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/common/Logger.cpp b/src/cxx/lib/common/Logger.cpp
new file mode 100644
index 0000000..8772aeb
--- /dev/null
+++ b/src/cxx/lib/common/Logger.cpp
@@ -0,0 +1,183 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Logger.cpp
+ *
+ * \brief Preprocessor-based logger mechanism.
+ *
+ * \author Roman Petrovski
+ * \author Marek Balint
+ */
+
+
+#include "common/Logger.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+std::istream& operator>>(std::istream& is, LogLevel::value_type &l)
+{
+ std::string token;
+ is >> token;
+
+ if ((token == "none") || (token == "NONE") || (token == "0"))
+ {
+ l = LogLevel::NONE;
+ }
+ else if ((token == "fatal") || (token == "FATAL") || (token == "1"))
+ {
+ l = LogLevel::FATAL;
+ }
+ else if ((token == "error") || (token == "ERROR") || (token == "2"))
+ {
+ l = LogLevel::ERROR_TYPE;
+ }
+ else if ((token == "warning") || (token == "WARNING") || (token == "3"))
+ {
+ l = LogLevel::WARNING;
+ }
+ else if ((token == "info") || (token == "INFO") || (token == "4"))
+ {
+ l = LogLevel::INFO;
+ }
+ else if ((token == "debug") || (token == "DEBUG") || (token == "5"))
+ {
+ l = LogLevel::DEBUG;
+ }
+ else if ((token == "trace") || (token == "TRACE") || (token == "6"))
+ {
+ l = LogLevel::TRACE;
+ }
+ else
+ {
+ is.setstate(std::ios_base::failbit);
+ }
+
+ return is;
+}
+
+std::ostream& operator<<(std::ostream& os, LogLevel::value_type l)
+{
+ switch(l) {
+ case LogLevel::NONE:
+ {
+ //os << "NONE";
+ break;
+ }
+ case LogLevel::FATAL:
+ {
+ os << "FATAL";
+ break;
+ }
+ case LogLevel::ERROR_TYPE:
+ {
+ os << "ERROR";
+ break;
+ }
+ case LogLevel::WARNING:
+ {
+ os << "WARNING";
+ break;
+ }
+ case LogLevel::INFO:
+ {
+ os << "INFO";
+ break;
+ }
+ case LogLevel::DEBUG:
+ {
+ os << "DEBUG";
+ break;
+ }
+ case LogLevel::TRACE:
+ {
+ os << "TRACE";
+ break;
+ }
+ }
+
+ return os;
+}
+
+LogLevel::value_type BCL2FASTQ_MIN_LOG_LEVEL = LogLevel::INFO;
+
+
+namespace detail {
+
+
+boost::recursive_mutex LogStream::clogMutex_;
+unsigned int LogStream::numErrors_ = 0;
+unsigned int LogStream::numWarnings_ = 0;
+
+LogStream::LogStream()
+: lock_(clogMutex_)
+, ias_(std::clog)
+, logLevel_(LogLevel::WARNING)
+{
+ ++numWarnings_;
+}
+
+LogStream::LogStream(LogLevel::value_type logLevel)
+: lock_(clogMutex_)
+, ias_(std::clog)
+, logLevel_(logLevel)
+{
+ if (logLevel_ == ::bcl2fastq::common::LogLevel::WARNING)
+ {
+ ++numWarnings_;
+ }
+ else if (logLevel_ == ::bcl2fastq::common::LogLevel::ERROR_TYPE ||
+ logLevel_ == ::bcl2fastq::common::LogLevel::FATAL)
+ {
+ ++numErrors_;
+ }
+}
+
+LogStream::LogStream(const LogStream &that)
+: lock_(clogMutex_)
+, ias_(std::clog)
+, logLevel_(that.logLevel_)
+{
+}
+
+LogStream::operator bool () const
+{
+ return logLevel_ > BCL2FASTQ_MIN_LOG_LEVEL;
+}
+
+std::ostream & operator <<(std::ostream &os, const ThreadTimestamp &) {
+
+ const boost::posix_time::ptime currentTime = boost::posix_time::second_clock::local_time();
+
+ // IMPORTANT: this is the way to serialize date without causing any dynamic memory operations to occur
+ const std::tm t = boost::posix_time::to_tm(currentTime.time_of_day());
+ const boost::posix_time::ptime::date_type d = currentTime.date();
+ return os
+ << d.year() << '-'
+ << std::setfill('0') << std::setw(2) << d.month().as_number() << '-'
+ << std::setfill('0') << std::setw(2) << d.day() << ' '
+ << std::setfill('0') << std::setw(2) << t.tm_hour << ':'
+ << std::setfill('0') << std::setw(2) << t.tm_min << ':'
+ << std::setfill('0') << std::setw(2) << t.tm_sec << ' '
+ << '[' << boost::this_thread::get_id() << ']' << ' '
+ ;
+}
+
+
+} // namespace detail
+
+
+} // namespace comomn
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/common/Options.cpp b/src/cxx/lib/common/Options.cpp
new file mode 100644
index 0000000..1ab5ba9
--- /dev/null
+++ b/src/cxx/lib/common/Options.cpp
@@ -0,0 +1,223 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Options.cpp
+ *
+ * \brief Implementation of the program options processing.
+ *
+ * \author Come Raczy
+ * \author Mauricio Varea
+ * \author Marek Balint
+ */
+
+
+#include "common/Logger.hh"
+#include "common/Options.hh"
+
+#ifndef WIN32
+#include <sys/ioctl.h>
+#endif
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+static const unsigned short min_description_length = 50;
+
+static unsigned short getStderrLineLength()
+{
+#ifndef WIN32
+ winsize ret = {0,0,0,0};
+ if (-1 == ioctl(STDERR_FILENO, TIOCGWINSZ, &ret))
+ {
+ return boost::program_options::options_description::m_default_line_length;
+ }
+ return ret.ws_col;
+#else
+ return boost::program_options::options_description::m_default_line_length;
+#endif
+}
+
+
+Options::Options()
+: optionsInitialized_(false)
+, namedOptions_("Command-line options",getStderrLineLength(),getStderrLineLength()-min_description_length)
+, unnamedOptions_()
+, positionalOptions_()
+{
+}
+
+Options::~Options()
+{
+}
+
+void Options::initOptions()
+{
+ if (optionsInitialized_)
+ {
+ return;
+ }
+
+ namedOptions_.add_options()(
+ "help,h",
+ "produce help message and exit"
+ );
+ namedOptions_.add_options()(
+ "version,v",
+ "print program version information"
+ );
+ namedOptions_.add_options()(
+ "min-log-level,l",
+ boost::program_options::value<LogLevel::value_type>(&BCL2FASTQ_MIN_LOG_LEVEL)
+ ->default_value(BCL2FASTQ_MIN_LOG_LEVEL),
+ "minimum log level\n"
+ "recognized values: NONE, FATAL, ERROR, WARNING, INFO, DEBUG, TRACE"
+ );
+
+ this->initNamedOptions(namedOptions_);
+ this->initUnnamedOptions(unnamedOptions_);
+ this->initPositionalOptions(positionalOptions_);
+
+ optionsInitialized_ = true;
+}
+
+std::string Options::usage() const
+{
+ std::ostringstream os;
+
+ os << this->usagePrefix() << std::endl;
+ os << namedOptions_ << std::endl;
+ os << this->usageSuffix() << std::endl;
+
+ return os.str();
+}
+
+std::string Options::usageSuffix() const
+{
+ return "";
+}
+
+Options::Action::value_type Options::parse(int argc, char *argv[])
+{
+ boost::program_options::variables_map vm;
+
+ try
+ {
+ this->initOptions();
+ boost::program_options::options_description allOptions("Allowed options");
+ allOptions.add(namedOptions_);
+ allOptions.add(unnamedOptions_);
+
+ boost::program_options::store(
+ boost::program_options::command_line_parser(argc, argv)
+ .options(allOptions)
+ .positional(positionalOptions_)
+ .run(),
+ vm
+ );
+
+ boost::program_options::notify(vm);
+
+ if (vm.count("help"))
+ {
+ return Action::HELP;
+ }
+ else if (vm.count("version"))
+ {
+ return Action::VERSION;
+ }
+
+ std::stringstream log;
+ log << "Command-line invocation: ";
+ std::copy(&argv[0], &argv[argc], std::ostream_iterator<std::string>(log, " "));
+ BCL2FASTQ_LOG(LogLevel::NONE) << log.str() << std::endl;
+ BCL2FASTQ_LOG(LogLevel::INFO) << "Minimum log level: " << BCL2FASTQ_MIN_LOG_LEVEL << std::endl;
+ }
+ catch(const boost::program_options::multiple_values &e)
+ {
+ std::clog << this->usage();
+ std::clog
+ << "Failed to parse the options: " << e.what() << ": "
+ << e.get_option_name() << std::endl
+ ;
+ return Action::ABORT;
+ }
+ catch (const boost::program_options::multiple_occurrences &e)
+ {
+ std::clog << this->usage() << std::endl;
+ std::clog
+ << "Failed to parse the options: " << e.what() << ": "
+ << e.get_option_name() << std::endl
+ ;
+ return Action::ABORT;
+ }
+ catch (const boost::program_options::required_option &e)
+ {
+ std::clog << this->usage() << std::endl;
+ std::clog
+ << "Failed to parse the options: " << e.what() << ": "
+ << e.get_option_name() << std::endl
+ ;
+ return Action::ABORT;
+ }
+ catch (const std::exception &e)
+ {
+ std::clog << this->usage() << std::endl;
+ std::clog
+ << "Failed to parse the options: "
+ << e.what() << std::endl
+ ;
+ return Action::ABORT;
+ }
+
+ // If we were using a standard ini file, we could call this:
+ // store(parse_config_file("example.cfg", desc), vm);
+ this->storeSampleSheetSettings(vm);
+
+ this->postProcess(vm);
+
+ return Action::RUN;
+}
+
+
+std::ostream& operator<<(std::ostream& os, Options::Action::value_type a)
+{
+ switch(a) {
+ case Options::Action::RUN:
+ {
+ os << "RUN";
+ break;
+ }
+ case Options::Action::HELP:
+ {
+ os << "HELP";
+ break;
+ }
+ case Options::Action::VERSION:
+ {
+ os << "VERSION";
+ break;
+ }
+ case Options::Action::ABORT:
+ {
+ os << "ABORT";
+ break;
+ }
+ }
+
+ return os;
+}
+
+
+} // namespace comomn
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/common/ProgramInfo.cpp b/src/cxx/lib/common/ProgramInfo.cpp
new file mode 100644
index 0000000..1b48833
--- /dev/null
+++ b/src/cxx/lib/common/ProgramInfo.cpp
@@ -0,0 +1,87 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ProgramInfo.cpp
+ *
+ * \brief Basic program information.
+ *
+ * \author Mauricio Varea
+ * \author Marek Balint
+ */
+
+
+#include <iostream>
+#include <iomanip>
+
+#include "common/ProgramInfo.hh"
+#include "config.h"
+
+
+namespace bcl2fastq {
+
+
+namespace common {
+
+
+ProgramInfo::ProgramInfo(const std::string &cmd, unsigned int width)
+: command(cmd)
+, binaryName(this->command.filename().string())
+, projectNameShort(BCL2FASTQ_NAME_SHORT)
+, projectNameLong(BCL2FASTQ_NAME_LONG)
+, copyright(BCL2FASTQ_COPYRIGHT)
+, version(BCL2FASTQ_VERSION)
+, width_(width)
+{
+}
+
+
+namespace {
+
+
+/// \brief Output helper.
+/// \param os Output stream to print to.
+/// \param str String to be printed out.
+/// \param width Line width.
+static void outputLine(std::ostream& os, const std::string &str, unsigned int width)
+{
+ os
+ //<< std::setfill(' ') << std::setw((width - str.length()) / 2)
+ << str << std::endl
+ ;
+}
+
+
+} // anonymous namespace
+
+
+std::ostream& operator<<(std::ostream& os, ProgramInfo pi)
+{
+ //os
+ //<< std::setfill('=') << std::setw(pi.width_)
+ //<< "=" << std::endl
+ //;
+
+ outputLine(os, pi.projectNameLong, pi.width_);
+ outputLine(os, pi.binaryName + " v" + pi.version, pi.width_);
+ outputLine(os, pi.copyright, pi.width_);
+
+ //os
+ //<< std::setfill('=') << std::setw(pi.width_)
+ //<< "=" << std::endl
+ //;
+
+ return os;
+}
+
+
+} // namespace common
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/common/SystemCompatibility.cpp b/src/cxx/lib/common/SystemCompatibility.cpp
new file mode 100644
index 0000000..0a5f6ed
--- /dev/null
+++ b/src/cxx/lib/common/SystemCompatibility.cpp
@@ -0,0 +1,63 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file SystemCompatibility.cpp
+ *
+ * \brief Interface layer for system-dependent functionalities.
+ *
+ * \author Come Raczy
+ * \author Marek Balint
+ */
+
+
+#include <signal.h>
+
+#include "common/SystemCompatibility.hh"
+#include "common/Logger.hh"
+
+#ifdef WIN32
+#include <Windows.h>
+#endif
+
+namespace bcl2fastq {
+namespace common {
+
+
+void terminateWithCoreDump()
+{
+ ::raise(SIGSEGV);
+}
+
+
+bool isLittleEndian()
+{
+ const unsigned long v = 0x0706050403020100;
+ const unsigned char * const p = reinterpret_cast<const unsigned char *>(&v);
+ for (unsigned i = 0; i < sizeof(v); ++i)
+ {
+ if (p[i] != i)
+ {
+ return false;
+ }
+ }
+ return true;
+}
+
+
+void adjustMaxFileHandles()
+{
+#ifdef WIN32
+ // increase the maximum number of file handles
+ _setmaxstdio(2048);
+#endif
+}
+
+} // namespace common
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/common/config.h.in b/src/cxx/lib/common/config.h.in
new file mode 100644
index 0000000..1c184a2
--- /dev/null
+++ b/src/cxx/lib/common/config.h.in
@@ -0,0 +1,178 @@
+/**
+ ** BCL to FASTQ file converter
+ ** Copyright (c) 2007-2017 Illumina, Inc.
+ **
+ ** This software is covered by the accompanying EULA
+ ** and certain third party copyright/licenses, and any user of this
+ ** source file is bound by the terms therein.
+ **
+ ** \file config.h
+ **
+ ** \brief Various system-specific definitions (configured by cmake)
+ **
+ **/
+
+/* cpp/include/config.h.cmake. Manually edited */
+/* cpp/include/config.h.in. Generated from configure.ac by autoheader. */
+
+#ifndef HG_BCL2FASTQ_COMMON_CONFIG_H
+#define HG_BCL2FASTQ_COMMON_CONFIG_H
+
+
+/* Endianness of the architecture */
+#cmakedefine BCL2FASTQ_IS_BIG_ENDIAN 1
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#cmakedefine HAVE_INTTYPES_H 1
+
+/* Define to 1 if you have the <malloc.h> header file. */
+#cmakedefine HAVE_MALLOC_H 1
+
+/* Define to 1 if you have the <mcheck.h> header file. */
+#cmakedefine HAVE_MCHECK_H 1
+
+/* Define to 1 if you have the <memory.h> header file. */
+#cmakedefine HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the <signal.h> header file. */
+#cmakedefine HAVE_SIGNAL_H 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#cmakedefine HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#cmakedefine HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#cmakedefine HAVE_STRING_H 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#cmakedefine HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <time.h> header file. */
+#cmakedefine HAVE_TIME_H 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#cmakedefine HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#cmakedefine HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#cmakedefine HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the `floorf' function. */
+#cmakedefine HAVE_FLOORF 1
+
+/* Define to 1 if you have the `round' function. */
+#cmakedefine HAVE_ROUND 1
+
+/* Define to 1 if you have the `roundf' function. */
+#cmakedefine HAVE_ROUNDF 1
+
+/* Define to 1 if you have the `powf' function. */
+#cmakedefine HAVE_POWF 1
+
+/* Define to 1 if you have the `erf' function. */
+#cmakedefine HAVE_ERF 1
+
+/* Define to 1 if you have the `erff' function. */
+#cmakedefine HAVE_ERFF 1
+
+/* Define to 1 if you have the `erfc' function. */
+#cmakedefine HAVE_ERFC 1
+
+/* Define to 1 if you have the `erfcf' function. */
+#cmakedefine HAVE_ERFCF 1
+
+/* Define to 1 if you have the numa library. */
+#cmakedefine HAVE_NUMA 1
+
+/* Define to 1 if you have the `zlib' library */
+#cmakedefine HAVE_ZLIB 1
+
+/* Define to 1 if you have the `sysconf' library */
+#cmakedefine HAVE_SYSCONF 1
+
+/* Define to 1 if you have the `clock' library */
+#cmakedefine HAVE_CLOCK 1
+
+/* Define to 1 if you have the `bzip2' library */
+#cmakedefine HAVE_BZIP2 1
+#cmakedefine HAVE_BZLIB 1
+
+/* Define to 1 if you have the `fftw3f' library */
+#cmakedefine HAVE_FFTW3F 1
+
+/* Define to 1 if you have the `cpgplot' library */
+#cmakedefine HAVE_CPGPLOT 1
+
+/* Define to 1 if you have the `pgplot' library */
+#cmakedefine HAVE_PGPLOT 1
+
+/* Define to 1 if you have the `X11' library */
+#cmakedefine HAVE_X11 1
+
+/* Define to 1 if you have the `g2c' library */
+#cmakedefine HAVE_G2C 1
+
+/* Define to 1 if you have the `boost_xxx_yyy' library
+ (-lboost_xxx_yyy). */
+#cmakedefine HAVE_LIBBOOST_DATE_TIME 1
+#cmakedefine HAVE_LIBBOOST_FILESYSTEM 1
+#cmakedefine HAVE_LIBBOOST_IOSTREAMS 1
+#cmakedefine HAVE_LIBBOOST_PROGRAM_OPTIONS 1
+#cmakedefine HAVE_LIBBOOST_PYTHON 1
+#cmakedefine HAVE_LIBBOOST_REGEX 1
+#cmakedefine HAVE_LIBBOOST_SERIALIZATION 1
+#cmakedefine HAVE_LIBBOOST_SYSTEM 1
+
+/* Define to 1 if you have the `cppunit' library (-lcppunit). */
+#cmakedefine HAVE_CPPUNIT 1
+
+/* Name of package */
+#cmakedefine PACKAGE @PACKAGE@
+
+/* Top level namespace */
+#cmakedefine NAMESPACE @NAMESPACE@
+
+/* Define to the address where bug reports for this package should be sent. */
+/* #undef PACKAGE_BUGREPORT bcl2fastq_bug at illumina.com */
+
+/* Short name */
+#cmakedefine BCL2FASTQ_NAME_SHORT "@BCL2FASTQ_NAME_SHORT@"
+
+/* Long name */
+#cmakedefine BCL2FASTQ_NAME_LONG "@BCL2FASTQ_NAME_LONG@"
+
+/* Version number */
+#cmakedefine BCL2FASTQ_VERSION "@BCL2FASTQ_VERSION@"
+
+/* Version number: marketing */
+#cmakedefine BCL2FASTQ_VERSION_MAJOR "@BCL2FASTQ_VERSION_MAJOR@"
+
+/* Version number: year */
+#cmakedefine BCL2FASTQ_VERSION_MINOR "@BCL2FASTQ_VERSION_MINOR@"
+
+/* Version number: month */
+#cmakedefine BCL2FASTQ_VERSION_PATCH "@BCL2FASTQ_VERSION_PATCH@"
+
+/* Version number: day */
+#cmakedefine BCL2FASTQ_VERSION_BUILD "@BCL2FASTQ_VERSION_BUILD@"
+
+/* Copyright one-liner */
+#cmakedefine BCL2FASTQ_COPYRIGHT "@BCL2FASTQ_COPYRIGHT@"
+
+/* Location of installed share files */
+#define BCL2FASTQ_DATADIR "@BCL2FASTQ_DATADIR@"
+
+/* Location of installed bin files */
+#define BCL2FASTQ_VERSION_FULL "@BCL2FASTQ_VERSION_FULL@"
+
+/* Define to empty if `const' does not conform to ANSI C. */
+#undef const
+
+
+#endif /* HG_BCL2FASTQ_COMMON_CONFIG_H */
+
+
diff --git a/src/cxx/lib/common/cppunit/CMakeLists.txt b/src/cxx/lib/common/cppunit/CMakeLists.txt
new file mode 100644
index 0000000..ec43d99
--- /dev/null
+++ b/src/cxx/lib/common/cppunit/CMakeLists.txt
@@ -0,0 +1,23 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for any cppunit subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CPPUNIT_CMAKE})
+
+
diff --git a/src/cxx/lib/common/cppunit/RegistryNames.txt b/src/cxx/lib/common/cppunit/RegistryNames.txt
new file mode 100644
index 0000000..d555547
--- /dev/null
+++ b/src/cxx/lib/common/cppunit/RegistryNames.txt
@@ -0,0 +1 @@
+Exceptions
diff --git a/src/cxx/lib/common/cppunit/testExceptions.cpp b/src/cxx/lib/common/cppunit/testExceptions.cpp
new file mode 100644
index 0000000..3d81c53
--- /dev/null
+++ b/src/cxx/lib/common/cppunit/testExceptions.cpp
@@ -0,0 +1,40 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testExceptions.cpp
+ *
+ * \brief Commmon exception mechanism cppunit test declarations.
+ *
+ * \author Roman Petrovski
+ * \author Marek Balint
+ */
+
+
+#include <iostream>
+#include <sstream>
+#include <string>
+
+#include "RegistryName.hh"
+#include "testExceptions.hh"
+
+
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(TestExceptions, registryName("Exceptions"));
+
+
+void TestExceptions::setUp()
+{
+}
+
+void TestExceptions::tearDown()
+{
+}
+
+void TestExceptions::testErrorNumber()
+{
+ CPPUNIT_ASSERT(true);
+}
diff --git a/src/cxx/lib/common/cppunit/testExceptions.hh b/src/cxx/lib/common/cppunit/testExceptions.hh
new file mode 100644
index 0000000..8bb7a09
--- /dev/null
+++ b/src/cxx/lib/common/cppunit/testExceptions.hh
@@ -0,0 +1,48 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testExceptions.hh
+ *
+ * \brief Commmon exception mechanism cppunit test declarations.
+ *
+ * \author Roman Petrovski
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_COMMON_TEST_EXCEPTIONS_HH
+#define BCL2FASTQ_COMMON_TEST_EXCEPTIONS_HH
+
+
+#include <string>
+
+#include <cppunit/extensions/HelperMacros.h>
+
+#include "common/Exceptions.hh"
+
+
+/// \brief Test suite for exceptions.
+class TestExceptions : public CppUnit::TestFixture
+{
+private:
+
+ CPPUNIT_TEST_SUITE(TestExceptions);
+ CPPUNIT_TEST(testErrorNumber);
+ CPPUNIT_TEST_SUITE_END();
+
+public:
+
+ void setUp();
+ void tearDown();
+ void testErrorNumber();
+};
+
+
+#endif // #ifndef BCL2FASTQ_COMMON_TEST_EXCEPTIONS_HH
+
+
diff --git a/src/cxx/lib/config/Bcl2FastqOptions.cpp b/src/cxx/lib/config/Bcl2FastqOptions.cpp
new file mode 100644
index 0000000..0edb6ad
--- /dev/null
+++ b/src/cxx/lib/config/Bcl2FastqOptions.cpp
@@ -0,0 +1,608 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Bcl2FastqOptions.cpp
+ *
+ * \brief Implementation of the Bcl2Fastq options processing.
+ *
+ * \author Mauricio Varea
+ * \author Marek Balint
+ */
+
+
+#include <string>
+#include <sstream>
+#include <iostream>
+
+#include <cmath>
+
+#include <boost/filesystem/operations.hpp>
+#include <boost/tokenizer.hpp>
+#include <boost/lexical_cast.hpp>
+
+#include "config.h"
+#include "config/Bcl2FastqOptions.hh"
+#include <boost/algorithm/string.hpp>
+
+
+namespace bcl2fastq {
+
+
+namespace config {
+
+const std::string createFastqForIndexReadsSetting = "create-fastq-for-index-reads";
+const std::string findAdaptersWithSlidingWindowSetting = "find-adapters-with-sliding-window";
+const std::string reverseComplementSetting = "write-fastq-reverse-complement";
+const std::string useBasesMaskSetting = "use-bases-mask";
+
+// This gets around a bug in boost::program_options::value<BoostPath>
+// which doesn't allow spaces in the path name
+std::istream& operator>>(std::istream& is, BoostPath& boostPath)
+{
+ std::string pathName;
+
+ std::getline(is, pathName);
+ boostPath.path_ = pathName;
+
+ return is;
+}
+
+Bcl2FastqOptions::Bcl2FastqOptions()
+: inputDir_()
+, runfolderDir_()
+, intensitiesDir_()
+, outputDir_()
+, interopDir_()
+, statsDir_()
+, reportsDir_()
+, sampleSheetPath_()
+, bclLoaderThreadsCount_(0)
+, fastqCreatorThreadsCount_(0)
+, fastqWriterThreadsCount_(0)
+, tilesFilterString_()
+, minimumTrimmedReadLength_()
+, useBasesMasks_()
+, maskShortAdapterReads_()
+, adapterStringency_()
+, generateReverseComplementFastqs_()
+, includeNonPfClusters_()
+, ignoreMissingBcls_()
+, ignoreMissingFilters_()
+, ignoreMissingPositions_()
+, ignoreMissingControls_()
+, createFastqsForIndexReads_()
+, findAdaptersWithSlidingWindow_(false)
+, noBgzfCompression_(false)
+, fastqCompressionLevel_(1)
+, barcodeMismatchCountsDelimitedString_()
+, barcodeMismatchCounts_()
+, autoSetToZeroBarcodeMismatches_(false)
+, noLaneSplitting_(false)
+, sampleSheet_()
+{
+}
+
+common::ThreadVector::size_type Bcl2FastqOptions::getBclLoaderThreadsCount() const
+{
+ return bclLoaderThreadsCount_;
+}
+
+common::ThreadVector::size_type Bcl2FastqOptions::getFastqCreatorThreadsCount() const
+{
+ return fastqCreatorThreadsCount_;
+}
+
+common::ThreadVector::size_type Bcl2FastqOptions::getAppliedFastqWriterThreadsCount() const
+{
+ return getFastqWriterThreadsCount() == 0
+ ? getBclLoaderThreadsCount()
+ : getFastqWriterThreadsCount();
+}
+
+common::ThreadVector::size_type Bcl2FastqOptions::getFastqWriterThreadsCount() const
+{
+ return fastqWriterThreadsCount_;
+}
+
+std::vector<std::string> Bcl2FastqOptions::getTilesFilterList() const
+{
+ std::vector<std::string> tilesFilterList;
+
+ if (!tilesFilterString_.empty())
+ {
+ boost::split(tilesFilterList, tilesFilterString_, boost::is_any_of(","));
+
+ std::for_each(tilesFilterList.begin(),
+ tilesFilterList.end(),
+ boost::bind(boost::algorithm::trim<std::string>, _1, std::locale()));
+ }
+
+ return tilesFilterList;
+}
+
+std::size_t Bcl2FastqOptions::getMinimumTrimmedReadLength() const
+{
+ return minimumTrimmedReadLength_;
+}
+
+const Bcl2FastqOptions::UseBasesMaskContainer& Bcl2FastqOptions::getUseBasesMasks() const
+{
+ return useBasesMasks_;
+}
+
+std::size_t Bcl2FastqOptions::getMaskShortAdapterReads() const
+{
+ return maskShortAdapterReads_;
+}
+
+float Bcl2FastqOptions::getAdapterStringency() const
+{
+ return adapterStringency_;
+}
+
+bool Bcl2FastqOptions::getIgnoreMissingBcls() const
+{
+ return ignoreMissingBcls_;
+}
+
+bool Bcl2FastqOptions::getIgnoreMissingFilters() const
+{
+ return ignoreMissingFilters_;
+}
+
+bool Bcl2FastqOptions::getIgnoreMissingPositions() const
+{
+ return ignoreMissingPositions_;
+}
+
+bool Bcl2FastqOptions::getIgnoreMissingControls() const
+{
+ return ignoreMissingControls_;
+}
+
+bool Bcl2FastqOptions::getGenerateReverseComplementFastqs() const
+{
+ return generateReverseComplementFastqs_;
+}
+
+bool Bcl2FastqOptions::getIncludeNonPfClusters() const
+{
+ return includeNonPfClusters_;
+}
+
+bool Bcl2FastqOptions::getCreateFastqsForIndexReads() const
+{
+ return createFastqsForIndexReads_;
+}
+
+bool Bcl2FastqOptions::useBgzfCompression() const
+{
+ return !noBgzfCompression_;
+}
+
+int Bcl2FastqOptions::getFastqCompressionLevel() const
+{
+ return fastqCompressionLevel_;
+}
+
+void Bcl2FastqOptions::initNamedOptions(boost::program_options::options_description &options)
+{
+ options.add_options()(
+ "input-dir,i",
+ boost::program_options::value<BoostPath>(&inputDir_)
+ ->default_value(BoostPath(),"<runfolder-dir>/Data/Intensities/BaseCalls/"),
+ "path to input directory"
+ );
+
+ options.add_options()(
+ "runfolder-dir,R",
+ boost::program_options::value<BoostPath>(&runfolderDir_)
+ ->default_value(BoostPath(boost::filesystem::current_path()),"./"),
+ "path to runfolder directory"
+ );
+
+ options.add_options()(
+ "intensities-dir",
+ boost::program_options::value<BoostPath>(&intensitiesDir_)
+ ->default_value(BoostPath(),"<input-dir>/../"),
+ "path to intensities directory\n"
+ "If intensities directory is specified, --input-dir must also be specified."
+ );
+
+ options.add_options()(
+ "output-dir,o",
+ boost::program_options::value<BoostPath>(&outputDir_)
+ ->default_value(BoostPath(),"<input-dir>"),
+ "path to demultiplexed output"
+ );
+
+ options.add_options()(
+ "interop-dir",
+ boost::program_options::value<BoostPath>(&interopDir_)
+ ->default_value(BoostPath(),"<runfolder-dir>/InterOp/"),
+ "path to demultiplexing statistics directory"
+ );
+
+ options.add_options()(
+ "stats-dir",
+ boost::program_options::value<BoostPath>(&statsDir_)
+ ->default_value(BoostPath(),"<output-dir>/Stats/"),
+ "path to human-readable demultiplexing statistics directory"
+ );
+
+ options.add_options()(
+ "reports-dir",
+ boost::program_options::value<BoostPath>(&reportsDir_)
+ ->default_value(BoostPath(),"<output-dir>/Reports/"),
+ "path to reporting directory"
+ );
+
+ options.add_options()(
+ "sample-sheet",
+ boost::program_options::value<BoostPath>(&sampleSheetPath_)
+ ->default_value(BoostPath(),"<runfolder-dir>/SampleSheet.csv"),
+ "path to the sample sheet"
+ );
+
+ options.add_options()(
+ "loading-threads,r",
+ boost::program_options::value<common::ThreadVector::size_type>(&bclLoaderThreadsCount_)
+ ->default_value(4),
+ "number of threads used for loading BCL data"
+ );
+
+ options.add_options()(
+ "processing-threads,p",
+ boost::program_options::value<common::ThreadVector::size_type>(&fastqCreatorThreadsCount_),
+ "number of threads used for processing demultiplexed data"
+ );
+
+ options.add_options()(
+ "writing-threads,w",
+ boost::program_options::value<common::ThreadVector::size_type>(&fastqWriterThreadsCount_)
+ ->default_value(4),
+ "number of threads used for writing FASTQ data.\n"
+ "This should not be set higher than the number of samples. "
+ "If set =0 then these threads will be placed in the "
+ "same pool as the loading threads, and the number "
+ "of shared threads will be determined by --loading-threads."
+ );
+
+ options.add_options()(
+ "tiles",
+ boost::program_options::value<std::string>(&tilesFilterString_),
+ "comma-separated list of regular expressions to select only a subset of the tiles available in the flow-cell."
+ " Multiple entries allowed, each applies to the corresponding base-calls.\n"
+ "For example:\n"
+ " * to select all the tiles ending with '5' in all lanes:\n --tiles [0-9][0-9][0-9]5\n"
+ " * to select tile 2 in lane 1 and all the tiles in the other lanes:\n --tiles s_1_0002,s_[2-8]"
+ );
+
+ options.add_options()(
+ "minimum-trimmed-read-length",
+ boost::program_options::value<std::size_t>(&minimumTrimmedReadLength_)
+ ->default_value(35),
+ "minimum read length after adapter trimming"
+ );
+
+ options.add_options()(
+ useBasesMaskSetting.c_str(),
+ boost::program_options::value<UseBasesMaskContainer>(&useBasesMasks_)
+ ->default_value(UseBasesMaskContainer(), ""),
+ "specifies how to use each cycle."
+ );
+
+ options.add_options()(
+ "mask-short-adapter-reads",
+ boost::program_options::value<std::size_t>(&maskShortAdapterReads_)
+ ->default_value(22),
+ "smallest number of remaining bases (after masking bases below the minimum trimmed read length) below which whole read is masked"
+ );
+
+ options.add_options()(
+ "adapter-stringency",
+ boost::program_options::value<float>(&adapterStringency_)
+ ->default_value(0.9,"0.9"),
+ "adapter stringency"
+ );
+
+ options.add_options()(
+ "ignore-missing-bcls",
+ boost::program_options::bool_switch(&ignoreMissingBcls_)
+ ->default_value(false),
+ "assume 'N'/'#' for missing calls"
+ );
+
+ options.add_options()(
+ "ignore-missing-filter",
+ boost::program_options::bool_switch(&ignoreMissingFilters_)
+ ->default_value(false),
+ "assume 'true' for missing filters"
+ );
+
+ options.add_options()(
+ "ignore-missing-positions",
+ boost::program_options::bool_switch(&ignoreMissingPositions_)
+ ->default_value(false),
+ "assume [0,i] for missing positions, where i is incremented starting from 0"
+ );
+
+ options.add_options()(
+ "ignore-missing-controls",
+ boost::program_options::bool_switch(&ignoreMissingControls_)
+ ->default_value(false),
+ "(deprecated) assume 0 for missing controls"
+ );
+
+ options.add_options()(
+ reverseComplementSetting.c_str(),
+ boost::program_options::bool_switch(&generateReverseComplementFastqs_)
+ ->default_value(false),
+ "generate FASTQs containing reverse complements of actual data"
+ );
+
+ options.add_options()(
+ "with-failed-reads",
+ boost::program_options::bool_switch(&includeNonPfClusters_)
+ ->default_value(false),
+ "include non-PF clusters"
+ );
+
+ options.add_options()(
+ createFastqForIndexReadsSetting.c_str(),
+ boost::program_options::bool_switch(&createFastqsForIndexReads_)
+ ->default_value(false),
+ "create FASTQ files also for index reads"
+ );
+
+ options.add_options()(
+ findAdaptersWithSlidingWindowSetting.c_str(),
+ boost::program_options::bool_switch(&findAdaptersWithSlidingWindow_)
+ ->default_value(false),
+ "find adapters with simple sliding window algorithm"
+ );
+
+ options.add_options()(
+ "no-bgzf-compression",
+ boost::program_options::bool_switch(&noBgzfCompression_)
+ ->default_value(false),
+ "turn off BGZF compression for FASTQ files"
+ );
+
+ options.add_options()(
+ "fastq-compression-level",
+ boost::program_options::value<int>(&fastqCompressionLevel_)
+ ->default_value(4),
+ "zlib compression level (1-9) used for FASTQ files"
+ );
+
+ options.add_options()(
+ "barcode-mismatches",
+ boost::program_options::value<std::string>(&barcodeMismatchCountsDelimitedString_)
+ ->default_value("1"),
+ "number of allowed mismatches per index\n"
+ "Multiple, comma delimited, entries allowed. Each entry is applied to the corresponding index; "
+ "last entry applies to all remaining indices.\n"
+ "Accepted values: 0, 1, 2."
+ );
+
+ options.add_options()(
+ "no-lane-splitting",
+ boost::program_options::bool_switch(&noLaneSplitting_)
+ ->default_value(false),
+ "do not split fastq files by lane."
+ );
+}
+
+void Bcl2FastqOptions::initUnnamedOptions(boost::program_options::options_description &options)
+{
+ options.add_options()(
+ "auto-set-to-zero-barcode-mismatches",
+ boost::program_options::bool_switch(&autoSetToZeroBarcodeMismatches_)
+ ->default_value(false),
+ "automatically set allowed barcode mismatches to 0 on collision"
+ );
+}
+
+void Bcl2FastqOptions::initPositionalOptions(boost::program_options::positional_options_description &options)
+{
+}
+
+std::string Bcl2FastqOptions::usagePrefix() const
+{
+ std::ostringstream os;
+
+ os << "Usage:" << std::endl;
+ os << " " << BCL2FASTQ_NAME_SHORT << " [options]" << std::endl;
+
+ return os.str();
+}
+
+std::string Bcl2FastqOptions::usageSuffix() const
+{
+ return "";
+}
+
+void Bcl2FastqOptions::storeSampleSheetSettings(boost::program_options::variables_map &vm)
+{
+ /// \todo Refactoring: Is there a better way for obtaining platform-independent path separator? (Consider CMake)
+ auto pathSeparator = boost::filesystem::path("/").make_preferred().native();
+
+ if (inputDir_.path_.empty())
+ {
+ inputDir_.path_ =
+ runfolderDir_.path_
+ / boost::filesystem::path("Data")
+ / boost::filesystem::path("Intensities")
+ / boost::filesystem::path("BaseCalls")
+ / pathSeparator
+ ;
+ }
+
+ // output-dir
+ if (outputDir_.path_.empty())
+ {
+ outputDir_.path_ = inputDir_.path_;
+ }
+
+ // sample sheet location
+ if (sampleSheetPath_.path_.empty())
+ {
+ sampleSheetPath_.path_ =
+ runfolderDir_.path_
+ / boost::filesystem::path("SampleSheet.csv");
+ }
+ else if (!boost::filesystem::exists(sampleSheetPath_.path_))
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "Sample sheet " << sampleSheetPath_.path_ << " does not exist" << std::endl;
+ }
+
+ if (!(sampleSheet_ = config::createSampleSheetCsv(sampleSheetPath_.path_,
+ outputDir_.path_)))
+ {
+ return;
+ }
+
+ // For settings not passed on the command line, look for them in the sample sheet.
+ if (vm[createFastqForIndexReadsSetting].defaulted())
+ {
+ SampleSheetCsv::TriState createFastqForIndexReads = sampleSheet_->createFastqForIndexReads();
+ if (createFastqForIndexReads != SampleSheetCsv::INDETERMINATE)
+ {
+ createFastqsForIndexReads_ = (createFastqForIndexReads == SampleSheetCsv::TRUE);
+ }
+
+ }
+ if (vm[findAdaptersWithSlidingWindowSetting].defaulted())
+ {
+ SampleSheetCsv::TriState findAdaptersWithIndels = sampleSheet_->findAdaptersWithIndels();
+ if (findAdaptersWithIndels != SampleSheetCsv::INDETERMINATE)
+ {
+ findAdaptersWithSlidingWindow_ = (findAdaptersWithIndels != SampleSheetCsv::TRUE);
+ }
+ }
+ if (vm[reverseComplementSetting].defaulted())
+ {
+ SampleSheetCsv::TriState reverseComplement = sampleSheet_->generateReverseComplementFastqs();
+ if (reverseComplement != SampleSheetCsv::INDETERMINATE)
+ {
+ generateReverseComplementFastqs_ = (reverseComplement == SampleSheetCsv::TRUE);
+ }
+ }
+}
+
+void Bcl2FastqOptions::postProcess(boost::program_options::variables_map &vm)
+{
+ /// \todo Refactoring: Is there a better way for obtaining platform-independent path separator? (Consider CMake)
+ auto pathSeparator = boost::filesystem::path("/").make_preferred().native();
+
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Runfolder path: '" << runfolderDir_.path_.string() << "'" << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Input path: '" << inputDir_.path_.string() << "'" << std::endl;
+
+ // intensities-dir
+ if (intensitiesDir_.path_.empty())
+ {
+ intensitiesDir_.path_ = boost::filesystem::path(inputDir_.path_ / pathSeparator).parent_path().parent_path() / pathSeparator;
+ }
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Intensities path: '" << intensitiesDir_.path_.string() << "'" << std::endl;
+
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Output path: '" << outputDir_.path_.string() << "'" << std::endl;
+
+ // interop-dir
+ if (interopDir_.path_.empty())
+ {
+ interopDir_.path_ =
+ runfolderDir_.path_
+ / boost::filesystem::path("InterOp")
+ / pathSeparator
+ ;
+ }
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "InterOp path: '" << interopDir_.path_.string() << "'" << std::endl;
+
+ // stats-dir
+ if (statsDir_.path_.empty())
+ {
+ statsDir_.path_ =
+ outputDir_.path_
+ / boost::filesystem::path("Stats")
+ / pathSeparator
+ ;
+ }
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Stats path: '" << statsDir_.path_.string() << "'" << std::endl;
+
+ // reports-dir
+ if (reportsDir_.path_.empty())
+ {
+ reportsDir_.path_ =
+ outputDir_.path_
+ / boost::filesystem::path("Reports")
+ / pathSeparator
+ ;
+ }
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Reports path: '" << reportsDir_.path_.string() << "'" << std::endl;
+
+ // thread counts
+ {
+ std::size_t cpusCount = boost::thread::hardware_concurrency();
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Detected CPUs: " << cpusCount << std::endl;
+
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Loading threads: " << bclLoaderThreadsCount_ << std::endl;
+ if (fastqCreatorThreadsCount_ == 0)
+ {
+ fastqCreatorThreadsCount_ = cpusCount;
+ }
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Processing threads: " << fastqCreatorThreadsCount_ << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Writing threads: " << getAppliedFastqWriterThreadsCount() << std::endl;
+ }
+
+ // barcode mismatches
+ boost::tokenizer<boost::char_separator<char> > tknzr(
+ barcodeMismatchCountsDelimitedString_,
+ boost::char_separator<char>(",:|+")
+ );
+ std::transform(
+ tknzr.begin(),
+ tknzr.end(),
+ std::back_inserter(barcodeMismatchCounts_),
+ static_cast<std::size_t (*) (const std::string&)>(&boost::lexical_cast<std::size_t>)
+ );
+
+ std::stringstream barcodeMismatchCountsDelimitedString;
+ std::copy(barcodeMismatchCounts_.begin(), barcodeMismatchCounts_.end(), std::ostream_iterator<MismatchesCount>(barcodeMismatchCountsDelimitedString, " "));
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Allowed barcode mismatches: " << barcodeMismatchCountsDelimitedString.str() << std::endl;
+
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Tiles: " << (tilesFilterString_.empty() ? "<ALL>" : tilesFilterString_) << std::endl;;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Minimum trimmed read length: " << minimumTrimmedReadLength_ << std::endl;
+
+ std::ostringstream useBasesMaskInfoStr;
+ BOOST_FOREACH(const std::string useBasesMask, useBasesMasks_)
+ {
+ useBasesMaskInfoStr << std::endl << "\t" << useBasesMask;
+ }
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Use bases masks: " << (useBasesMasks_.empty() ? "<NONE>" : useBasesMaskInfoStr.str()) << std::endl;
+
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Mask short adapter reads: " << maskShortAdapterReads_ << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Adapter stringency: " << adapterStringency_ << std::endl;
+ const std::string trimMethodText = (findAdaptersWithSlidingWindow_) ?
+ "Sliding window" : "Allow matches with indels";
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Adapter trimming method: " << trimMethodText << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Ignore missing BCLs: " << (ignoreMissingBcls_ ? "YES" : "NO") << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Ignore missing filters: " << (ignoreMissingFilters_ ? "YES" : "NO") << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Ignore missing positions: " << (ignoreMissingPositions_ ? "YES" : "NO") << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Ignore missing controls: " << (ignoreMissingControls_ ? "YES" : "NO") << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Include non-PF clusters: " << (includeNonPfClusters_ ? "YES" : "NO") << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Create FASTQs for index reads: " << (createFastqsForIndexReads_ ? "YES" : "NO") << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Use bgzf compression for FASTQ files: " << (noBgzfCompression_ ? "NO" : "YES") << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "FASTQ compression level: " << fastqCompressionLevel_ << std::endl;
+}
+
+
+} // namespace config
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/config/CMakeLists.txt b/src/cxx/lib/config/CMakeLists.txt
new file mode 100644
index 0000000..9535163
--- /dev/null
+++ b/src/cxx/lib/config/CMakeLists.txt
@@ -0,0 +1,23 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the cxx/lib/config subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CXX_LIBRARY_CMAKE})
+
+
diff --git a/src/cxx/lib/config/SampleSheetCsv.cpp b/src/cxx/lib/config/SampleSheetCsv.cpp
new file mode 100644
index 0000000..2ef5c8d
--- /dev/null
+++ b/src/cxx/lib/config/SampleSheetCsv.cpp
@@ -0,0 +1,1014 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file SampleSheetCsv.cpp
+ *
+ * \brief Implementation of sample sheet helper.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <algorithm>
+#include <utility>
+#include <set>
+
+#include <boost/format.hpp>
+#include <boost/regex.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/algorithm/string.hpp>
+
+#include "common/Logger.hh"
+#include "common/Debug.hh"
+#include "common/Exceptions.hh"
+#include "common/DirectoryValidator.hh"
+#include "config/SampleSheetCsv.hh"
+
+namespace bcl2fastq {
+
+
+namespace config {
+
+SampleSheetCsv::ExcludedTileRange::ExcludedTileRange(const std::string& flowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber firstTile,
+ common::TileNumber lastTile)
+: flowcell_(flowcell)
+, laneNumber_(laneNumber)
+, firstTile_(firstTile)
+, lastTile_(lastTile)
+{
+}
+
+bool SampleSheetCsv::ExcludedTileRange::isExcluded(const std::string& flowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber) const
+{
+ std::string lowerFlowcell = flowcell;
+ boost::algorithm::to_lower(lowerFlowcell);
+
+ if ( (flowcell_.empty() || flowcell_ == lowerFlowcell) &&
+ (laneNumber_ == 0 || laneNumber_ == laneNumber))
+ {
+ return (tileNumber <= lastTile_ && tileNumber >= firstTile_);
+ }
+
+ return false;
+}
+
+namespace detail {
+
+
+///////////////////////////////////////////////////////////////////////////////
+// section metadata
+///////////////////////////////////////////////////////////////////////////////
+
+/// \brief Sample sheet section metadata.
+struct SectionMetadata
+{
+public:
+
+ /// \brief Section type.
+ struct SectionType
+ {
+ enum value_type
+ {
+ NA = -1, ///< N/A
+ DATA = 0, ///< [Data] section.
+ SETTINGS, ///< [Settings] section.
+ COUNT ///< Total number of section types.
+ };
+ };
+
+public:
+
+ /// \brief String identifiers of individual sections.
+ static const char * const sectionNames_[SectionMetadata::SectionType::COUNT];
+
+public:
+
+ /// \brief Section type.
+ SectionType::value_type type_;
+
+ /// \brief The first line of the section.
+ common::CsvGrammarAttribute::const_iterator begin_;
+
+ /// \brief The one-past-the-end line of the section.
+ common::CsvGrammarAttribute::const_iterator end_;
+};
+
+const char * const SectionMetadata::sectionNames_[SectionMetadata::SectionType::COUNT] = {
+ "Data", // SectionType::DATA
+ "Settings" // SectionType::SETTINGS
+};
+
+
+///////////////////////////////////////////////////////////////////////////////
+// sample sheet parsing: get individual sections
+///////////////////////////////////////////////////////////////////////////////
+
+/// \brief Find all recognized sections in sample sheet data.
+/// \param sampleSheetData Source data to be parsed.
+/// \param sectionInserter Insert iterator for inserting section metadata.
+template<typename Inserter>
+static void createSections(const common::CsvGrammarAttribute &sampleSheetData, Inserter sectionInserter)
+{
+ // /\[((section_name_1)|(section_name_2)|...|(section_name_N)|(\w+))\]/
+ std::string sectionHeaderStr;
+ sectionHeaderStr.append("\\[(");
+ bool needOr = false;
+ for (std::size_t i = 0; i < SectionMetadata::SectionType::COUNT; ++i)
+ {
+ if (needOr)
+ {
+ sectionHeaderStr.append("|");
+ }
+ needOr = true;
+ sectionHeaderStr.append("(");
+ sectionHeaderStr.append(SectionMetadata::sectionNames_[i]);
+ sectionHeaderStr.append(")");
+ }
+ if (needOr)
+ {
+ sectionHeaderStr.append("|");
+ }
+ sectionHeaderStr.append("(\\w+)");
+ sectionHeaderStr.append(")\\]");
+ // 0: whole string; 1: section name (outermost parentheses); !2!: section_name_1; ...
+ std::size_t sectionHeaderIndexBase = 2;
+ const boost::regex sectionHeaderRegex(sectionHeaderStr, boost::regex::icase);
+
+ SectionMetadata sectionMetadata;
+ sectionMetadata.type_ = SectionMetadata::SectionType::NA;
+ sectionMetadata.begin_ = sampleSheetData.begin();
+ sectionMetadata.end_ = sampleSheetData.begin();
+ std::vector<int> sectionPresent(SectionMetadata::SectionType::COUNT, 0);
+
+ BOOST_FOREACH (const common::CsvGrammarAttribute::value_type &csvLine, std::make_pair(sampleSheetData.begin(), sampleSheetData.end()))
+ {
+ boost::cmatch res;
+ if (boost::regex_match(csvLine.front().c_str(), res, sectionHeaderRegex))
+ {
+ if (sectionMetadata.type_ != SectionMetadata::SectionType::NA)
+ {
+ *sectionInserter = sectionMetadata;
+ ++sectionInserter;
+ sectionMetadata.type_ = SectionMetadata::SectionType::NA;
+ sectionMetadata.begin_ = sectionMetadata.end_;
+ }
+
+ // magic number: +1: this is for the last catch-all component of regexp (\w+)
+ BCL2FASTQ_ASSERT_MSG(res.size() == (sectionHeaderIndexBase+SectionMetadata::SectionType::COUNT+1), "Unexpected result size: did the regular expression change?");
+
+ for (std::size_t i = sectionHeaderIndexBase; i < (sectionHeaderIndexBase+SectionMetadata::SectionType::COUNT); ++i)
+ {
+ if (res[i].matched)
+ {
+ BCL2FASTQ_ASSERT_MSG(sectionMetadata.type_ == SectionMetadata::SectionType::NA, "Only one section name should match: did the regular expression change?");
+
+ sectionMetadata.type_ = static_cast<SectionMetadata::SectionType::value_type>(i-sectionHeaderIndexBase);
+ if (sectionPresent.at(sectionMetadata.type_))
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError((boost::format("Multiple '[%s]' sections in sample sheet.") % SectionMetadata::sectionNames_[sectionMetadata.type_]).str()));
+ }
+ sectionPresent.at(sectionMetadata.type_) = true;
+ }
+ }
+ }
+ ++sectionMetadata.end_;
+ }
+ if (sectionMetadata.type_ != SectionMetadata::SectionType::NA)
+ {
+ *sectionInserter = sectionMetadata;
+ ++sectionInserter;
+ }
+}
+
+
+///////////////////////////////////////////////////////////////////////////////
+// section parsing: chain of responsibility base class
+///////////////////////////////////////////////////////////////////////////////
+
+/// \brief Base class for chain of responsibility for parsing individual sample sheet sections.
+class SectionParser
+{
+public:
+
+ /// \brief Target storage for parsed data.
+ struct Data
+ {
+ public:
+
+ /// \brief Constructor.
+ /// \param maskAdapters Adapters to be masked.
+ /// \param trimAdapters Adapters to be trim.
+ /// \param samples Samples container.
+ Data(
+ SampleSheetCsv::AdaptersContainer &maskAdapters,
+ SampleSheetCsv::AdaptersContainer &trimAdapters,
+ common::SampleMetadataContainer &samples,
+ SampleSheetCsv::TriState &createFastqForIndexReads,
+ SampleSheetCsv::TriState &findAdaptersWithIndels,
+ SampleSheetCsv::TriState &generateReverseComplementFastqs,
+ common::CycleNumber &read1EndWithCycle,
+ common::CycleNumber &read2EndWithCycle,
+ common::CycleNumber &read1StartFromCycle,
+ common::CycleNumber &read2StartFromCycle,
+ common::CycleNumber &read1UmiLength,
+ common::CycleNumber &read2UmiLength,
+ common::CycleNumber &read1UmiStartFromCycle,
+ common::CycleNumber &read2UmiStartFromCycle,
+ SampleSheetCsv::TriState &trimUmi,
+ SampleSheetCsv::ExcludedTilesContainer &excludedTiles
+ );
+
+ public:
+
+ /// \brief Adapters to be masked.
+ SampleSheetCsv::AdaptersContainer &maskAdapters_;
+
+ /// \brief Adapters to be trimmed.
+ SampleSheetCsv::AdaptersContainer &trimAdapters_;
+
+ /// \brief Sample metadata.
+ common::SampleMetadataContainer &samples_;
+
+ /// \brief Create fastq for index reads.
+ SampleSheetCsv::TriState &createFastqForIndexReads_;
+
+ /// \brief Find adapters with indels.
+ SampleSheetCsv::TriState &findAdaptersWithIndels_;
+
+ /// \brief Generate reverse complement fastqs.
+ SampleSheetCsv::TriState &generateReverseComplementFastqs_;
+
+ /// \brief End cycle number for read 1
+ common::CycleNumber &read1EndWithCycle_;
+
+ /// \brief End cycle number for read 2
+ common::CycleNumber &read2EndWithCycle_;
+
+ /// \brief Start cycle number for read 1
+ common::CycleNumber &read1StartFromCycle_;
+
+ /// \brief Start cycle number for read 2
+ common::CycleNumber &read2StartFromCycle_;
+
+ /// \brief UMI length for read 1
+ common::CycleNumber &read1UmiLength_;
+
+ /// \brief UMI length for read 2
+ common::CycleNumber &read2UmiLength_;
+
+ /// \brief UMI start cycle for read 1
+ common::CycleNumber &read1UmiStartFromCycle_;
+
+ /// \brief UMI start cycle for read 2
+ common::CycleNumber &read2UmiStartFromCycle_;
+
+ /// \brief Trim UMI if true
+ SampleSheetCsv::TriState &trimUmi_;
+
+ /// \brief Excluded tiles
+ SampleSheetCsv::ExcludedTilesContainer &excludedTiles_;
+ };
+
+public:
+
+ /// \brief Pure virtual destructor.
+ virtual ~SectionParser() = 0;
+
+public:
+
+ /// \brief Add handler to the chain.
+ void addHandler(SectionParser §ionParser);
+
+public:
+
+ /// \brief Parse sample sheet section.
+ /// \param target Target data storage for storing result.
+ /// \param source Source data to be parsed.
+ /// \retval true Data has been parsed.
+ /// \retval false Data has not been parsed by any of handlers.
+ virtual bool parse(SectionParser::Data &target, const SectionMetadata &source) = 0;
+
+private:
+
+ /// \brief Next handler in the chain.
+ SectionParser *next_;
+};
+
+SectionParser::~SectionParser()
+{
+}
+
+void SectionParser::addHandler(SectionParser §ionParser)
+{
+ if (next_)
+ {
+ next_->addHandler(sectionParser);
+ }
+ else
+ {
+ next_ = §ionParser;
+ }
+}
+
+bool SectionParser::parse(SectionParser::Data &target, const SectionMetadata &source)
+{
+ // default behavior: pass to the next handler (if any)
+ if (next_)
+ {
+ return next_->parse(target, source);
+ }
+ return false;
+}
+
+SectionParser::Data::Data(
+ SampleSheetCsv::AdaptersContainer &maskAdapters,
+ SampleSheetCsv::AdaptersContainer &trimAdapters,
+ common::SampleMetadataContainer &samples,
+ SampleSheetCsv::TriState &createFastqForIndexReads,
+ SampleSheetCsv::TriState &findAdaptersWithIndels,
+ SampleSheetCsv::TriState &generateReverseComplementFastqs,
+ common::CycleNumber &read1EndWithCycle,
+ common::CycleNumber &read2EndWithCycle,
+ common::CycleNumber &read1StartFromCycle,
+ common::CycleNumber &read2StartFromCycle,
+ common::CycleNumber &read1UmiLength,
+ common::CycleNumber &read2UmiLength,
+ common::CycleNumber &read1UmiStartFromCycle,
+ common::CycleNumber &read2UmiStartFromCycle,
+ SampleSheetCsv::TriState &trimUmi,
+ SampleSheetCsv::ExcludedTilesContainer& excludedTiles
+)
+: maskAdapters_(maskAdapters)
+, trimAdapters_(trimAdapters)
+, samples_(samples)
+, createFastqForIndexReads_(createFastqForIndexReads)
+, findAdaptersWithIndels_(findAdaptersWithIndels)
+, generateReverseComplementFastqs_(generateReverseComplementFastqs)
+, read1EndWithCycle_(read1EndWithCycle)
+, read2EndWithCycle_(read2EndWithCycle)
+, read1StartFromCycle_(read1StartFromCycle)
+, read2StartFromCycle_(read2StartFromCycle)
+, read1UmiLength_(read1UmiLength)
+, read2UmiLength_(read2UmiLength)
+, read1UmiStartFromCycle_(read1UmiStartFromCycle)
+, read2UmiStartFromCycle_(read2UmiStartFromCycle)
+, trimUmi_(trimUmi)
+, excludedTiles_(excludedTiles)
+{
+}
+
+
+///////////////////////////////////////////////////////////////////////////////
+// section parsing: [Data]
+///////////////////////////////////////////////////////////////////////////////
+
+/// \brief Concrete handler for chain of responsibility for parsing sample sheet data section.
+class DataSectionParser : public SectionParser
+{
+public:
+
+ /// \brief Column in the samples data table.
+ struct Column
+ {
+ enum value_type
+ {
+ NA = -1, ///< N/A
+ SAMPLE_ID = 0, ///< Sample ID.
+ SAMPLE_NAME, ///< Sample name.
+ PROJECT, ///< Project name.
+ LANE, ///< Lane number.
+ BARCODE_1, ///< Barcode #1.
+ BARCODE_2, ///< Barcode #2.
+ COUNT ///< Total number of recognized columns.
+ };
+ };
+
+public:
+
+ /// \brief String identifiers of individual columns.
+ static const std::string columnNames_[DataSectionParser::Column::COUNT][3];
+
+public:
+
+ /// \brief Parse column string identifier.
+ /// \param columnName Column string identifier.
+ /// \return Column numeric unique identifier.
+ static Column::value_type identifyColumnName(std::string columnName);
+
+public:
+ virtual ~DataSectionParser();
+
+public:
+
+ virtual bool parse(SectionParser::Data &target, const SectionMetadata &source);
+
+private:
+
+ virtual void parseLanes(const std::string& lanes,
+ common::SampleMetadata& sampleMetadata) const;
+
+ virtual void validateCharacters(const std::string& value,
+ const std::string& columnName) const;
+
+ virtual void addBarcodes(common::SampleMetadata& sampleMetadata,
+ std::string& barcode1,
+ std::string& barcode2,
+ std::set< std::pair<common::LaneNumber, std::string> >& uniqueBarcodes);
+};
+
+const std::string DataSectionParser::columnNames_[DataSectionParser::Column::COUNT][3] = {
+ { "sampleid" , "sample_id" , "" }, // Column::SAMPLE_ID
+ { "samplename" , "sample_name" , "" }, // Column::SAMPLE_NAME
+ { "project" , "sample_project", "" }, // Column::PROJECT
+ { "lane" , "lanes" , "" }, // Column::LANE
+ { "index" , "" , "" }, // Column::BARCODE_1
+ { "index2" , "" , "" }, // Column::BARCODE_2
+};
+
+DataSectionParser::Column::value_type DataSectionParser::identifyColumnName(std::string columnName)
+{
+ boost::algorithm::to_lower(columnName);
+ for (std::size_t column = 0; column < Column::COUNT; ++column)
+ {
+ for (std::size_t i = 0; columnNames_[column][i][0] != '\0'; ++i)
+ {
+ if (columnName == columnNames_[column][i])
+ {
+ return static_cast<DataSectionParser::Column::value_type>(column);
+ }
+ }
+ }
+ return Column::NA;
+}
+
+DataSectionParser::~DataSectionParser()
+{
+}
+
+bool DataSectionParser::parse(SectionParser::Data &target, const SectionMetadata &source)
+{
+ if (source.type_ != SectionMetadata::SectionType::DATA)
+ {
+ return SectionParser::parse(target, source);
+ }
+
+ const std::size_t columnNotPresent = -1;
+ std::vector<std::size_t> colsIndicies(Column::COUNT, columnNotPresent);
+ bool colsIndiciesInitialized = false;
+
+ std::set< std::pair<common::LaneNumber, std::string> > uniqueBarcodes;
+
+ BOOST_FOREACH (const common::CsvGrammarAttribute::value_type &line, std::make_pair(source.begin_, source.end_))
+ {
+ if (!colsIndiciesInitialized)
+ {
+ size_t colIdx = 0;
+ BOOST_FOREACH (const common::CsvGrammarAttribute::value_type::value_type &value, std::make_pair(line.begin(), line.end()))
+ {
+ Column::value_type colId = identifyColumnName(value);
+ if (colId != Column::NA)
+ {
+ colsIndicies.at(colId) = colIdx;
+ colsIndiciesInitialized = true;
+ }
+ ++colIdx;
+ }
+ }
+ else
+ {
+ std::vector<std::string> values(Column::COUNT, "");
+ for (std::size_t colId = 0; colId < Column::COUNT; ++colId)
+ {
+ if (colsIndicies.at(colId) != columnNotPresent)
+ {
+ values.at(colId) = line.at(colsIndicies.at(colId));
+ boost::algorithm::trim(values.at(colId));
+ }
+ }
+
+ if (values.at(Column::SAMPLE_ID).empty() && values.at(Column::SAMPLE_NAME).empty())
+ {
+ // skip empty lines
+ continue;
+ }
+
+ // new sample
+ target.samples_.push_back(common::SampleMetadata());
+ common::SampleMetadata &sampleMetadata = target.samples_.back();
+ sampleMetadata.id_ = values.at(Column::SAMPLE_ID);
+ sampleMetadata.name_ = values.at(Column::SAMPLE_NAME);
+ sampleMetadata.project_ = values.at(Column::PROJECT);
+
+ validateCharacters(sampleMetadata.id_, columnNames_[Column::SAMPLE_ID][0]);
+ validateCharacters(sampleMetadata.name_, columnNames_[Column::SAMPLE_NAME][0]);
+ validateCharacters(sampleMetadata.project_, columnNames_[Column::PROJECT][0]);
+
+ parseLanes(values.at(Column::LANE),
+ sampleMetadata);
+
+ std::string& barcode1 = values.at(Column::BARCODE_1);
+ std::string& barcode2 = values.at(Column::BARCODE_2);
+ if (!barcode1.empty() || !barcode2.empty())
+ {
+ addBarcodes(sampleMetadata,
+ barcode1,
+ barcode2,
+ uniqueBarcodes);
+ }
+ }
+ }
+ return true;
+}
+
+void DataSectionParser::parseLanes(const std::string& lanes,
+ common::SampleMetadata& sampleMetadata) const
+{
+ if (lanes.empty())
+ {
+ return;
+ }
+
+ std::vector<std::string> sampleLaneStrings;
+ boost::split(sampleLaneStrings, lanes, boost::is_any_of("+"));
+ for (std::string& lane : sampleLaneStrings)
+ {
+ boost::algorithm::trim(lane);
+ try
+ {
+ sampleMetadata.lanes_.push_back(boost::lexical_cast<common::LaneNumber>(lane));
+ }
+ catch (boost::bad_lexical_cast&)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Invalid character in lane: '" + lane + "'."));
+ }
+ }
+}
+
+void DataSectionParser::validateCharacters(const std::string& value,
+ const std::string& columnName) const
+{
+ if (!value.empty() && !boost::regex_match(value, boost::regex("[a-zA-Z0-9_-]+")))
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Invalid characters in " + columnName +
+ ": '" + value + "'. Only alphanumeric, '-', or '_' characters are allowed."));
+ }
+}
+
+void DataSectionParser::addBarcodes(common::SampleMetadata& sampleMetadata,
+ std::string& barcode1,
+ std::string& barcode2,
+ std::set< std::pair<common::LaneNumber, std::string> >& uniqueBarcodes)
+{
+ if (barcode1.find_first_not_of("ACGTN") != std::string::npos)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError((boost::format("Error parsing barcode '%s' in sample sheet.") % barcode1).str()));
+ }
+ if (barcode2.find_first_not_of("ACGTN") != std::string::npos)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError((boost::format("Error parsing barcode '%s' in sample sheet.") % barcode2).str()));
+ }
+
+ sampleMetadata.barcodes_.resize(sampleMetadata.barcodes_.size()+1);
+ sampleMetadata.barcodes_.back().push_back(barcode1);
+ sampleMetadata.barcodes_.back().push_back(barcode2);
+
+ for (common::LaneNumber laneNumber : sampleMetadata.lanes_)
+ {
+ std::pair< std::set< std::pair<common::LaneNumber, std::string> >::iterator, bool > insertPos = uniqueBarcodes.insert(std::make_pair(
+ laneNumber, barcode1 + ((barcode1.empty() || barcode2.empty() ? "" : "-") + barcode2) ));
+
+ if (!insertPos.second)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError((boost::format("Duplicate barcode '%s' found in sample sheet.")
+ % insertPos.first->second).str()));
+ }
+ }
+}
+
+
+///////////////////////////////////////////////////////////////////////////////
+// section parsing: [Settings]
+///////////////////////////////////////////////////////////////////////////////
+
+/// \brief Concrete handler for chain of responsibility for parsing sample sheet settings section.
+class SettingsSectionParser : public SectionParser
+{
+public:
+
+ virtual ~SettingsSectionParser();
+
+public:
+
+ virtual bool parse(SectionParser::Data &target, const SectionMetadata &source);
+
+private:
+
+ virtual void handleExcludedTiles(const std::string& key,
+ const std::string& value,
+ SectionParser::Data &target) const;
+
+ virtual SampleSheetCsv::TriState getBoolSetting(const std::string& key, const std::string& value) const;
+
+ virtual size_t getIntSetting(const std::string& value) const;
+
+ void addAdapters(SampleSheetCsv::AdaptersContainer &adapterContainer,
+ common::ReadNumber readNumber,
+ const std::string &adapters) const;
+};
+
+SettingsSectionParser::~SettingsSectionParser()
+{
+}
+
+static const std::string excludeTiles("excludetiles");
+static const std::string excludeTilesLane("excludetileslane");
+static const std::string flowcellStr("flowcell");
+
+bool SettingsSectionParser::parse(SectionParser::Data &target, const SectionMetadata &source)
+{
+ if (source.type_ != SectionMetadata::SectionType::SETTINGS)
+ {
+ return SectionParser::parse(target, source);
+ }
+
+ static const std::string maskAdapter("maskadapter");
+ static const std::string maskAdapterRead("maskadapterread");
+ static const std::string trimAdapter("trimadapter");
+ static const std::string trimAdapterDeprecated("adapter");
+ static const std::string trimAdapterRead("trimadapterread");
+ static const std::string trimAdapterReadDeprecated("adapterread");
+
+ static const std::string createFastqForIndexReads("createfastqforindexreads");
+ static const std::string findAdapterWithIndelsDeprecated("findadapterwithindels");
+ static const std::string findAdaptersWithIndels("findadapterswithindels");
+ static const std::string generateReverseComplementFastqs("reversecomplement");
+
+ static const std::string read1EndWithCycle("read1endwithcycle");
+ static const std::string read2EndWithCycle("read2endwithcycle");
+ static const std::string read1StartFromCycle("read1startfromcycle");
+ static const std::string read2StartFromCycle("read2startfromcycle");
+ static const std::string read1UmiLength("read1umilength");
+ static const std::string read2UmiLength("read2umilength");
+ static const std::string read1UmiStartFromCycle("read1umistartfromcycle");
+ static const std::string read2UmiStartFromCycle("read2umistartfromcycle");
+ static const std::string trimUmi("trimumi");
+
+ BOOST_FOREACH (const common::CsvGrammarAttribute::value_type &line, std::make_pair(source.begin_, source.end_))
+ {
+ std::string key = line.at(0);
+ boost::algorithm::to_lower(key);
+ boost::algorithm::trim(key);
+
+ std::string value = line.at(1);
+ boost::algorithm::trim(value);
+
+ if (key == maskAdapter)
+ {
+ addAdapters(target.maskAdapters_,
+ 0,
+ value);
+ }
+ else if ((key == trimAdapter) || (key == trimAdapterDeprecated))
+ {
+ addAdapters(target.trimAdapters_,
+ 0,
+ value);
+ }
+ else if ((key.find(maskAdapterRead) == 0) && (key.size() == maskAdapterRead.size()+1))
+ {
+ const char readKey = key.at(key.size()-1);
+ if (readKey >= '1' && readKey <= '9')
+ {
+ addAdapters(target.maskAdapters_,
+ readKey-'0',
+ value);
+ }
+ }
+ else if (((key.find(trimAdapterRead) == 0) && (key.size() == trimAdapterRead.size()+1)) || ((key.find(trimAdapterReadDeprecated) == 0) && (key.size() == trimAdapterReadDeprecated.size()+1)))
+ {
+ const char readKey = key.at(key.size()-1);
+ if (readKey >= '1' && readKey <= '9')
+ {
+ addAdapters(target.trimAdapters_,
+ readKey-'0',
+ value);
+ }
+ }
+ else if (key == createFastqForIndexReads)
+ {
+ target.createFastqForIndexReads_ = getBoolSetting(key, value);
+ }
+ else if (key == findAdaptersWithIndels || key == findAdapterWithIndelsDeprecated)
+ {
+ if (key == findAdapterWithIndelsDeprecated) {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING)
+ << "Option '" << findAdapterWithIndelsDeprecated
+ << "' is deprecated. Please use '"
+ << findAdaptersWithIndels << "'." << std::endl;
+ }
+ target.findAdaptersWithIndels_ = getBoolSetting(key, value);
+ }
+ else if (key == generateReverseComplementFastqs)
+ {
+ target.generateReverseComplementFastqs_ = getBoolSetting(key, value);
+ }
+ else if (key == read1EndWithCycle)
+ {
+ target.read1EndWithCycle_ = getIntSetting(value);
+ }
+ else if (key == read2EndWithCycle)
+ {
+ target.read2EndWithCycle_ = getIntSetting(value);
+ }
+ else if (key == read1StartFromCycle)
+ {
+ target.read1StartFromCycle_ = getIntSetting(value);
+ }
+ else if (key == read2StartFromCycle)
+ {
+ target.read2StartFromCycle_ = getIntSetting(value);
+ }
+ else if (key == read1UmiLength)
+ {
+ target.read1UmiLength_ = getIntSetting(value);
+ }
+ else if (key == read2UmiLength)
+ {
+ target.read2UmiLength_ = getIntSetting(value);
+ }
+ else if (key == read1UmiStartFromCycle)
+ {
+ target.read1UmiStartFromCycle_ = getIntSetting(value);
+ }
+ else if (key == read2UmiStartFromCycle)
+ {
+ target.read2UmiStartFromCycle_ = getIntSetting(value);
+ }
+ else if (key == trimUmi)
+ {
+ target.trimUmi_ = getBoolSetting(key, value);
+ }
+ else if (key.substr(0, excludeTiles.size()) == excludeTiles)
+ {
+ handleExcludedTiles(key, value, target);
+ }
+ }
+
+ return true;
+}
+
+void SettingsSectionParser::handleExcludedTiles(const std::string& key,
+ const std::string& value,
+ SectionParser::Data& target) const
+{
+ std::vector<std::string> tileRanges;
+ boost::split(tileRanges, value, boost::is_any_of("+,"));
+
+ common::LaneNumber laneNumber = 0;
+ std::string flowcell;
+ if (key.substr(0, excludeTilesLane.size()) == excludeTilesLane)
+ {
+ BCL2FASTQ_ASSERT_MSG(key.size() > excludeTilesLane.size(),
+ "Config parameter: ExcludedTilesLane must include the lane number. (e.g. ExcludedTilesLane5)");
+
+ size_t flowcellPos = key.find(flowcellStr, excludeTilesLane.size());
+
+ if (flowcellPos != std::string::npos)
+ {
+ BCL2FASTQ_ASSERT_MSG(key.size() > flowcellPos + flowcell.size(),
+ "Config parameter ExcludeTilesLaneXFlowcell must include the flowcell id. (e.g. ExcludeTilesLane5FlowcellABCD");
+
+ laneNumber = boost::lexical_cast<common::LaneNumber>(
+ key.substr(excludeTilesLane.size(), flowcellPos-excludeTilesLane.size()));
+
+ flowcell = key.substr(flowcellPos + flowcellStr.size());
+ }
+ else
+ {
+ laneNumber = boost::lexical_cast<common::LaneNumber>(key.substr(excludeTilesLane.size()));
+ }
+ }
+
+ BOOST_FOREACH(const std::string& tileRange, tileRanges)
+ {
+ std::vector<std::string> tileRangeVec;
+ boost::split(tileRangeVec, tileRange, boost::is_any_of("-"));
+
+ BCL2FASTQ_ASSERT_MSG(tileRangeVec.size() == 1 || tileRangeVec.size() == 2, "Formatting error in ExcludedTiles config parameter");
+
+ common::TileNumber firstTile = boost::lexical_cast<common::TileNumber>(tileRangeVec[0]);
+ common::TileNumber lastTile = tileRangeVec.size() == 2 ?
+ boost::lexical_cast<common::TileNumber>(tileRangeVec[1]) :
+ firstTile;
+
+ target.excludedTiles_.push_back(
+ SampleSheetCsv::ExcludedTileRange(flowcell,
+ laneNumber,
+ firstTile,
+ lastTile));
+ }
+}
+
+SampleSheetCsv::TriState SettingsSectionParser::getBoolSetting(const std::string& key, const std::string& value) const
+{
+ static const std::string trueStrings[] = { "true", "t", "yes", "y", "1" };
+ static const std::string falseStrings[] = { "false", "f", "no", "n", "0" };
+ static const std::set<std::string> trueSet(trueStrings, trueStrings + sizeof(trueStrings)/sizeof(trueStrings[0]));
+ static const std::set<std::string> falseSet(falseStrings, falseStrings + sizeof(falseStrings)/sizeof(falseStrings[0]));
+
+ std::string valueLowerCase = value;
+ boost::algorithm::to_lower(valueLowerCase);
+
+ if (trueSet.find(valueLowerCase) != trueSet.end())
+ {
+ return SampleSheetCsv::TRUE;
+ }
+
+ if (falseSet.find(valueLowerCase) != falseSet.end())
+ {
+ return SampleSheetCsv::FALSE;
+ }
+
+ BOOST_THROW_EXCEPTION(common::InputDataError("Error parsing sample sheet setting: '" + key +
+ "'. Expected bool (true, false, t, f, yes, no, y, n, 1, 0), got: '" + value + "'"));
+
+ return SampleSheetCsv::INDETERMINATE;
+}
+
+size_t SettingsSectionParser::getIntSetting(const std::string& value) const
+{
+ return boost::lexical_cast<size_t>(value);
+}
+
+void SettingsSectionParser::addAdapters(SampleSheetCsv::AdaptersContainer &adapterContainer,
+ common::ReadNumber readNumber,
+ const std::string &adapters) const
+{
+ std::vector<std::string> adapterList;
+ boost::split(adapterList, adapters, boost::is_any_of("+"));
+
+ BOOST_FOREACH(std::string &adapter, adapterList)
+ {
+ boost::algorithm::trim(adapter);
+ if (adapter.find_first_not_of("ACGT") != std::string::npos)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError((boost::format("Error parsing adapter '%s' in sample sheet.") % adapter).str()));
+ }
+
+ if (!adapter.empty())
+ {
+ adapterContainer.push_back(std::make_pair(readNumber, adapter));
+ }
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// section parsing: create chain of responsibility
+///////////////////////////////////////////////////////////////////////////////
+
+
+/// \brief Initialize section parser.
+/// \return Section parser.
+static SectionParser * createSectionParser()
+{
+ static DataSectionParser dataSectionParser;
+ static SettingsSectionParser settingsSectionParser;
+
+ dataSectionParser.addHandler(settingsSectionParser);
+ return &dataSectionParser;
+}
+
+/// \brief Section parser.
+static SectionParser * sectionParser = createSectionParser();
+
+
+} // namespace detail
+
+
+///////////////////////////////////////////////////////////////////////////////
+// SampleSheetCsv
+///////////////////////////////////////////////////////////////////////////////
+
+
+SampleSheetCsv::SampleSheetCsv(const common::CsvGrammarAttribute& sampleSheetData,
+ const boost::filesystem::path& outputDir)
+: maskAdapters_()
+, trimAdapters_()
+, samples_()
+, createFastqForIndexReads_(SampleSheetCsv::INDETERMINATE)
+, findAdaptersWithIndels_(SampleSheetCsv::INDETERMINATE)
+, generateReverseComplementFastqs_(SampleSheetCsv::INDETERMINATE)
+, read1EndWithCycle_(0)
+, read2EndWithCycle_(0)
+, read1StartFromCycle_(0)
+, read2StartFromCycle_(0)
+, read1UmiLength_(0)
+, read2UmiLength_(0)
+, read1UmiStartFromCycle_(1)
+, read2UmiStartFromCycle_(1)
+, trimUmi_(SampleSheetCsv::INDETERMINATE)
+, excludedTiles_()
+{
+ typedef std::vector<detail::SectionMetadata> SectionsContainer;
+ SectionsContainer sectionsContainer;
+ detail::createSections(sampleSheetData, std::back_inserter(sectionsContainer));
+
+ detail::SectionParser::Data sectionDataTarget(maskAdapters_,
+ trimAdapters_,
+ samples_,
+ createFastqForIndexReads_,
+ findAdaptersWithIndels_,
+ generateReverseComplementFastqs_,
+ read1EndWithCycle_,
+ read2EndWithCycle_,
+ read1StartFromCycle_,
+ read2StartFromCycle_,
+ read1UmiLength_,
+ read2UmiLength_,
+ read1UmiStartFromCycle_,
+ read2UmiStartFromCycle_,
+ trimUmi_,
+ excludedTiles_);
+
+ BOOST_FOREACH (const detail::SectionMetadata §ionDataSource, sectionsContainer)
+ {
+ detail::sectionParser->parse(sectionDataTarget, sectionDataSource);
+
+ BOOST_FOREACH(common::SampleMetadata& sampleMetadata,
+ std::make_pair(sectionDataTarget.samples_.begin(),
+ sectionDataTarget.samples_.end()))
+ {
+ common::DirectoryValidator::getSingleton().validateNoClash(outputDir / sampleMetadata.project_, "Project");
+ }
+ }
+}
+
+SampleSheetCsv::AdaptersContainer::const_iterator SampleSheetCsv::maskAdaptersBegin() const
+{
+ return maskAdapters_.begin();
+}
+
+SampleSheetCsv::AdaptersContainer::const_iterator SampleSheetCsv::maskAdaptersEnd() const
+{
+ return maskAdapters_.end();
+}
+
+SampleSheetCsv::AdaptersContainer::const_iterator SampleSheetCsv::trimAdaptersBegin() const
+{
+ return trimAdapters_.begin();
+}
+
+SampleSheetCsv::AdaptersContainer::const_iterator SampleSheetCsv::trimAdaptersEnd() const
+{
+ return trimAdapters_.end();
+}
+
+bool SampleSheetCsv::isTileExcluded(const std::string& flowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber) const
+{
+ BOOST_FOREACH(const SampleSheetCsv::ExcludedTileRange& excludedTile, excludedTiles_)
+ {
+ if (excludedTile.isExcluded(flowcell, laneNumber, tileNumber)) return true;
+ }
+
+ return false;
+}
+
+SampleSheetCsvPtr createSampleSheetCsv(const boost::filesystem::path& sampleSheet,
+ const boost::filesystem::path& outputDir)
+{
+ common::CsvGrammarAttribute sampleSheetData;
+
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Sample sheet: '" << sampleSheet.string() << "'" << std::endl;
+
+ if (boost::filesystem::exists(sampleSheet))
+ {
+ sampleSheetData = common::parseCsvFile(sampleSheet);
+ }
+ else
+ {
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Sample sheet: NOT FOUND" << std::endl;
+ }
+
+ return SampleSheetCsvPtr( new SampleSheetCsv(sampleSheetData,
+ outputDir));
+}
+
+
+} // namespace config
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/config/cppunit/CMakeLists.txt b/src/cxx/lib/config/cppunit/CMakeLists.txt
new file mode 100644
index 0000000..ec43d99
--- /dev/null
+++ b/src/cxx/lib/config/cppunit/CMakeLists.txt
@@ -0,0 +1,23 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for any cppunit subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CPPUNIT_CMAKE})
+
+
diff --git a/src/cxx/lib/config/cppunit/RegistryNames.txt b/src/cxx/lib/config/cppunit/RegistryNames.txt
new file mode 100644
index 0000000..dca4be7
--- /dev/null
+++ b/src/cxx/lib/config/cppunit/RegistryNames.txt
@@ -0,0 +1 @@
+SampleSheetCsv
diff --git a/src/cxx/lib/config/cppunit/testSampleSheetCsv.cpp b/src/cxx/lib/config/cppunit/testSampleSheetCsv.cpp
new file mode 100644
index 0000000..927cf06
--- /dev/null
+++ b/src/cxx/lib/config/cppunit/testSampleSheetCsv.cpp
@@ -0,0 +1,310 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testSampleSheetCsv.cpp
+ *
+ * \brief Sample sheet helper cppunit test declarations.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <string>
+
+#include "RegistryName.hh"
+#include "testSampleSheetCsv.hh"
+
+#include "common/CsvGrammar.hh"
+#include "config/SampleSheetCsv.hh"
+
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(TestSampleSheetCsv, registryName("SampleSheetCsv"));
+
+
+void TestSampleSheetCsv::setUp()
+{
+}
+
+void TestSampleSheetCsv::tearDown()
+{
+}
+
+void TestSampleSheetCsv::testNova()
+{
+ const std::string sampleSheetData(
+ "[Header],,,,\n"
+ "Investigator Name,Isabelle,,,\n"
+ "Project Name,Nova,,,\n"
+ "Experiment Name,Orbital death ray research volume LXXIV,,,\n"
+ "Date,5/27/2025,,,\n"
+ "Workflow,GenerateFASTQ,,,\n"
+ ",,,,\n"
+ "[UnknownSection1],,,,\n"
+ "Key1,Value1,,,\n"
+ "Key2,Value2,,,\n"
+ ",,,,\n"
+ ",,,,\n"
+ "[Settings],,,,\n"
+ "MaskAdapter,CGCGTATACGCGTATA,,,\n"
+ "MaskAdapterRead2,GCGCATATGCGCATAT,,,\n"
+ "TrimAdapter,TATACGCGTATACGCG,,,\n"
+ "TrimAdapterRead2,ATATGCGCATATGCGC,,,\n"
+ ",,,,\n"
+ "[UnknownSection2],,,,\n"
+ "Key1,Value1,,,\n"
+ "Key2,Value2,,,\n"
+ ",,,,\n"
+ "[Data],,,,\n"
+ "LaNe,SampleID,SampleName,project,unknown,index,index2\n"
+ "1,Aa,aA,P1,+_)(*(*,AAAAAAAA,AAAAAAAA\n"
+ "2,Cc,cC,P1,!@#$,CCCCCCCC,CCCCCCCC\n"
+ "3+4,Gg,gG,P2,#$%,GGGGGGGG,GGGGGGGG\n"
+ "5,Tt,tT,P2,#2s^,TTTTTTTT,TTTTTTTT\n"
+ "7,Xx,xX,P3,~!@,ACGTACGT,ACGTACGT\n"
+ ",,,,\n"
+ "[UnknownSection3],,,,\n"
+ "Key1,Value1,,,\n"
+ "Key2,Value2,,,\n"
+ ",,,,\n"
+ );
+
+ bcl2fastq::config::SampleSheetCsv sampleSheetCsv(bcl2fastq::common::parseCsvData(
+ sampleSheetData.begin(),
+ sampleSheetData.end()
+ ),
+ boost::filesystem::path());
+
+ bcl2fastq::config::SampleSheetCsv::AdaptersContainer::const_iterator maskAdaptersIter = sampleSheetCsv.maskAdaptersBegin();
+ const bcl2fastq::config::SampleSheetCsv::AdaptersContainer::const_iterator maskAdaptersEnd = sampleSheetCsv.maskAdaptersEnd();
+ CPPUNIT_ASSERT(maskAdaptersIter != maskAdaptersEnd);
+ CPPUNIT_ASSERT_EQUAL(maskAdaptersIter->first, bcl2fastq::common::ReadNumber(0));
+ CPPUNIT_ASSERT_EQUAL(maskAdaptersIter->second, std::string("CGCGTATACGCGTATA"));
+ ++maskAdaptersIter;
+ CPPUNIT_ASSERT_EQUAL(maskAdaptersIter->first, bcl2fastq::common::ReadNumber(2));
+ CPPUNIT_ASSERT_EQUAL(maskAdaptersIter->second, std::string("GCGCATATGCGCATAT"));
+ ++maskAdaptersIter;
+ CPPUNIT_ASSERT(maskAdaptersIter == maskAdaptersEnd);
+
+ bcl2fastq::config::SampleSheetCsv::AdaptersContainer::const_iterator trimAdaptersIter = sampleSheetCsv.trimAdaptersBegin();
+ const bcl2fastq::config::SampleSheetCsv::AdaptersContainer::const_iterator trimAdaptersEnd = sampleSheetCsv.trimAdaptersEnd();
+ CPPUNIT_ASSERT(trimAdaptersIter != trimAdaptersEnd);
+ CPPUNIT_ASSERT_EQUAL(trimAdaptersIter->first, bcl2fastq::common::ReadNumber(0));
+ CPPUNIT_ASSERT_EQUAL(trimAdaptersIter->second, std::string("TATACGCGTATACGCG"));
+ ++trimAdaptersIter;
+ CPPUNIT_ASSERT_EQUAL(trimAdaptersIter->first, bcl2fastq::common::ReadNumber(2));
+ CPPUNIT_ASSERT_EQUAL(trimAdaptersIter->second, std::string("ATATGCGCATATGCGC"));
+ ++trimAdaptersIter;
+ CPPUNIT_ASSERT(trimAdaptersIter == trimAdaptersEnd);
+
+ bcl2fastq::common::SampleMetadataContainer::const_iterator sampleMetadataIter = sampleSheetCsv.getSampleMetadata().begin();
+ const bcl2fastq::common::SampleMetadataContainer::const_iterator sampleMetadataEnd = sampleSheetCsv.getSampleMetadata().end();
+ CPPUNIT_ASSERT(sampleMetadataIter != sampleMetadataEnd);
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->id_, std::string("Aa"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->name_, std::string("aA"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->project_, std::string("P1"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->lanes_.size(), 1UL);
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->lanes_[0], 1UL);
+ {
+ bcl2fastq::common::SampleMetadata::BarcodesContainer::const_iterator barcodesIter = sampleMetadataIter->barcodes_.begin();
+ const bcl2fastq::common::SampleMetadata::BarcodesContainer::const_iterator barcodesEnd = sampleMetadataIter->barcodes_.end();
+ CPPUNIT_ASSERT(barcodesIter != barcodesEnd);
+ {
+ bcl2fastq::common::SampleMetadata::BarcodesContainer::value_type::const_iterator componentsIter = barcodesIter->begin();
+ const bcl2fastq::common::SampleMetadata::BarcodesContainer::value_type::const_iterator componentsEnd = barcodesIter->end();
+ CPPUNIT_ASSERT(componentsIter != componentsEnd);
+ CPPUNIT_ASSERT_EQUAL(*componentsIter, std::string("AAAAAAAA"));
+ ++componentsIter;
+ CPPUNIT_ASSERT(componentsIter != componentsEnd);
+ CPPUNIT_ASSERT_EQUAL(*componentsIter, std::string("AAAAAAAA"));
+ ++componentsIter;
+ CPPUNIT_ASSERT(componentsIter == componentsEnd);
+ }
+ }
+ ++sampleMetadataIter;
+ CPPUNIT_ASSERT(sampleMetadataIter != sampleMetadataEnd);
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->id_, std::string("Cc"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->name_, std::string("cC"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->project_, std::string("P1"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->lanes_.size(), 1UL);
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->lanes_[0], 2UL);
+ {
+ bcl2fastq::common::SampleMetadata::BarcodesContainer::const_iterator barcodesIter = sampleMetadataIter->barcodes_.begin();
+ const bcl2fastq::common::SampleMetadata::BarcodesContainer::const_iterator barcodesEnd = sampleMetadataIter->barcodes_.end();
+ CPPUNIT_ASSERT(barcodesIter != barcodesEnd);
+ {
+ bcl2fastq::common::SampleMetadata::BarcodesContainer::value_type::const_iterator componentsIter = barcodesIter->begin();
+ const bcl2fastq::common::SampleMetadata::BarcodesContainer::value_type::const_iterator componentsEnd = barcodesIter->end();
+ CPPUNIT_ASSERT(componentsIter != componentsEnd);
+ CPPUNIT_ASSERT_EQUAL(*componentsIter, std::string("CCCCCCCC"));
+ ++componentsIter;
+ CPPUNIT_ASSERT(componentsIter != componentsEnd);
+ CPPUNIT_ASSERT_EQUAL(*componentsIter, std::string("CCCCCCCC"));
+ ++componentsIter;
+ CPPUNIT_ASSERT(componentsIter == componentsEnd);
+ }
+ }
+ ++sampleMetadataIter;
+ CPPUNIT_ASSERT(sampleMetadataIter != sampleMetadataEnd);
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->id_, std::string("Gg"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->name_, std::string("gG"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->project_, std::string("P2"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->lanes_.size(), 2UL);
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->lanes_[0], 3UL);
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->lanes_[1], 4UL);
+ {
+ bcl2fastq::common::SampleMetadata::BarcodesContainer::const_iterator barcodesIter = sampleMetadataIter->barcodes_.begin();
+ const bcl2fastq::common::SampleMetadata::BarcodesContainer::const_iterator barcodesEnd = sampleMetadataIter->barcodes_.end();
+ CPPUNIT_ASSERT(barcodesIter != barcodesEnd);
+ {
+ bcl2fastq::common::SampleMetadata::BarcodesContainer::value_type::const_iterator componentsIter = barcodesIter->begin();
+ const bcl2fastq::common::SampleMetadata::BarcodesContainer::value_type::const_iterator componentsEnd = barcodesIter->end();
+ CPPUNIT_ASSERT(componentsIter != componentsEnd);
+ CPPUNIT_ASSERT_EQUAL(*componentsIter, std::string("GGGGGGGG"));
+ ++componentsIter;
+ CPPUNIT_ASSERT(componentsIter != componentsEnd);
+ CPPUNIT_ASSERT_EQUAL(*componentsIter, std::string("GGGGGGGG"));
+ ++componentsIter;
+ CPPUNIT_ASSERT(componentsIter == componentsEnd);
+ }
+ }
+ ++sampleMetadataIter;
+ CPPUNIT_ASSERT(sampleMetadataIter != sampleMetadataEnd);
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->id_, std::string("Tt"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->name_, std::string("tT"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->project_, std::string("P2"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->lanes_.size(), 1UL);
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->lanes_[0], 5UL);
+ {
+ bcl2fastq::common::SampleMetadata::BarcodesContainer::const_iterator barcodesIter = sampleMetadataIter->barcodes_.begin();
+ const bcl2fastq::common::SampleMetadata::BarcodesContainer::const_iterator barcodesEnd = sampleMetadataIter->barcodes_.end();
+ CPPUNIT_ASSERT(barcodesIter != barcodesEnd);
+ {
+ bcl2fastq::common::SampleMetadata::BarcodesContainer::value_type::const_iterator componentsIter = barcodesIter->begin();
+ const bcl2fastq::common::SampleMetadata::BarcodesContainer::value_type::const_iterator componentsEnd = barcodesIter->end();
+ CPPUNIT_ASSERT(componentsIter != componentsEnd);
+ CPPUNIT_ASSERT_EQUAL(*componentsIter, std::string("TTTTTTTT"));
+ ++componentsIter;
+ CPPUNIT_ASSERT(componentsIter != componentsEnd);
+ CPPUNIT_ASSERT_EQUAL(*componentsIter, std::string("TTTTTTTT"));
+ ++componentsIter;
+ CPPUNIT_ASSERT(componentsIter == componentsEnd);
+ }
+ }
+ ++sampleMetadataIter;
+ CPPUNIT_ASSERT(sampleMetadataIter != sampleMetadataEnd);
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->id_, std::string("Xx"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->name_, std::string("xX"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->project_, std::string("P3"));
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->lanes_.size(), 1UL);
+ CPPUNIT_ASSERT_EQUAL(sampleMetadataIter->lanes_[0], 7UL);
+ {
+ bcl2fastq::common::SampleMetadata::BarcodesContainer::const_iterator barcodesIter = sampleMetadataIter->barcodes_.begin();
+ const bcl2fastq::common::SampleMetadata::BarcodesContainer::const_iterator barcodesEnd = sampleMetadataIter->barcodes_.end();
+ CPPUNIT_ASSERT(barcodesIter != barcodesEnd);
+ {
+ bcl2fastq::common::SampleMetadata::BarcodesContainer::value_type::const_iterator componentsIter = barcodesIter->begin();
+ const bcl2fastq::common::SampleMetadata::BarcodesContainer::value_type::const_iterator componentsEnd = barcodesIter->end();
+ CPPUNIT_ASSERT(componentsIter != componentsEnd);
+ CPPUNIT_ASSERT_EQUAL(*componentsIter, std::string("ACGTACGT"));
+ ++componentsIter;
+ CPPUNIT_ASSERT(componentsIter != componentsEnd);
+ CPPUNIT_ASSERT_EQUAL(*componentsIter, std::string("ACGTACGT"));
+ ++componentsIter;
+ CPPUNIT_ASSERT(componentsIter == componentsEnd);
+ }
+ }
+ ++sampleMetadataIter;
+ CPPUNIT_ASSERT(sampleMetadataIter == sampleMetadataEnd);
+}
+
+void TestSampleSheetCsv::testInvalidCharacters()
+{
+ const std::string invalidSampleIdData(
+ "[Data],,,,\n"
+ "SampleID,SampleName,project,unknown,index,index2\n"
+ "Aa@,aA,P1,+_)(*(*,AAAAAAAA,AAAAAAAA\n"
+ ",,,,\n"
+ );
+
+ CPPUNIT_ASSERT_THROW(bcl2fastq::config::SampleSheetCsv sampleSheetCsv(bcl2fastq::common::parseCsvData(
+ invalidSampleIdData.begin(),
+ invalidSampleIdData.end()),
+ boost::filesystem::path()),
+ bcl2fastq::common::InputDataError);
+
+ const std::string invalidSampleNameData(
+ "[Data],,,,\n"
+ "SampleID,SampleName,project,unknown,index,index2\n"
+ "Aa,aA&,P1,+_)(*(*,AAAAAAAA,AAAAAAAA\n"
+ ",,,,\n"
+ );
+
+ CPPUNIT_ASSERT_THROW(bcl2fastq::config::SampleSheetCsv sampleSheetCsv(bcl2fastq::common::parseCsvData(
+ invalidSampleNameData.begin(),
+ invalidSampleNameData.end()),
+ boost::filesystem::path()),
+ bcl2fastq::common::InputDataError);
+
+ const std::string invalidProjectData(
+ "[Data],,,,\n"
+ "SampleID,SampleName,project,unknown,index,index2\n"
+ "Aa,aA,P1),+_)(*(*,AAAAAAAA,AAAAAAAA\n"
+ ",,,,\n"
+ );
+
+ CPPUNIT_ASSERT_THROW(bcl2fastq::config::SampleSheetCsv sampleSheetCsv(bcl2fastq::common::parseCsvData(
+ invalidProjectData.begin(),
+ invalidProjectData.end()),
+ boost::filesystem::path()),
+ bcl2fastq::common::InputDataError);
+
+ const std::string invalidLaneData(
+ "[Data],,,,\n"
+ "Lane,SampleID,SampleName,project,unknown,index,index2\n"
+ "1;2;3,Aa,aA,P1,+_)(*(*,AAAAAAAA,AAAAAAAA\n"
+ ",,,,\n"
+ );
+
+ CPPUNIT_ASSERT_THROW(bcl2fastq::config::SampleSheetCsv sampleSheetCsv(bcl2fastq::common::parseCsvData(
+ invalidLaneData.begin(),
+ invalidLaneData.end()),
+ boost::filesystem::path()),
+ bcl2fastq::common::InputDataError);
+}
+
+void TestSampleSheetCsv::testDuplicateBarcode()
+{
+ const std::string duplicateBarcodeAltLaneData(
+ "[Data],,,,\n"
+ "Lane,SampleID,SampleName,project,unknown,index,index2\n"
+ "1+2+3,Aa,aA,P1,+_)(*(*,AAAAAAAA,AAAAAAAA\n"
+ "4,Aa,aA,P1,+_)(*(*,AAAAAAAA,AAAAAAAA\n"
+ ",,,,\n"
+ );
+
+ CPPUNIT_ASSERT_NO_THROW(bcl2fastq::config::SampleSheetCsv sampleSheetCsv(
+ bcl2fastq::common::parseCsvData(
+ duplicateBarcodeAltLaneData.begin(),
+ duplicateBarcodeAltLaneData.end()),
+ boost::filesystem::path()));
+
+ const std::string duplicateBarcodeSameLaneData(
+ "[Data],,,,\n"
+ "Lane,SampleID,SampleName,project,unknown,index,index2\n"
+ "1+2+3,Aa,aA,P1,+_)(*(*,AAAAAAAA,AAAAAAAA\n"
+ "1,Bb,bB,P2,+_)(*(*,AAAAAAAA,AAAAAAAA\n"
+ ",,,,\n"
+ );
+
+ CPPUNIT_ASSERT_THROW(
+ bcl2fastq::config::SampleSheetCsv sampleSheetCsv(
+ bcl2fastq::common::parseCsvData(duplicateBarcodeSameLaneData.begin(),
+ duplicateBarcodeSameLaneData.end()),
+ boost::filesystem::path()),
+ bcl2fastq::common::InputDataError);
+
+}
diff --git a/src/cxx/lib/config/cppunit/testSampleSheetCsv.hh b/src/cxx/lib/config/cppunit/testSampleSheetCsv.hh
new file mode 100644
index 0000000..b3853a3
--- /dev/null
+++ b/src/cxx/lib/config/cppunit/testSampleSheetCsv.hh
@@ -0,0 +1,51 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testSampleSheetCsv.hh
+ *
+ * \brief Sample sheet helper cppunit test declarations.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_TEST_SAMPLESHEETCSV_HH
+#define BCL2FASTQ_LAYOUT_TEST_SAMPLESHEETCSV_HH
+
+
+#include <string>
+
+#include <cppunit/extensions/HelperMacros.h>
+
+#include "config/SampleSheetCsv.hh"
+
+
+/// \brief Test suite for SampleSheetCsv.
+class TestSampleSheetCsv : public CppUnit::TestFixture
+{
+private:
+
+ CPPUNIT_TEST_SUITE(TestSampleSheetCsv);
+ CPPUNIT_TEST(testNova);
+ CPPUNIT_TEST(testInvalidCharacters);
+ CPPUNIT_TEST(testDuplicateBarcode);
+ CPPUNIT_TEST_SUITE_END();
+
+public:
+
+ void setUp();
+ void tearDown();
+ void testNova();
+ void testInvalidCharacters();
+ void testDuplicateBarcode();
+};
+
+
+#endif // #ifndef BCL2FASTQ_LAYOUT_TEST_SAMPLESHEETCSV_HH
+
+
diff --git a/src/cxx/lib/conversion/BclLoader.cpp b/src/cxx/lib/conversion/BclLoader.cpp
new file mode 100644
index 0000000..5f1aee5
--- /dev/null
+++ b/src/cxx/lib/conversion/BclLoader.cpp
@@ -0,0 +1,1028 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BclLoader.cpp
+ *
+ * \brief Implementation of BCL loader.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#include <errno.h>
+#include <algorithm>
+#include <utility>
+#include <chrono>
+#include <thread>
+
+#include <boost/format.hpp>
+#include <boost/filesystem.hpp>
+
+#include "common/Debug.hh"
+#include "common/Exceptions.hh"
+#include "common/Types.hh"
+#include "conversion/BclLoader.hh"
+#include "data/Instrument.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace conversion {
+
+typedef std::shared_ptr<BclLoaderTaskManager> BclLoaderTaskManagerPtr;
+
+namespace
+{
+ std::string getTileSpec(
+ const layout::LaneInfo &laneInfo)
+ {
+ std::stringstream ss;
+ ss << "Lane: " << laneInfo.getNumber() << " Tile: xxxx";
+
+ return ss.str();
+ }
+
+ std::string getTileSpec(
+ const layout::LaneInfo &laneInfo,
+ const layout::TileInfo &tileInfo)
+ {
+ std::stringstream ss;
+ ss << "Lane: " << laneInfo.getNumber() << " Tile: " << tileInfo.getNumber();
+
+ return ss.str();
+ }
+}
+
+BclLoadTask::BclLoadTask(
+ std::shared_ptr<BclLoaderTaskManager>& taskManager,
+ data::RawBclBufferGroup& bclData,
+ std::mutex* aggregatedBclFileMutex,
+ common::TileAggregationMode aggregateTilesMode,
+ const layout::LaneInfo &laneInfo,
+ const layout::CycleInfo &cycleInfo,
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfoIter,
+ bool ignoreMissingBcls,
+ size_t bclIdx,
+ DemuxBuffer& outputBuffer
+)
+: Task(taskManager)
+, bclData_(bclData)
+, aggregatedBclFileMutex_(aggregatedBclFileMutex)
+, aggregateTilesMode_(aggregateTilesMode)
+, laneInfo_(laneInfo)
+, cycleInfo_(cycleInfo)
+, tileInfoIter_(tileInfoIter)
+, ignoreMissingBcls_(ignoreMissingBcls)
+, bclIdx_(bclIdx)
+, outputBuffer_(outputBuffer)
+{
+}
+
+bool BclLoadTask::execute(int32_t)
+{
+ switch (aggregateTilesMode_)
+ {
+ case common::TileAggregationMode::AGGREGATED:
+ return readAggregatedTiles();
+ break;
+ case common::TileAggregationMode::NON_AGGREGATED:
+ return readSeparateTiles();
+ break;
+ case common::TileAggregationMode::CBCL:
+ return readCbclTiles();
+ break;
+ default:
+ BCL2FASTQ_ASSERT_MSG(false, "Unexpected aggregation mode type");
+ break;
+ }
+
+ return false;
+}
+
+bool BclLoadTask::readSeparateTiles()
+{
+ static int sequentialId = 0; ++sequentialId;
+
+ size_t tileIdx = 0;
+ for (auto& rawDataForTile : bclData_)
+ {
+ data::BclFile bclFile(rawDataForTile.cycleData_[bclIdx_].bcls_,
+ rawDataForTile.gzipDecompressors_[bclIdx_],
+ ignoreMissingBcls_,
+ true);
+
+ auto clustersCount = bclFile.getClustersCount();
+
+ common::resizeAndRealloc(outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].bcls_, clustersCount);
+ auto bytesRead = bclFile.read(outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].bcls_.data(), clustersCount);
+ if (bytesRead != clustersCount)
+ {
+ int errnum = errno;
+ if (ignoreMissingBcls_)
+ {
+ // Assume the truncated file is corrupt.
+ std::stringstream ss;
+ ss << "BCL file '" << bclFile.getPath() << "' ";
+ if (bytesRead > 0) {
+ ss << "corrupt: " << std::endl;
+ }
+ else {
+ ss << "truncated: " << std::endl;
+ }
+ ss << "bytes_read=" << bytesRead << " bytes_expected=" << clustersCount;
+
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << ss.str() << std::endl;
+
+ bytesRead = 0;
+ outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].bcls_.resize(bytesRead > 0 ? bytesRead : 0);
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, (boost::format("BCL file '%s' truncated: bytes_read=%d bytes_expected=%d") % bclFile.getPath().string() % bytesRead % clustersCount).str()));
+ }
+ }
+
+ ++tileIdx;
+ }
+
+ return true;
+}
+
+bool BclLoadTask::readAggregatedTiles()
+{
+ std::lock_guard<std::mutex> lock(*aggregatedBclFileMutex_);
+
+ int32_t tileIdx = 0;
+ auto rawDataIter = bclData_.begin();
+ while (rawDataIter != bclData_.end())
+ {
+ common::ClustersCount clustersCount = tileInfoIter_->getClustersCount();
+
+ if (clustersCount != 0)
+ {
+ std::vector<char> &bclVector = outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].bcls_;
+ common::resizeAndRealloc(bclVector, clustersCount);
+
+ std::shared_ptr<data::BclFile> bclFile(std::make_shared<data::BclFile>(
+ rawDataIter->cycleData_[bclIdx_].bcls_,
+ rawDataIter->gzipDecompressors_[bclIdx_],
+ ignoreMissingBcls_,
+ tileInfoIter_->getIndex() == 0));
+
+ std::streamsize bytesRead = 0;
+ if (bclFile->seek(0, rawDataIter->uncompressedBclOffset_))
+ {
+ bytesRead = bclFile->read(bclVector.data(), clustersCount);
+ }
+ else
+ {
+ // Explicitly clear the empty tile.
+ outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].bcls_.resize(0);
+ }
+
+ checkBytesRead(bytesRead,
+ clustersCount,
+ tileIdx,
+ rawDataIter->cycleData_[bclIdx_].bcls_.path_);
+ }
+ else
+ {
+ // Explicitly clear the empty tile.
+ outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].bcls_.resize(0);
+ }
+
+ ++rawDataIter;
+ ++tileInfoIter_;
+ ++tileIdx;
+ }
+
+ return true;
+}
+
+std::stringstream &BclLoadTask::getTileDescriptor(std::stringstream &msg, data::PerCycleData perCycleData)
+{
+ if (perCycleData.bcls_.path_.string().empty()) {
+ msg << "BCL data for lane " << laneInfo_.getNumber()
+ << " cycle " << cycleInfo_.getNumber()
+ << " tile " << tileInfoIter_->getNumber();
+ } else {
+ msg << "Tile: " << tileInfoIter_->getNumber() << ", BCL file '" << perCycleData.bcls_.path_ << "'";
+ }
+
+ return msg;
+}
+
+bool BclLoadTask::readCbclTiles()
+{
+ size_t tileIdx = 0;
+ for (data::RawBclBuffer& rawDataForTile : bclData_)
+ {
+ auto& gzipDecompressor = rawDataForTile.gzipDecompressors_[bclIdx_];
+
+ gzipDecompressor.flush();
+ gzipDecompressor.reset();
+
+ auto& cycleData = rawDataForTile.cycleData_[bclIdx_];
+
+ boost::iostreams::basic_array_source<char> inputSrc(cycleData.bcls_.data(),
+ cycleData.bcls_.size());
+ boost::iostreams::stream<boost::iostreams::basic_array_source<char>> istr(inputSrc);
+
+ outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].bcls_.resize(cycleData.uncompressedBlockSize_);
+ outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].includeNonPf_ = cycleData.includeNonPf_;
+ outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].numBitsPerQscore_ = cycleData.numBitsPerQscore_;
+ outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].remappedQscores_ = cycleData.remappedQscores_;
+
+ std::streamsize bytesRead = 0;
+ bool decompressed = true;
+ try
+ {
+ bytesRead = gzipDecompressor.read(istr,
+ outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].bcls_.data(),
+ outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].bcls_.size());
+ }
+ catch (...)
+ {
+ bytesRead = 0;
+ decompressed = false;
+
+ std::stringstream msg;
+ getTileDescriptor(msg, cycleData);
+ msg << " corrupt.";
+
+ if (ignoreMissingBcls_)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << msg.str() << std::endl;
+ outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].bcls_.resize(0);
+ }
+ else
+ {
+ int errnum = errno;
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, msg.str()));
+ }
+ }
+
+ if (decompressed && bytesRead != cycleData.uncompressedBlockSize_)
+ {
+ std::stringstream msg;
+ getTileDescriptor(msg, cycleData);
+ msg << " truncated: bytes_read=" << bytesRead << " bytes_expected=" << cycleData.uncompressedBlockSize_;
+
+ int errnum = errno;
+ if (ignoreMissingBcls_)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << msg.str() << std::endl;
+ outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].bcls_.resize(bytesRead > 0 ? bytesRead : 0);
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, msg.str()));
+ }
+ }
+
+ ++tileIdx;
+ }
+
+ return true;
+}
+
+void BclLoadTask::checkBytesRead(int32_t bytesRead,
+ int32_t expectedBytes,
+ int32_t tileIdx,
+ const boost::filesystem::path& bclPath)
+{
+ if (bytesRead < expectedBytes)
+ {
+ int errnum = errno;
+ if (ignoreMissingBcls_) {
+ if (!bclPath.string().empty()) {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "BCL file '" << bclPath << "' truncated: bytes_read=" << bytesRead << " bytes_expected=" << expectedBytes << ":" << std::strerror(errnum) << " (" << errnum << ")" << std::endl;
+ }
+ outputBuffer_.bclBuffers_[tileIdx].bcls_[bclIdx_].bcls_.resize(bytesRead > 0 ? bytesRead : 0);
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, (boost::format("BCL file '%s' truncated: bytes_read=%d bytes_expected=%d") % bclPath.string() % bytesRead % expectedBytes).str()));
+ }
+ }
+}
+
+PositionsLoadTask::PositionsLoadTask(
+ std::shared_ptr<BclLoaderTaskManager>& taskManager,
+ data::RawBclBufferGroup& bclData,
+ bool aggregateTilesFlag,
+ const layout::LaneInfo &laneInfo,
+ layout::LaneInfo::TileInfosContainer::const_iterator &tileInfoIter,
+ const std::vector<common::ClustersCount>& clustersCounts,
+ bool ignoreMissingPositions,
+ data::BclBufferVec& outputBuffer,
+ std::shared_ptr<data::BclBuffer::PatternedPositionsContainer>& patternedFlowcellPositions
+)
+: Task(taskManager)
+, bclData_(bclData)
+, aggregateTilesFlag_(aggregateTilesFlag)
+, laneInfo_(laneInfo)
+, tileInfoIter_(tileInfoIter)
+, clustersCounts_(clustersCounts)
+, ignoreMissingPositions_(ignoreMissingPositions)
+, outputBuffer_(outputBuffer)
+, patternedFlowcellPositions_(patternedFlowcellPositions)
+{
+}
+
+bool PositionsLoadTask::execute(int32_t)
+{
+ if (patternedFlowcellPositions_)
+ {
+ if (execute(*bclData_.begin()->positions_,
+ *patternedFlowcellPositions_))
+ {
+ patternedFlowcellPositions_->setReady();
+ return true;
+ }
+ else
+ {
+ patternedFlowcellPositions_->setReady();
+ return false;
+ }
+ }
+ else
+ {
+ size_t bufferIndex = 0;
+ for (auto& inputBuffer : bclData_)
+ {
+ auto& outputBuffer = outputBuffer_[bufferIndex].positions_;
+ if (!outputBuffer)
+ {
+ outputBuffer = std::make_shared<data::BclBuffer::PositionsContainer>();
+ }
+
+ execute(*inputBuffer.positions_,
+ *outputBuffer);
+
+ ++bufferIndex;
+ ++tileInfoIter_;
+ }
+ }
+
+ return true;
+}
+
+bool PositionsLoadTask::execute(const common::RawDataBuffer& inputBuffer,
+ data::BclBuffer::PositionsContainer& outputBuffer)
+{
+ bool skipParsingHeader = !patternedFlowcellPositions_ && (aggregateTilesFlag_ && (tileInfoIter_->getIndex() != 0));
+ auto bytesRead = data::PositionsFileFactory::read(inputBuffer,
+ ignoreMissingPositions_,
+ skipParsingHeader,
+ outputBuffer,
+ tileInfoIter_->getClustersCount());
+
+ return true;
+}
+
+
+FilterLoadTask::FilterLoadTask(
+ std::shared_ptr<BclLoaderTaskManager>& taskManager,
+ data::RawBclBufferGroup& bclData,
+ common::TileAggregationMode tileAggregationMode,
+ const layout::LaneInfo &laneInfo,
+ layout::LaneInfo::TileInfosContainer::const_iterator &tileInfoIter,
+ const std::vector<common::ClustersCount>& clustersCounts,
+ bool ignoreMissingFilters,
+ data::BclBufferVec &outputBuffer
+)
+: Task(taskManager)
+, bclData_(bclData)
+, tileAggregationMode_(tileAggregationMode)
+, aggregateTilesFlag_(data::Instrument::isFilterFileAggregated(tileAggregationMode))
+, laneInfo_(laneInfo)
+, tileInfoIter_(tileInfoIter)
+, clustersCounts_(clustersCounts)
+, ignoreMissingFilters_(ignoreMissingFilters)
+, outputBuffer_(outputBuffer)
+{
+}
+
+bool FilterLoadTask::execute(int32_t)
+{
+ size_t bufferIndex = 0;
+ for (auto& rawDataForTile : bclData_)
+ {
+ readFilterFile(rawDataForTile.filters_,
+ *tileInfoIter_,
+ outputBuffer_[bufferIndex],
+ aggregateTilesFlag_,
+ aggregateTilesFlag_ && (tileInfoIter_->getIndex() != 0));
+
+ ++bufferIndex;
+ ++tileInfoIter_;
+ }
+
+ return true;
+}
+
+void FilterLoadTask::readFilterFile(common::RawDataBuffer& filterData,
+ const layout::TileInfo& tileInfo,
+ data::BclBuffer& outputBuffer,
+ bool isAggregatedTiles,
+ bool skipHeader)
+{
+ data::FilterFile filterFile(filterData,
+ ignoreMissingFilters_,
+ skipHeader);
+
+ auto clustersCount = isAggregatedTiles ? tileInfo.getClustersCount() : filterFile.getClustersCount();
+
+ common::resizeAndRealloc(outputBuffer.filters_, clustersCount);
+ auto recordsRead = filterFile.read(outputBuffer.filters_.data(), clustersCount);
+
+ validateFilterRecords(recordsRead,
+ clustersCount,
+ filterData.path_,
+ outputBuffer);
+}
+
+void FilterLoadTask::validateFilterRecords(std::size_t recordsRead,
+ common::ClustersCount clustersCount,
+ const boost::filesystem::path& filePath,
+ data::BclBuffer& outputBuffer)
+{
+ int errnum = errno;
+ if (recordsRead != clustersCount)
+ {
+ if (ignoreMissingFilters_)
+ {
+ if (tileAggregationMode_ == common::TileAggregationMode::CBCL)
+ {
+ // If cbcl filter file is corrupt then we can't map the tile indices so ignore this tile.
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "CBCL filter file '"
+ << filePath << "' corrupt: this tile will be ignored." << std::endl;
+
+ outputBuffer.filters_.resize(0);
+
+ for (data::PerCycleData &perCycleData : outputBuffer.bcls_)
+ {
+ perCycleData.bcls_.resize(0);
+ perCycleData.uncompressedBlockSize_ = 0;
+ }
+ }
+ else
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "Filter file '" << filePath << "' truncated: records_read=" << recordsRead << " records_expected=" << clustersCount << ":" << std::strerror(errnum) << " (" << errnum << ")" << std::endl;
+ outputBuffer.filters_.resize(recordsRead);
+ static const data::FilterFile::Record defaultRecord(0x01);
+ outputBuffer.filters_.resize(clustersCount, defaultRecord);
+ }
+
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, (boost::format("Filter file '%s' truncated: records_real=%d records_expected=%d") % filePath.string() % recordsRead % clustersCount).str()));
+ }
+ }
+}
+
+ControlLoadTask::ControlLoadTask(
+ std::shared_ptr<BclLoaderTaskManager>& taskManager,
+ data::RawBclBufferGroup& bclData,
+ const layout::LaneInfo &laneInfo,
+ bool ignoreMissingControls,
+ data::BclBufferVec &outputBuffer
+)
+: Task(taskManager)
+, bclData_(bclData)
+, laneInfo_(laneInfo)
+, ignoreMissingControls_(ignoreMissingControls)
+, outputBuffer_(outputBuffer)
+{
+}
+
+bool ControlLoadTask::execute(int32_t)
+{
+
+ int32_t bufferIndex = -1;
+ for (auto& rawDataForTile : bclData_)
+ {
+ ++bufferIndex;
+
+ if (rawDataForTile.controls_.empty())
+ {
+ outputBuffer_[bufferIndex].controls_.clear();
+
+ // There might not be a control file
+ continue;
+ }
+
+ data::ControlFile controlFile(rawDataForTile.controls_,
+ ignoreMissingControls_);
+
+ common::ClustersCount clustersCount = controlFile.getClustersCount();
+
+ common::resizeAndRealloc(outputBuffer_[bufferIndex].controls_, clustersCount);
+ std::size_t recordsRead = controlFile.read(&*outputBuffer_[bufferIndex].controls_.begin(), clustersCount);
+
+ int errnum = errno;
+ if (recordsRead != clustersCount)
+ {
+ if (ignoreMissingControls_)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "Control file '" << rawDataForTile.controls_.path_ << "' truncated: records_read=" << recordsRead << " records_expected=" << clustersCount << ":" << std::strerror(errnum) << " (" << errnum << ")" << std::endl;
+ outputBuffer_[bufferIndex].controls_.resize(recordsRead);
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, (boost::format("Control file '%s' truncated: records_real=%d records_expected=%d") % rawDataForTile.controls_.path_.string() % recordsRead % clustersCount).str()));
+ }
+ }
+ }
+
+ return true;
+}
+
+std::atomic<uint32_t> BclLoaderTaskManager::numTaskManagers_(0);
+std::condition_variable BclLoaderTaskManager::cvAllTaskManagersDone_;
+std::mutex BclLoaderTaskManager::mut_;
+
+BclLoaderTaskManager::BclLoaderTaskManager(bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions,
+ bool ignoreMissingControls,
+ common::TileAggregationMode aggregateTilesFlag,
+ std::shared_ptr<data::RawBclBufferGroup>& inputBuffer,
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& inputBuffersToRecycle,
+ std::shared_ptr<DemuxBuffer>& outputBuffer,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBufferQueue,
+ std::shared_ptr<data::BclBuffer::PatternedPositionsContainer>& patternedFlowcellPositions)
+: TaskManager()
+, ignoreMissingBcls_(ignoreMissingBcls)
+, ignoreMissingFilters_(ignoreMissingFilters)
+, ignoreMissingPositions_(ignoreMissingPositions)
+, ignoreMissingControls_(ignoreMissingControls)
+, aggregateTilesFlag_(aggregateTilesFlag)
+, uniqueFailedReadIndex_(0)
+, inputBuffer_(inputBuffer)
+, inputBuffersToRecycle_(inputBuffersToRecycle)
+, outputBuffer_(outputBuffer)
+, outputBufferQueue_(outputBufferQueue)
+, patternedFlowcellPositions_(patternedFlowcellPositions)
+{
+ ++numTaskManagers_;
+}
+
+BclLoaderTaskManager::~BclLoaderTaskManager()
+{
+ auto prevInputBuffer = inputBuffer_->getPrevBuffer();
+ if (prevInputBuffer)
+ {
+ prevInputBuffer->waitForFinished();
+ prevInputBuffer->reset();
+ inputBuffersToRecycle_.addData(prevInputBuffer);
+ }
+
+ // All tasks are complete.
+ // Never touch this buffer again. It could be recycled after this point.
+ inputBuffer_->setFinished();
+
+ // TODO: This can block. Do this on a separate thread. Be careful about ensuring the thread completes.
+ postExecute(*outputBuffer_);
+
+ // Get ready to set the demux info
+ for (auto& buffer : outputBuffer_->bclBuffers_)
+ {
+ buffer.samples_.resize(buffer.bcls_.front().getNumClusters());
+ }
+
+ outputBufferQueue_.addData(outputBuffer_);
+
+ --numTaskManagers_;
+
+ if (numTaskManagers_ == 0)
+ {
+ // This could be called more than once on different threads. I think
+ // this is ok.
+ cvAllTaskManagersDone_.notify_all();
+ }
+}
+
+void BclLoaderTaskManager::waitForAllTasksToFinish()
+{
+ std::unique_lock<std::mutex> lock(mut_);
+ cvAllTaskManagersDone_.wait(lock, [] { return BclLoaderTaskManager::numTaskManagers_ == 0; });
+}
+
+void BclLoaderTaskManager::validateCycleMetadata(data::PerCycleData& cycleData,
+ data::PerCycleData& prevCycleData)
+{
+ if (prevCycleData.remappedQscores_.empty() || cycleData.remappedQscores_.empty())
+ {
+ return;
+ }
+
+ if (cycleData.numBitsPerQscore_ != prevCycleData.numBitsPerQscore_ ||
+ cycleData.remappedQscores_ != prevCycleData.remappedQscores_ ||
+ (cycleData.includeNonPf_ && !prevCycleData.includeNonPf_))
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Corrupt cbcl header detected."));
+ }
+}
+
+void BclLoaderTaskManager::postExecute(DemuxBuffer& postExBuffer)
+{
+ for (data::BclBuffer& outputBuffer : postExBuffer.bclBuffers_)
+ {
+ if (patternedFlowcellPositions_)
+ {
+ patternedFlowcellPositions_->waitForReady();
+ if (outputBufferQueue_.isTerminated())
+ {
+ return;
+ }
+
+ outputBuffer.positions_ = patternedFlowcellPositions_;
+ }
+
+ const auto largestBuffer = std::max_element(
+ outputBuffer.bcls_.begin(),
+ outputBuffer.bcls_.end(),
+ [] (const data::PerCycleData& a, const data::PerCycleData& b)
+ { return a.bcls_.size() < b.bcls_.size(); }
+ );
+
+ const auto bufferSizeNonFiltered = largestBuffer->bcls_.size();
+
+ // Calculate the number of clusters (could be more than 1 per byte)
+ const auto clustersCount = bufferSizeNonFiltered * largestBuffer->getNumClustersPerByte();
+
+ const size_t largestFilteredSize = std::max_element(
+ outputBuffer.bcls_.begin(),
+ outputBuffer.bcls_.end(),
+ [] (const data::PerCycleData& a, const data::PerCycleData& b)
+ {
+ if (!a.includeNonPf_ && !b.includeNonPf_) return a.bcls_.size() < b.bcls_.size();
+ if (!a.includeNonPf_) return false;
+ if (!b.includeNonPf_) return true;
+ return false;
+ }
+ )->bcls_.size();
+
+ common::CycleNumber cycleNumber = 1;
+ for (auto &perCycleData : outputBuffer.bcls_)
+ {
+ if (cycleNumber > 1)
+ {
+ validateCycleMetadata(perCycleData, outputBuffer.bcls_[cycleNumber-2]);
+ }
+
+ auto& bclBuffer = perCycleData.bcls_;
+
+ const auto currentSize = bclBuffer.size();
+ const auto bufferSize = perCycleData.includeNonPf_ ? bufferSizeNonFiltered : largestFilteredSize;
+ if (bufferSize != currentSize)
+ {
+ bclMismatchCount("BCL",cycleNumber,outputBuffer.tileInfo_->getNumber(),currentSize,bufferSize);
+ common::resizeAndRealloc(bclBuffer, bufferSize, 0);
+ }
+ ++cycleNumber;
+ }
+ if (outputBuffer.positions_->size() != clustersCount)
+ {
+ const std::string type = (!ignoreMissingPositions_ && ignoreMissingBcls_) ? "BCL" : "positions";
+ bclMismatchCount(type,1,outputBuffer.tileInfo_->getNumber(), outputBuffer.positions_->size(), clustersCount);
+ createUniqueFakePositions(outputBuffer, clustersCount);
+ }
+ if (outputBuffer.filters_.size() != clustersCount)
+ {
+ // For cbcl files, mark reads as failed filter if the filter file is missing.
+ // We can't use this data.
+ const std::string type = (!ignoreMissingFilters_ && ignoreMissingBcls_) ? "BCL" : "filter";
+ bclMismatchCount(type,1,outputBuffer.tileInfo_->getNumber(),outputBuffer.filters_.size(), clustersCount);
+ common::resizeAndRealloc(outputBuffer.filters_,
+ clustersCount,
+ data::FilterFile::Record(aggregateTilesFlag_ == common::TileAggregationMode::CBCL ? 0 : 1));
+ }
+ if (outputBuffer.controls_.size() != clustersCount)
+ {
+ const std::string type = (!ignoreMissingControls_ && ignoreMissingBcls_) ? "BCL" : "control";
+ if (!outputBuffer.controls_.empty())
+ {
+ bclMismatchCount(type,1,outputBuffer.tileInfo_->getNumber(),outputBuffer.controls_.size(),clustersCount);
+ }
+ common::resizeAndRealloc(outputBuffer.controls_, clustersCount, 0);
+ }
+ }
+}
+
+void BclLoaderTaskManager::bclMismatchCount(const std::string& fileType,
+ common::CycleNumber cycleNumber,
+ common::TileNumber tileNumber,
+ common::RawDataBuffer::size_type realSize,
+ common::RawDataBuffer::size_type expectedSize)
+{
+ if ((ignoreMissingBcls_ && "BCL" == fileType)
+ || (ignoreMissingPositions_ && "positions" == fileType)
+ || (ignoreMissingFilters_ && "filter" == fileType)
+ || (ignoreMissingControls_ && "control" == fileType))
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "Mismatching cluster count in " << fileType << " file: Cycle #" << cycleNumber << ", Tile#" << tileNumber << ": bytes_read=" << realSize << " bytes_expected=" << expectedSize << std::endl;
+ } else {
+ BOOST_THROW_EXCEPTION(common::InputDataError((boost::format("Mismatching cluster count in %s file: Cycle #%d, Tile#%d: bytes_read=%d bytes_expected=%d") % fileType % cycleNumber % tileNumber % realSize % expectedSize).str()));
+ }
+}
+
+void BclLoaderTaskManager::createUniqueFakePositions(data::BclBuffer& outputBuffer,
+ data::PerCycleData::BclsContainer::size_type bufferSize)
+{
+ common::resizeAndRealloc(*outputBuffer.positions_, bufferSize);
+
+ for (data::PerCycleData::BclsContainer::size_type i = 0; i < bufferSize; ++i)
+ {
+ (*outputBuffer.positions_)[i].y_ = uniqueFailedReadIndex_++;
+ }
+
+}
+
+
+BclLoader::BclLoader(
+ TaskQueue& taskQueue,
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& inputBuffersToUse,
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& inputBuffersToRecycle,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBuffersToSubmit,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBuffersToUse,
+ const layout::Layout &layout,
+ const layout::LaneInfo &laneInfo,
+ bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions,
+ bool ignoreMissingControls,
+ uint32_t numTilesPerBuffer
+)
+: Stage("Bcl loading", taskQueue)
+, inputBuffersToUse_(inputBuffersToUse)
+, inputBuffersToRecycle_(inputBuffersToRecycle)
+, outputBuffersToSubmit_(outputBuffersToSubmit)
+, outputBuffersToUse_(outputBuffersToUse)
+, layout_(layout)
+, laneInfo_(laneInfo)
+, ignoreMissingBcls_(ignoreMissingBcls)
+, ignoreMissingFilters_(ignoreMissingFilters)
+, ignoreMissingPositions_(ignoreMissingPositions)
+, ignoreMissingControls_(ignoreMissingControls)
+, currentTileInfo_(laneInfo_.getTileInfos().begin())
+, aggregatedBclMutexes_()
+, cycleBciFiles_()
+, patternedFlowcellPositions_(layout.getFlowcellInfo().isPatternedFlowcell() ?
+ std::make_shared<data::BclBuffer::PatternedPositionsContainer>() :
+ std::shared_ptr<data::BclBuffer::PatternedPositionsContainer>())
+, clustersCounts_()
+, numBuffersProcessed_(0)
+, numBuffersCreated_(0)
+, numTilesPerBuffer_(numTilesPerBuffer)
+, prevInputBuffer_()
+, terminated_(false)
+{
+ common::CycleNumber cyclesCount = laneInfo.getNumCyclesToLoad();
+ BCL2FASTQ_ASSERT_MSG(cyclesCount != 0, "There are no cycles to be processed");
+
+ for (common::CycleNumber i = 0; i < cyclesCount; ++i)
+ {
+ aggregatedBclMutexes_.push_back(new std::mutex);
+ cycleBciFiles_.push_back(new data::CycleBCIFile(ignoreMissingBcls_));
+ }
+}
+
+void BclLoader::terminate()
+{
+ this->getTaskQueue().terminate();
+ inputBuffersToUse_.terminate();
+
+ outputBuffersToSubmit_.terminate();
+
+ if (patternedFlowcellPositions_)
+ {
+ patternedFlowcellPositions_->setReady();
+ }
+
+ terminated_ = true;
+}
+
+std::shared_ptr<BclLoaderTaskManager> BclLoader::getNewTaskManager()
+{
+ std::string longTileId = getTileSpec(laneInfo_);
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << longTileId << " BclLoader::getNewTaskManager " << std::endl;
+
+ std::shared_ptr<data::RawBclBufferGroup> inputBuffer;
+ if (!inputBuffersToUse_.tryGetData(inputBuffer))
+ {
+ if (!terminated_)
+ {
+ BclLoaderTaskManager::waitForAllTasksToFinish();
+ }
+
+ outputBuffersToSubmit_.setFinished();
+
+ // Only get here if we're done. Otherwise, tryGetData would wait.
+ return std::shared_ptr<BclLoaderTaskManager>();
+ }
+
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfoIter = inputBuffer->begin()->tileInfo_;
+ longTileId = getTileSpec(laneInfo_, *tileInfoIter);
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << longTileId << " Pull got: : " << std::to_string(inputBuffer->getGroupNumber()) << " Tile: "
+<< std::to_string((tileInfoIter)->getNumber()) << std::endl;
+
+ std::shared_ptr<DemuxBuffer> outputBuffer;
+ if (numBuffersCreated_ < 3)
+ {
+ if (!outputBuffersToUse_.tryGetDataCheckEmpty(outputBuffer,
+ longTileId + " Group: " + std::to_string(numBuffersCreated_) + " Push DemuxBuffer"))
+ {
+ // There aren't any buffers available. Make a new one.
+ // The memory allocation isn't too bad here, since we're recycling the buffers.
+
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << longTileId << " BclLoader::getNewTaskManager new demux buffer " << std::endl;
+ outputBuffer = std::make_shared<DemuxBuffer>(laneInfo_.getSampleInfos().size(),
+ data::ClustersPerTask,
+ laneInfo_.getNumCyclesToLoad());
+ ++numBuffersCreated_;
+ }
+ }
+ else
+ {
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << longTileId << " BclLoader::getNewTaskManager >=3 " << std::endl;
+ if (!outputBuffersToUse_.tryGetData(outputBuffer,
+ longTileId + " Group: " + std::to_string(numBuffersProcessed_) + " Push DemuxBuffer"))
+ {
+ return std::shared_ptr<BclLoaderTaskManager>();
+ }
+ }
+
+ outputBuffer->setGroupNumber(inputBuffer->getGroupNumber());
+
+ // Set previous buffer
+ inputBuffer->setPrevBuffer(prevInputBuffer_);
+ prevInputBuffer_ = inputBuffer;
+
+ ++numBuffersProcessed_;
+ return std::make_shared<BclLoaderTaskManager>(ignoreMissingBcls_,
+ ignoreMissingFilters_,
+ ignoreMissingPositions_,
+ ignoreMissingControls_,
+ layout_.getFlowcellInfo().getAggregateTilesMode(),
+ inputBuffer,
+ inputBuffersToRecycle_,
+ outputBuffer,
+ outputBuffersToSubmit_,
+ patternedFlowcellPositions_);
+}
+
+bool BclLoader::startNewTasks()
+{
+ std::string longTileId = getTileSpec(laneInfo_);
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << longTileId << " BclLoader::startNewTask " << std::endl;
+
+ TaskQueue &taskQueue = this->getTaskQueue();
+ if (taskQueue.isTerminated() || outputBuffersToSubmit_.isTerminated())
+ {
+ // Only occurs if there was an error
+ terminate();
+
+ return false;
+ }
+
+ // TODO: Tidy. currentTileInfo_ is not guaranteed to define the correct tile spec. It is used as a counter here.
+ if (currentTileInfo_ == laneInfo_.getTileInfos().end())
+ {
+ BclLoaderTaskManager::waitForAllTasksToFinish();
+ outputBuffersToSubmit_.setFinished();
+
+ return false;
+ }
+
+ // Does clustersCounts_ always work here?
+ BclLoaderTaskManagerPtr taskManager = getNewTaskManager();
+
+ if (!taskManager)
+ {
+ return false;
+ }
+
+ auto allOutputBuffers = taskManager->getOutputBuffer();
+ auto& outputBuffer = allOutputBuffers->bclBuffers_;
+ data::RawBclBufferGroup& inputBuffer = *taskManager->getInputBuffer();
+
+ layout::LaneInfo::TileInfosContainer::const_iterator myCurrentTileInfo = inputBuffer.begin()->tileInfo_;
+ uint32_t numTiles = std::min(numTilesPerBuffer_, (uint32_t)std::distance(inputBuffer.begin(), inputBuffer.end()));
+
+ longTileId = getTileSpec(laneInfo_, *myCurrentTileInfo);
+
+ // If there's no cycle data (missing bcl tile file),
+ // ensure the output buffer is empty and return true - i.e., don't submit the empty task.
+ if (!inputBuffer.hasCycleData())
+ {
+ for (data::BclBuffer& bclBuffer : outputBuffer)
+ {
+ for (auto& bufferForSingleCycle : bclBuffer.bcls_)
+ {
+ bufferForSingleCycle.bcls_.resize(0);
+ }
+ }
+
+ return true;
+ }
+
+ const auto aggregateTilesMode = layout_.getFlowcellInfo().getAggregateTilesMode();
+ const bool isPatternedFlowcell = layout_.getFlowcellInfo().isPatternedFlowcell();
+
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfoIter = myCurrentTileInfo;
+ clustersCounts_.clear();
+
+ outputBuffer.resize(numTiles);
+
+ size_t bufferNum = 0;
+ for (auto& buffer : outputBuffer)
+ {
+ buffer.bcls_.resize(laneInfo_.getNumCyclesToLoad());
+ buffer.tileInfo_ = myCurrentTileInfo + bufferNum;
+ ++bufferNum;
+ }
+
+ // Add the positions task first, since if it's a patterned flowcell, we
+ // only need to load it once, but all tiles will need the data.
+ if (!isPatternedFlowcell || numBuffersProcessed_ == 1)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << "Queue new positions-load task " << std::endl;
+ taskQueue.addData(std::make_shared<PositionsLoadTask>(
+ taskManager,
+ inputBuffer,
+ data::Instrument::isPositionFileAggregated(aggregateTilesMode),
+ laneInfo_,
+ myCurrentTileInfo,
+ clustersCounts_,
+ ignoreMissingPositions_,
+ outputBuffer,
+ patternedFlowcellPositions_
+ ), longTileId + " Task: Position load");
+ }
+
+ BclFilesContainer::size_type bclIdx = 0;
+ for (const auto& readInfo : laneInfo_.readInfos())
+ {
+ // We want all the cycles for the index reads, even if they were masked.
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << "Queue new bcl-loader tasks " << std::endl;
+ for (const auto& cycleInfo : readInfo.cyclesToLoad())
+ {
+ taskQueue.addData(std::make_shared<BclLoadTask>(
+ taskManager,
+ inputBuffer,
+ (aggregateTilesMode == common::TileAggregationMode::AGGREGATED ||
+ aggregateTilesMode == common::TileAggregationMode::CBCL) ? &aggregatedBclMutexes_[bclIdx] : NULL,
+ aggregateTilesMode,
+ laneInfo_,
+ cycleInfo,
+ tileInfoIter,
+ ignoreMissingBcls_,
+ bclIdx,
+ *allOutputBuffers
+ ));
+
+ ++bclIdx;
+ }
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << "Queue new filter-load task " << std::endl;
+ taskQueue.addData(std::make_shared<FilterLoadTask>(
+ taskManager,
+ inputBuffer,
+ aggregateTilesMode,
+ laneInfo_,
+ myCurrentTileInfo,
+ clustersCounts_,
+ ignoreMissingFilters_,
+ outputBuffer
+ ), longTileId + " Task: filter-load");
+
+ taskQueue.addData(std::make_shared<ControlLoadTask>(
+ taskManager,
+ inputBuffer,
+ laneInfo_,
+ ignoreMissingControls_,
+ outputBuffer
+ ));
+
+ currentTileInfo_ += outputBuffer.size();
+
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << longTileId << " ~BclLoader::startNewTask " << std::endl;
+ return true;
+}
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/conversion/BclReader.cpp b/src/cxx/lib/conversion/BclReader.cpp
new file mode 100644
index 0000000..41a6c89
--- /dev/null
+++ b/src/cxx/lib/conversion/BclReader.cpp
@@ -0,0 +1,767 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BclReader.cpp
+ *
+ * \brief Implementation of BCL reader.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+#include "conversion/BclReader.hh"
+#include "data/CbclFile.hh"
+#include "data/TileBclFileReader.hh"
+#include "data/AggregatedBclFileReader.hh"
+#include "data/BclFile.hh"
+#include "data/Instrument.hh"
+
+#include "common/Debug.hh"
+#include "common/Exceptions.hh"
+#include "common/Types.hh"
+#include "io/GzipCompressor.hh"
+
+#include <errno.h>
+#include <algorithm>
+#include <utility>
+
+#include <boost/format.hpp>
+#include <boost/filesystem.hpp>
+
+#include <boost/iostreams/filtering_stream.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+
+namespace bcl2fastq {
+namespace conversion {
+
+namespace
+{
+ std::string getTileSpec(
+ const layout::LaneInfo &laneInfo,
+ const layout::TileInfo &tileInfo)
+ {
+ std::stringstream ss;
+ ss << "Lane: " << laneInfo.getNumber() << " Tile: " << tileInfo.getNumber();
+
+ return ss.str();
+ }
+}
+
+
+BclReadTask::BclReadTask(
+ std::shared_ptr<BclReaderTaskManager>& taskManager,
+ data::BclFileReader& bclFileReader,
+ data::RawBclBufferGroup& outputBuffer
+)
+: Task(taskManager)
+, bclFileReader_(bclFileReader)
+, outputBuffer_(outputBuffer)
+{
+}
+
+bool BclReadTask::execute(int32_t)
+{
+ return bclFileReader_.read(outputBuffer_);
+}
+
+PositionsReadTask::PositionsReadTask(
+ std::shared_ptr<BclReaderTaskManager>& taskManager,
+ std::shared_ptr<io::SyncFile> &positionsFile,
+ const boost::filesystem::path &intensitiesDir,
+ bool aggregateTilesFlag,
+ bool isPatternedFlowcell,
+ const layout::LaneInfo &laneInfo,
+ layout::LaneInfo::TileInfosContainer::const_iterator &tileInfoIter,
+ bool ignoreMissingPositions,
+ data::RawBclBufferGroup& outputBuffer,
+ std::shared_ptr<PatternedPositionsContainer>& patternedFlowcellPositions
+)
+: Task(taskManager)
+, positionsFile_(positionsFile)
+, intensitiesDir_(intensitiesDir)
+, aggregateTilesFlag_(aggregateTilesFlag)
+, isPatternedFlowcell_(isPatternedFlowcell)
+, laneInfo_(laneInfo)
+, tileInfoIter_(tileInfoIter)
+, ignoreMissingPositions_(ignoreMissingPositions)
+, outputBuffer_(outputBuffer)
+, patternedFlowcellPositions_(patternedFlowcellPositions)
+{
+}
+
+bool PositionsReadTask::execute(int32_t)
+{
+ static int sequentialId = 0; ++sequentialId;
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "PositionsReadTask::execute " << sequentialId << std::endl;
+ if (isPatternedFlowcell_ && patternedFlowcellPositions_)
+ {
+ if (readEntirePosFile(*patternedFlowcellPositions_))
+ {
+ patternedFlowcellPositions_->setReady();
+ return true;
+ }
+ else
+ {
+ patternedFlowcellPositions_->setReady();
+ return false;
+ }
+ }
+ else
+ {
+ for (auto& outputBuffer : outputBuffer_)
+ {
+ if (!outputBuffer.positions_)
+ {
+ outputBuffer.positions_ = std::make_shared<common::RawDataBuffer>();
+ }
+
+ if (!(aggregateTilesFlag_ ? readPartOfAggregatedPosFile(*outputBuffer.positions_) : readEntirePosFile(*outputBuffer.positions_)))
+ {
+ return false;
+ }
+
+ ++tileInfoIter_;
+ }
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "PositionsReadTask::execute done" << sequentialId << std::endl;
+ return true;
+}
+
+bool PositionsReadTask::readEntirePosFile(common::RawDataBuffer& outputBuffer)
+{
+ static int sequentialId = 0; ++sequentialId;
+
+ size_t headerSize;
+ boost::filesystem::path posFilePath;
+ if (!data::PositionsFileFactory::doesFileExist(intensitiesDir_,
+ aggregateTilesFlag_,
+ isPatternedFlowcell_,
+ laneInfo_.getNumber(),
+ tileInfoIter_->getNumber(),
+ headerSize,
+ posFilePath))
+ {
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "FAIL PositionsReadTask readEntireFile " << sequentialId << " " << posFilePath.string() << std::endl;
+ if (ignoreMissingPositions_)
+ {
+ return true;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(ENOENT, "Unable to find positions file: '" + posFilePath.string() + "'"));
+ }
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "PositionsReadTask readEntireFile " << sequentialId << " " << posFilePath.string() << std::endl;
+
+ io::UnprocessedFile positionsFile(posFilePath,
+ ignoreMissingPositions_);
+
+ outputBuffer.path_ = positionsFile.getPath();
+ return positionsFile.readEntireFile(outputBuffer);
+}
+
+bool PositionsReadTask::readPartOfAggregatedPosFile(common::RawDataBuffer& outputBuffer)
+{
+ static int sequentialId = 0; ++sequentialId;
+
+ io::SyncFile::SyncFileReader fileReader(*positionsFile_,
+ std::distance(laneInfo_.getTileInfos().begin(), tileInfoIter_));
+
+ size_t headerSize = 0;
+ if (!positionsFile_->isOpen())
+ {
+ boost::filesystem::path filePath;
+ if (!(data::PositionsFileFactory::doesFileExist(intensitiesDir_,
+ aggregateTilesFlag_,
+ isPatternedFlowcell_,
+ laneInfo_.getNumber(),
+ tileInfoIter_->getNumber(),
+ headerSize,
+ filePath)))
+ {
+ if (ignoreMissingPositions_)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "SKIP PositionsReadTask readPart...File " << sequentialId << " " << filePath.string() << std::endl;
+ return true;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(ENOENT, "Unable to find position file: '" + filePath.string() + "'"));
+ }
+ }
+
+ fileReader.openFile(filePath,
+ ignoreMissingPositions_);
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "PositionsReadTask readPart...File " << sequentialId << std::endl;
+ std::size_t skippedClustersCount = tileInfoIter_->getSkippedClustersCount();
+ bool skipHeader = (tileInfoIter_ == laneInfo_.getTileInfos().begin()) && (tileInfoIter_->getIndex() != 0);
+ if (skippedClustersCount || skipHeader)
+ {
+ fileReader.seek(skippedClustersCount*sizeof(data::PositionsFile::Record) + (skipHeader ? headerSize : 0));
+ }
+
+ outputBuffer.path_ = positionsFile_->getPath();
+ fileReader.read(outputBuffer,
+ (skipHeader ? 0 : headerSize) + sizeof(data::PositionsFile::Record)*tileInfoIter_->getClustersCount());
+
+ return true;
+}
+
+FilterReadTask::FilterReadTask(
+ std::shared_ptr<BclReaderTaskManager>& taskManager,
+ std::shared_ptr<io::SyncFile> &filterFile,
+ const boost::filesystem::path &inputDir,
+ common::TileAggregationMode tileAggregationMode,
+ const layout::LaneInfo &laneInfo,
+ layout::LaneInfo::TileInfosContainer::const_iterator &tileInfoIter,
+ bool ignoreMissingFilters,
+ data::RawBclBufferGroup& outputBuffer
+)
+: Task(taskManager)
+, filterFile_(filterFile)
+, inputDir_(inputDir)
+, tileAggregationMode_(tileAggregationMode)
+, aggregateTilesFlag_(data::Instrument::isFilterFileAggregated(tileAggregationMode))
+, laneInfo_(laneInfo)
+, tileInfoIter_(tileInfoIter)
+, ignoreMissingFilters_(ignoreMissingFilters)
+, outputBuffer_(outputBuffer)
+{
+}
+
+bool FilterReadTask::execute(int32_t)
+{
+ static int sequentialId = 0; ++sequentialId;
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "FilterReadTask::execute " << sequentialId << std::endl;
+ for (data::RawBclBuffer& rawBclBuffer : outputBuffer_)
+ {
+ if (!(aggregateTilesFlag_ ? readPartOfAggregatedFilterFile(rawBclBuffer.filters_) : readEntireFilterFile(rawBclBuffer)))
+ {
+ return false;
+ }
+
+ ++tileInfoIter_;
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "FilterReadTask::execute done" << sequentialId << std::endl;
+ return true;
+}
+
+bool FilterReadTask::readEntireFilterFile(data::RawBclBuffer& rawBclBuffer)
+{
+ static int sequentialId = 0; ++sequentialId;
+
+ common::RawDataBuffer& rawFiltersBuffer = rawBclBuffer.filters_;
+
+ // Ensure there isn't any data from previous tiles
+ rawFiltersBuffer.clear();
+
+ size_t headerSize;
+ boost::filesystem::path filePath;
+ if (!(data::FilterFile::doesFileExist(inputDir_,
+ aggregateTilesFlag_,
+ laneInfo_.getNumber(),
+ tileInfoIter_->getNumber(),
+ headerSize,
+ filePath)))
+ {
+ if (ignoreMissingFilters_)
+ {
+ if (tileAggregationMode_ == common::TileAggregationMode::CBCL)
+ {
+ // If cbcl filter file is missing then we can't map the tile indices so ignore this tile.
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "Unable to find CBCL filter file: '"
+ << filePath.string() << "'. This tile will be ignored." << std::endl;
+ }
+ return true;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(ENOENT, "Unable to find filter file: '" + filePath.string() + "'"));
+ }
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "FilterReadTask::execute " << sequentialId << " " << filePath.string() << std::endl;
+
+ bool status = false;
+ if (!aggregateTilesFlag_)
+ {
+ // A. NO WAIT APPROACH
+ io::UnprocessedFile filterFile(filePath,
+ ignoreMissingFilters_);
+
+ status = filterFile.readEntireFile(rawFiltersBuffer);
+ }
+ else
+ {
+ io::SyncFile::SyncFileReader fileReader(*filterFile_,
+ std::distance(laneInfo_.getTileInfos().begin(), tileInfoIter_));
+
+ // B. Sync read.
+ fileReader.openFile(filePath,
+ ignoreMissingFilters_);
+ status = fileReader.readEntireFile(rawFiltersBuffer);
+ }
+
+ rawFiltersBuffer.path_ = filePath;
+
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "...done FilterReadTask::execute " << sequentialId << std::endl;
+
+ return status;
+}
+
+bool FilterReadTask::readPartOfAggregatedFilterFile(common::RawDataBuffer& outputBuffer)
+{
+ io::SyncFile::SyncFileReader fileReader(*filterFile_,
+ std::distance(laneInfo_.getTileInfos().begin(), tileInfoIter_));
+
+ size_t headerSize = 0;
+ if (!filterFile_->isOpen())
+ {
+ boost::filesystem::path filePath;
+ if (!(data::FilterFile::doesFileExist(inputDir_,
+ aggregateTilesFlag_,
+ laneInfo_.getNumber(),
+ tileInfoIter_->getNumber(),
+ headerSize,
+ filePath)))
+ {
+ if (ignoreMissingFilters_)
+ {
+ return true;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(ENOENT, "Unable to find filter file: '" + filePath.string() + "'"));
+ }
+ }
+
+ fileReader.openFile(filePath,
+ ignoreMissingFilters_);
+ }
+
+
+ bool skipHeader = (tileInfoIter_ == laneInfo_.getTileInfos().begin()) && (tileInfoIter_->getIndex() != 0);
+ std::size_t skippedClustersCount = tileInfoIter_->getSkippedClustersCount();
+ if (skippedClustersCount || skipHeader)
+ {
+ fileReader.seek(skippedClustersCount + (skipHeader ? headerSize : 0));
+ }
+
+ outputBuffer.path_ = filterFile_->getPath();
+ fileReader.read(outputBuffer,
+ (skipHeader ? 0 : headerSize) + tileInfoIter_->getClustersCount());
+
+ return true;
+}
+
+ControlReadTask::ControlReadTask(
+ std::shared_ptr<BclReaderTaskManager>& taskManager,
+ const boost::filesystem::path &inputDir,
+ const layout::LaneInfo &laneInfo,
+ layout::LaneInfo::TileInfosContainer::const_iterator &tileInfoIter,
+ bool ignoreMissingControls,
+ data::RawBclBufferGroup& outputBuffer
+)
+: Task(taskManager)
+, inputDir_(inputDir)
+, laneInfo_(laneInfo)
+, tileInfoIter_(tileInfoIter)
+, ignoreMissingControls_(ignoreMissingControls)
+, outputBuffer_(outputBuffer)
+{
+}
+
+bool ControlReadTask::execute(int32_t)
+{
+ for (auto& outputBuffer : outputBuffer_)
+ {
+ outputBuffer.controls_.clear();
+
+ if (!readEntireControlFile(outputBuffer.controls_))
+ {
+ return false;
+ }
+
+ ++tileInfoIter_;
+ }
+
+ return true;
+}
+
+bool ControlReadTask::readEntireControlFile(common::RawDataBuffer& outputBuffer)
+{
+ boost::filesystem::path filePath;
+ if (!(data::ControlFile::doesFileExist(inputDir_,
+ laneInfo_.getNumber(),
+ tileInfoIter_->getNumber(),
+ filePath)))
+ {
+ return true;
+ }
+
+ io::UnprocessedFile controlFile(filePath,
+ ignoreMissingControls_);
+
+ return controlFile.readEntireFile(outputBuffer);
+}
+
+std::atomic<uint32_t> BclReaderTaskManager::numTaskManagers_(0);
+std::condition_variable BclReaderTaskManager::cvAllTaskManagersDone_;
+std::mutex BclReaderTaskManager::mut_;
+
+BclReaderTaskManager::BclReaderTaskManager(bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions,
+ bool ignoreMissingControls,
+ std::shared_ptr<data::RawBclBufferGroup>& outputBuffer,
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& outputBufferQueue,
+ std::shared_ptr<PatternedPositionsContainer>& patternedFlowcellPositions)
+: TaskManager()
+, ignoreMissingBcls_(ignoreMissingBcls)
+, ignoreMissingFilters_(ignoreMissingFilters)
+, ignoreMissingPositions_(ignoreMissingPositions)
+, ignoreMissingControls_(ignoreMissingControls)
+, outputBuffer_(outputBuffer)
+, outputBufferQueue_(outputBufferQueue)
+, patternedFlowcellPositions_(patternedFlowcellPositions)
+{
+ ++numTaskManagers_;
+}
+
+BclReaderTaskManager::~BclReaderTaskManager()
+{
+ // All tasks are complete.
+ postExecute();
+
+ // TODO: This can block. Do this on a separate thread. Be careful about ensuring the thread completes.
+ outputBufferQueue_.addData(outputBuffer_);
+ --numTaskManagers_;
+
+
+ if (numTaskManagers_ == 0)
+ {
+ // This could be called more than once on different threads. I think
+ // this is ok.
+ cvAllTaskManagersDone_.notify_all();
+ }
+}
+
+void BclReaderTaskManager::waitForAllTasksToFinish()
+{
+ std::unique_lock<std::mutex> lock(mut_);
+ cvAllTaskManagersDone_.wait(lock, [] { return BclReaderTaskManager::numTaskManagers_ == 0; });
+}
+
+void BclReaderTaskManager::postExecute()
+{
+ if (!patternedFlowcellPositions_)
+ {
+ // nothing to do
+ return;
+ }
+
+ patternedFlowcellPositions_->waitForReady();
+ for (auto& outputBufferForTile : *outputBuffer_)
+ {
+ outputBufferForTile.positions_ = patternedFlowcellPositions_;
+ }
+}
+
+BclReader::BclReader(
+ TaskQueue& taskQueue,
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& outputBuffersToSubmit,
+ ThreadSafeQueue<std::shared_ptr<data::RawBclBufferGroup>>& outputBuffersToUse,
+ const layout::Layout &layout,
+ const layout::LaneInfo &laneInfo,
+ bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions,
+ bool ignoreMissingControls,
+ const boost::filesystem::path &inputDir,
+ const boost::filesystem::path &intensitiesDir
+)
+: Stage("Bcl reading", taskQueue)
+, outputBuffersToSubmit_(outputBuffersToSubmit)
+, outputBuffersToUse_(outputBuffersToUse)
+, layout_(layout)
+, laneInfo_(laneInfo)
+, ignoreMissingBcls_(ignoreMissingBcls)
+, ignoreMissingFilters_(ignoreMissingFilters)
+, ignoreMissingPositions_(ignoreMissingPositions)
+, ignoreMissingControls_(ignoreMissingControls)
+, currentTileInfo_(laneInfo_.getTileInfos().begin())
+, inputDir_(inputDir)
+, intensitiesDir_(intensitiesDir)
+, bclFileReaders_()
+, positionsFile_(std::make_shared<io::SyncFile>(std::ios_base::in | std::ios_base::binary))
+, filterFile_(std::make_shared<io::SyncFile>(std::ios_base::in | std::ios_base::binary))
+, patternedFlowcellPositions_(layout.getFlowcellInfo().isPatternedFlowcell() ?
+ std::make_shared<PatternedPositionsContainer>() :
+ std::shared_ptr<PatternedPositionsContainer>())
+, numTilesPerBuffer_(1)
+, numBuffersCreated_(0)
+, groupNumber_(0)
+{
+ common::CycleNumber cyclesCount = laneInfo.getNumCyclesToLoad();
+ BCL2FASTQ_ASSERT_MSG(cyclesCount != 0, "There are no cycles to be processed");
+
+ auto aggregateTilesMode = layout_.getFlowcellInfo().getAggregateTilesMode();
+
+ size_t cycleIndex = 0;
+ for (const auto& readInfo : laneInfo.readInfos())
+ {
+ for (const auto& cycleInfo : readInfo.cyclesToLoad())
+ {
+ std::shared_ptr<io::SyncFile> bclFile = std::make_shared<io::SyncFile>(std::ios_base::in | std::ios_base::binary);
+
+ switch (aggregateTilesMode)
+ {
+ case common::TileAggregationMode::AGGREGATED:
+ bclFileReaders_.push_back(new data::AggregatedBclFileReader(inputDir,
+ laneInfo,
+ cycleInfo.getNumber(),
+ cycleIndex,
+ ignoreMissingBcls_,
+ bclFile,
+ std::make_shared<data::CycleBCIFile>(ignoreMissingBcls_)));
+ break;
+ case common::TileAggregationMode::CBCL:
+ bclFileReaders_.push_back(new data::CbclFileReader(inputDir,
+ laneInfo,
+ cycleInfo.getNumber(),
+ cycleIndex,
+ ignoreMissingBcls_,
+ layout.getTileFileMap(),
+ bclFile));
+ break;
+ case common::TileAggregationMode::NON_AGGREGATED:
+ bclFileReaders_.push_back(new data::TileBclFileReader(inputDir,
+ laneInfo,
+ cycleInfo.getNumber(),
+ cycleIndex,
+ ignoreMissingBcls_,
+ bclFile));
+ break;
+ default:
+ BCL2FASTQ_ASSERT_MSG(false, "Unexpected aggregation mode type");
+ }
+
+ ++cycleIndex;
+ }
+ }
+
+ calculateNumTilesPerBuffer();
+}
+
+void BclReader::calculateNumTilesPerBuffer()
+{
+ // We want a minimum number of clusters per sample
+ size_t desiredClustersCount = data::ClustersPerReadBuffer;
+ size_t totalClusters = 0;
+ for (auto& tile : laneInfo_.getTileInfos())
+ {
+ if (!tile.haveClustersCount() || tile.getClustersCount() == 0)
+ {
+ // Don't have the clusters counts
+ return;
+ }
+
+ totalClusters += tile.getClustersCount();
+ }
+
+ if (totalClusters <= desiredClustersCount)
+ {
+ numTilesPerBuffer_ = laneInfo_.getTileInfos().size();
+ return;
+ }
+
+ numTilesPerBuffer_ = laneInfo_.getTileInfos().size() / (totalClusters / desiredClustersCount);
+ if (numTilesPerBuffer_ == 0) numTilesPerBuffer_ = 1;
+}
+
+std::shared_ptr<BclReaderTaskManager> BclReader::getNewTaskManager()
+{
+ const std::string longTileId = getTileSpec(laneInfo_, *currentTileInfo_);
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << longTileId << " BclReader::getNewTaskManager 1 " << std::endl;
+
+ std::shared_ptr<data::RawBclBufferGroup> outputBuffer;
+
+ if (outputBuffersToSubmit_.isTerminated())
+ {
+ // This should only occur if an error was encountered in the BclLoader
+ return std::shared_ptr<BclReaderTaskManager>();
+ }
+
+ if (numBuffersCreated_ < 3)
+ {
+ if (!outputBuffersToUse_.tryGetDataCheckEmpty(outputBuffer,
+ longTileId + " Group: " + std::to_string(numBuffersCreated_) + " Push RawBclBufferGroup "))
+ {
+ // There aren't any buffers available. Make a new one.
+ // The memory allocation isn't too bad here, since we're recycling the buffers.
+ outputBuffer = std::make_shared<data::RawBclBufferGroup>(numTilesPerBuffer_, laneInfo_.getNumCyclesToLoad());
+ ++numBuffersCreated_;
+ }
+ else
+ {
+ outputBuffer->waitForFinished();
+ }
+ }
+ else
+ {
+ if (!outputBuffersToUse_.tryGetData(outputBuffer,
+ longTileId + " Group: " + std::to_string(numBuffersCreated_) + " Push RawBclBufferGroup "))
+ {
+ return std::shared_ptr<BclReaderTaskManager>();
+ }
+ else
+ {
+ outputBuffer->waitForFinished();
+ }
+ }
+
+ outputBuffer->setGroupNumber(groupNumber_++);
+
+ return std::make_shared<BclReaderTaskManager>(ignoreMissingBcls_,
+ ignoreMissingFilters_,
+ ignoreMissingPositions_,
+ ignoreMissingControls_,
+ outputBuffer,
+ outputBuffersToSubmit_,
+ patternedFlowcellPositions_);
+}
+
+void BclReader::terminate()
+{
+ this->getTaskQueue().terminate();
+ outputBuffersToUse_.terminate();
+}
+
+bool BclReader::startNewTasks()
+{
+ const std::string longTileId = getTileSpec(laneInfo_, *currentTileInfo_);
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << longTileId << " BclReader::startNewTask " << std::endl;
+
+ TaskQueue &taskQueue = this->getTaskQueue();
+ if (taskQueue.isTerminated())
+ {
+ outputBuffersToSubmit_.terminate();
+
+ // Only occurs if there was an error
+ return false;
+ }
+
+ if (currentTileInfo_ == laneInfo_.getTileInfos().end())
+ {
+ BclReaderTaskManager::waitForAllTasksToFinish();
+ outputBuffersToSubmit_.setFinished();
+
+ return false;
+ }
+
+ auto taskManager = getNewTaskManager();
+
+ if (!taskManager)
+ {
+ return false;
+ }
+
+ auto& outputBuffer = taskManager->getOutputBuffer();
+
+ size_t remainingTiles = std::distance(currentTileInfo_, laneInfo_.getTileInfos().end());
+ if (remainingTiles < numTilesPerBuffer_)
+ {
+ outputBuffer.resize(remainingTiles);
+ }
+
+ const auto aggregateTilesMode = layout_.getFlowcellInfo().getAggregateTilesMode();
+ const bool isPatternedFlowcell = layout_.getFlowcellInfo().isPatternedFlowcell();
+
+ auto tileInfoForBuffers = currentTileInfo_;
+ for (auto& buffer : outputBuffer)
+ {
+ buffer.cycleData_.resize(laneInfo_.getNumCyclesToLoad());
+ buffer.gzipDecompressors_.resize(buffer.cycleData_.size());
+ buffer.tileInfo_ = tileInfoForBuffers;
+ ++tileInfoForBuffers;
+ }
+
+ // Add the positions task first, since if it's a patterned flowcell, we
+ // only need to load it once, but all tiles will need the data.
+ if (!isPatternedFlowcell || numBuffersCreated_ == 1)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << longTileId << " Queue new positions-read task " << std::endl;
+ taskQueue.addData(std::make_shared<PositionsReadTask>(
+ taskManager,
+ positionsFile_,
+ intensitiesDir_,
+ data::Instrument::isPositionFileAggregated(aggregateTilesMode),
+ //aggregateTilesMode == common::TileAggregationMode::AGGREGATED ||
+ // aggregateTilesMode == common::TileAggregationMode::CBCL,
+ isPatternedFlowcell,
+ laneInfo_,
+ currentTileInfo_,
+ ignoreMissingPositions_,
+ outputBuffer,
+ patternedFlowcellPositions_
+ ), longTileId + " Task: Position read");
+ }
+
+ BclFilesContainer::size_type bclIdx = 0;
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << longTileId << " Queue new bcl-read tasks " << std::endl;
+ for (const auto& readInfo : laneInfo_.readInfos())
+ {
+ // We want all the cycles for the index reads, even if they were masked.
+ for (const auto& cycleInfo : readInfo.cyclesToLoad())
+ {
+ taskQueue.addData(std::make_shared<BclReadTask>(
+ taskManager,
+ bclFileReaders_.at(bclIdx),
+ outputBuffer
+ ));
+ //), longTileId + " Cycle: " + std::to_string(cycleInfo.getNumber()) + " Read: " + std::to_string(readInfo.getNumber()) + " Task: bcl-read");
+
+ ++bclIdx;
+ }
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << longTileId << " Queue new filter-read task " << std::endl;
+ taskQueue.addData(std::make_shared<FilterReadTask>(
+ taskManager,
+ filterFile_,
+ inputDir_,
+ aggregateTilesMode,
+ laneInfo_,
+ currentTileInfo_,
+ ignoreMissingFilters_,
+ outputBuffer
+ ), longTileId + " Task: filter-read");
+
+ taskQueue.addData(std::make_shared<ControlReadTask>(
+ taskManager,
+ inputDir_,
+ laneInfo_,
+ currentTileInfo_,
+ ignoreMissingControls_,
+ outputBuffer
+ ));
+
+ currentTileInfo_ += (remainingTiles < numTilesPerBuffer_ ? remainingTiles : numTilesPerBuffer_);
+
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << longTileId << "End BclReader::startNewTask " << std::endl;
+ return true;
+}
+
+} // namespace conversion
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/conversion/CMakeLists.txt b/src/cxx/lib/conversion/CMakeLists.txt
new file mode 100644
index 0000000..26c8f9d
--- /dev/null
+++ b/src/cxx/lib/conversion/CMakeLists.txt
@@ -0,0 +1,23 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the cxx/lib/conversion subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CXX_LIBRARY_CMAKE})
+
+
diff --git a/src/cxx/lib/conversion/Converter.cpp b/src/cxx/lib/conversion/Converter.cpp
new file mode 100644
index 0000000..0fb9986
--- /dev/null
+++ b/src/cxx/lib/conversion/Converter.cpp
@@ -0,0 +1,1168 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Converter.cpp
+ *
+ * \brief Implementation of BCL to FASTQ converter.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#include <boost/ref.hpp>
+#include <boost/bind.hpp>
+
+#include <boost/assign/list_of.hpp>
+#include <boost/filesystem/fstream.hpp>
+#include <boost/range/adaptor/reversed.hpp>
+
+#include "common/Logger.hh"
+#include "common/Exceptions.hh"
+#include "common/FileSystem.hh"
+#include "stats/BarcodeStats.hh"
+#include "stats/TileStats.hpp"
+#include "stats/DemultiplexingStatsXml.hh"
+#include "stats/ConversionStatsXml.hh"
+#include "stats/DemuxReportGenerator.hh"
+#include "stats/Json.hh"
+
+#include "conversion/Converter.hh"
+
+
+namespace bcl2fastq
+{
+namespace conversion
+{
+
+
+Converter::Converter(
+ const bcl2fastq::config::Bcl2FastqOptions &options,
+ const layout::Layout &layout,
+ const layout::LaneInfo &laneInfo,
+ bcl2fastq::conversion::ConverterStats &allStats
+)
+: layout_(layout)
+, laneInfo_(laneInfo)
+, statsFinalizer_(laneInfo.getSampleInfos().size()==1,
+ allStats.getConversionStats(laneInfo_),
+ allStats.getDemuxStats(laneInfo_))
+, allStats_(allStats)
+, inUseRawBclBufferQueue_(MAX_BCL_BUFFER_QUEUE_DEPTH)
+, recycledRawBclBufferQueue_()
+, inUseBclBufferQueue_(MAX_BCL_BUFFER_QUEUE_DEPTH)
+, recycledBclBufferQueue_()
+, inUseDemuxBufferQueue_()
+, inUseFastqBufferMap_()
+, recycledFastqBufferQueue_()
+, processTaskQueue_()
+, ioReadTaskQueue_()
+, ioWriteTaskQueue_()
+, terminator_()
+, barcodeTranslationTable_(
+ laneInfo.sampleInfosBegin(),
+ laneInfo.sampleInfosEnd(),
+ options.getBarcodeMismatches()
+ )
+, bclReader_(
+ ioReadTaskQueue_,
+ inUseRawBclBufferQueue_,
+ recycledRawBclBufferQueue_,
+ layout,
+ laneInfo,
+ options.getIgnoreMissingBcls(),
+ options.getIgnoreMissingFilters(),
+ options.getIgnoreMissingPositions(),
+ options.getIgnoreMissingControls(),
+ options.getInputDir(),
+ options.getIntensitiesDir()
+ )
+, bclLoader_(
+ processTaskQueue_,
+ inUseRawBclBufferQueue_,
+ recycledRawBclBufferQueue_,
+ inUseBclBufferQueue_,
+ recycledBclBufferQueue_,
+ layout,
+ laneInfo,
+ options.getIgnoreMissingBcls(),
+ options.getIgnoreMissingFilters(),
+ options.getIgnoreMissingPositions(),
+ options.getIgnoreMissingControls(),
+ bclReader_.getNumTilesPerBuffer()
+ )
+, demultiplexer_(
+ processTaskQueue_,
+ inUseBclBufferQueue_,
+ inUseDemuxBufferQueue_,
+ layout,
+ laneInfo,
+ barcodeTranslationTable_,
+ options.getIncludeNonPfClusters(),
+ options.getFastqCreatorThreadsCount(),
+ allStats.getDemuxStats(laneInfo)
+ )
+, fastqCreator_(
+ createNewFastqCreator(
+ processTaskQueue_,
+ inUseDemuxBufferQueue_,
+ recycledBclBufferQueue_,
+ inUseFastqBufferMap_,
+ recycledFastqBufferQueue_,
+ layout,
+ laneInfo,
+ options,
+ allStats))
+, fastqWriter_(
+ options.getFastqWriterThreadsCount() == 0 ? ioReadTaskQueue_ : ioWriteTaskQueue_,
+ inUseFastqBufferMap_,
+ recycledFastqBufferQueue_,
+ layout,
+ laneInfo,
+ options.getCreateFastqsForIndexReads(),
+ options.getOutputDir(),
+ options.noLaneSplitting()
+ )
+, processThreadPool_(options.getFastqCreatorThreadsCount(),
+ processTaskQueue_,
+ terminator_)
+, ioReadThreadPool_(options.getBclLoaderThreadsCount(),
+ ioReadTaskQueue_,
+ terminator_)
+, ioWriteThreadPool_(options.getFastqWriterThreadsCount(),
+ ioWriteTaskQueue_,
+ terminator_)
+{
+}
+
+std::unique_ptr<Stage> Converter::createNewFastqCreator(TaskQueue& processTaskQueue,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inUseDemuxBufferQueue,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& recycledBclBufferQueue,
+ ThreadSafeMap<uint32_t, std::shared_ptr<FastqBuffer>>& inUseFastqBufferMap,
+ ThreadSafeQueue<std::shared_ptr<FastqBuffer>>& recycledFastqBufferQueue,
+ const layout::Layout& layout,
+ const layout::LaneInfo& laneInfo,
+ const bcl2fastq::config::Bcl2FastqOptions& options,
+ bcl2fastq::conversion::ConverterStats& allStats)
+{
+ // The purpose of the FastqCreator being a template on the NumBasesPerByte is to avoid
+ // costly branch logic inside of FastqIterator. Surprisingly, including even a simple switch
+ // statement inside of the FastqIterator slows down all of bcl2fastq by 30%. This makes some
+ // sense if you consider that we can be iterating over trillions of bases. Also, the use
+ // of branches inside of FastqIterator makes vector AVX instructions impossible.
+ switch (layout.getNumBasesPerByte())
+ {
+ case common::NumBasesPerByte::ONE:
+ return std::unique_ptr<FastqCreator<common::NumBasesPerByte::ONE>>(
+ new FastqCreator<common::NumBasesPerByte::ONE>(
+ processTaskQueue,
+ inUseDemuxBufferQueue,
+ recycledBclBufferQueue,
+ inUseFastqBufferMap,
+ recycledFastqBufferQueue,
+ layout,
+ laneInfo,
+ options.getMaskShortAdapterReads(),
+ options.getAdapterStringency(),
+ options.getGenerateReverseComplementFastqs(),
+ options.getIncludeNonPfClusters(),
+ options.getCreateFastqsForIndexReads(),
+ options.useBgzfCompression(),
+ options.getFastqCompressionLevel(),
+ options.findAdaptersWithSlidingWindow(),
+ options.getFastqCreatorThreadsCount(),
+ allStats.getConversionStats(laneInfo),
+ allStats.getUnknownBarcodeStat(laneInfo)));
+ case common::NumBasesPerByte::TWO:
+ return std::unique_ptr<FastqCreator<common::NumBasesPerByte::TWO>>(
+ new FastqCreator<common::NumBasesPerByte::TWO>(
+ processTaskQueue,
+ inUseDemuxBufferQueue,
+ recycledBclBufferQueue,
+ inUseFastqBufferMap,
+ recycledFastqBufferQueue,
+ layout,
+ laneInfo,
+ options.getMaskShortAdapterReads(),
+ options.getAdapterStringency(),
+ options.getGenerateReverseComplementFastqs(),
+ options.getIncludeNonPfClusters(),
+ options.getCreateFastqsForIndexReads(),
+ options.useBgzfCompression(),
+ options.getFastqCompressionLevel(),
+ options.findAdaptersWithSlidingWindow(),
+ options.getFastqCreatorThreadsCount(),
+ allStats.getConversionStats(laneInfo),
+ allStats.getUnknownBarcodeStat(laneInfo)));
+ case common::NumBasesPerByte::FOUR:
+ return std::unique_ptr<FastqCreator<common::NumBasesPerByte::FOUR>>(
+ new FastqCreator<common::NumBasesPerByte::FOUR>(
+ processTaskQueue,
+ inUseDemuxBufferQueue,
+ recycledBclBufferQueue,
+ inUseFastqBufferMap,
+ recycledFastqBufferQueue,
+ layout,
+ laneInfo,
+ options.getMaskShortAdapterReads(),
+ options.getAdapterStringency(),
+ options.getGenerateReverseComplementFastqs(),
+ options.getIncludeNonPfClusters(),
+ options.getCreateFastqsForIndexReads(),
+ options.useBgzfCompression(),
+ options.getFastqCompressionLevel(),
+ options.findAdaptersWithSlidingWindow(),
+ options.getFastqCreatorThreadsCount(),
+ allStats.getConversionStats(laneInfo),
+ allStats.getUnknownBarcodeStat(laneInfo)));
+ default:
+ BCL2FASTQ_ASSERT_MSG(false, "Unsuported number of bases per byte. Supported values include: One, Two, or Four");
+ };
+
+ return std::unique_ptr<Stage>();
+}
+
+void Converter::run()
+{
+ StageRefsContainer stages;
+
+ stages.push_back(StageRefsContainer::value_type(bclReader_));
+ stages.push_back(StageRefsContainer::value_type(bclLoader_));
+ stages.push_back(StageRefsContainer::value_type(demultiplexer_));
+ stages.push_back(StageRefsContainer::value_type(*fastqCreator_));
+ stages.push_back(StageRefsContainer::value_type(fastqWriter_));
+
+ terminator_.add(&bclReader_);
+ terminator_.add(&bclLoader_);
+ terminator_.add(&demultiplexer_);
+ terminator_.add(fastqCreator_.get());
+ terminator_.add(&fastqWriter_);
+
+ common::ThreadVector stageThreads(stages.size());
+
+ stageThreads.execute(boost::bind(&Converter::stageThreadFunction, this, boost::ref(stages), _1));
+
+ // Done adding tasks.
+ processTaskQueue_.setFinished();
+ ioReadTaskQueue_.setFinished();
+ ioWriteTaskQueue_.setFinished();
+}
+
+Converter::StatsFinalizer::StatsFinalizer(bool isOnly1Sample,
+ const ConversionStatsForSampleTileBarcode& conversionStats,
+ DemultiplexTask::DemuxStats& demuxStats)
+: demuxStats_(demuxStats)
+, conversionStats_(conversionStats)
+, isOnly1Sample_(isOnly1Sample)
+{
+}
+
+Converter::StatsFinalizer::~StatsFinalizer()
+{
+ if (isOnly1Sample_)
+ {
+ // 1 barcodeStats per tile
+ size_t tileNum = 0;
+ for (auto& barcodeStats : demuxStats_.front().front())
+ {
+ barcodeStats.setBarcodeCount(conversionStats_.front().at(tileNum).front().first[stats::TileBarcodeStats::PF].clusterCount);
+ ++tileNum;
+ }
+ }
+}
+
+void Converter::stageThreadFunction(StageRefsContainer &stages, common::ThreadVector::size_type threadNum)
+{
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Running stage #" << threadNum << std::endl;
+ stages.at(threadNum).get().run();
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Stage #" << threadNum << " complete" << std::endl;
+}
+
+
+ConverterStats::ConverterStats( const layout::Layout &layout,
+ const boost::filesystem::path &fastqSummaryPartialFileName,
+ const boost::filesystem::path &demuxSummaryPartialFileName,
+ const boost::filesystem::path &adapterTrimmingTxtFile,
+ const boost::filesystem::path &interopFile,
+ const boost::filesystem::path &demuxStatsXmlFile,
+ const boost::filesystem::path &conversionStatsXmlFile,
+ const boost::filesystem::path &statsJsonFile)
+: layout_(layout)
+, fastqSummaryPartialFileName_(fastqSummaryPartialFileName)
+, demuxSummaryPartialFileName_(demuxSummaryPartialFileName)
+, adapterTrimmingTxtFile_(adapterTrimmingTxtFile)
+, interopFile_(interopFile)
+, demuxStatsXmlFile_(demuxStatsXmlFile)
+, conversionStatsXmlFile_(conversionStatsXmlFile)
+, statsJsonFile_(statsJsonFile)
+, demuxStats_()
+, conversionStats_()
+, topUnknownBarcodes_()
+, barcodeLaneStatsMap_()
+{
+ boost::filesystem::path statsPath0 = interopFile.parent_path();
+ boost::filesystem::path statsPath1 = demuxStatsXmlFile.parent_path();
+ boost::filesystem::path statsPath2 = conversionStatsXmlFile.parent_path();
+ BCL2FASTQ_ASSERT_MSG(statsPath1 == statsPath2, "All human-readable XML statistics need to live in the same directory");
+ std::vector<boost::filesystem::path> createList =
+ boost::assign::list_of(statsPath0)(statsPath1);
+ common::createDirectories(createList);
+}
+
+void ConverterStats::deserializeMetadata( const layout::LaneInfo &laneInfo,
+ size_t sampleIndex,
+ size_t barcodeIndex,
+ std::string &indexName,
+ std::string &sampleId,
+ std::string &projectName,
+ std::string &sampleName,
+ common::SampleNumber &sampleNumber) const
+{
+ const layout::LaneInfo::SampleInfosContainer::value_type &sampleInfo = laneInfo.getSampleInfos().at(sampleIndex);
+ indexName = sampleInfo.hasBarcodes() ? sampleInfo.getBarcode(barcodeIndex).toString() : layout::Barcode::DEFAULT_BARCODE;
+ sampleId = sampleInfo.getId();
+ projectName = sampleInfo.getProject();
+ sampleName = sampleInfo.getName();
+ sampleNumber = sampleInfo.getNumber();
+}
+
+void ConverterStats::dumpInteropStats() const
+{
+ data::InteropFile interopFile;
+ data::InteropFile::InteropFileStatePtr interopFileState(interopFile.open(interopFile_));
+ if (!interopFileState)
+ {
+ return;
+ }
+
+ std::vector<char> buffer; ///< \brief Intermediate buffer for demultiplexing statistics output.
+
+ const std::string flowcellId = layout_.getFlowcellInfo().getFlowcellId();
+
+ static const char version = 1;
+ interopFile.write(*interopFileState, &version, sizeof(version));
+ for (const auto& stats : demuxStats_)
+ {
+ layout::LaneInfo laneInfo = stats.first;
+ data::InteropFile::LaneNumber laneNumber = laneInfo.getNumber();
+ if( stats.second.empty() )
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "No InterOp stats for lane '" << laneNumber << "'" << std::endl;
+ continue;
+ }
+ for (const auto& summaryBarcodeStatsForSample : stats.second)
+ {
+ for (const auto& summaryBarcodeStatsForBarcode : summaryBarcodeStatsForSample)
+ {
+ for (const auto& summaryBarcodeStats : summaryBarcodeStatsForBarcode)
+ {
+ // skip sample #0 (i.e. the default sample)
+ if( 0 == summaryBarcodeStats.index.first.sampleIndex_ )
+ {
+ continue;
+ }
+ std::string indexName,sampleId,projectName,sampleName;
+ common::SampleNumber sampleNumber;
+ deserializeMetadata(laneInfo,
+ summaryBarcodeStats.index.first.sampleIndex_,
+ summaryBarcodeStats.index.first.barcodeIndex_,
+ indexName,
+ sampleId,
+ projectName,
+ sampleName,
+ sampleNumber);
+
+ data::InteropFile::IndexNameLength indexNameLength = indexName.size();
+ data::InteropFile::SampleIdLength sampleIdLength = sampleId.size();
+ data::InteropFile::ProjectNameLength projectNameLength = projectName.size();
+
+ std::replace(indexName.begin(), indexName.end(), '+', '-');
+
+ std::string flowcellId = layout_.getFlowcellInfo().getFlowcellId();
+ data::InteropFile::TileNumber tileNumber = laneInfo.getTileInfos()[summaryBarcodeStats.index.second].getNumber();
+ data::InteropFile::ClustersCount clustersCount = summaryBarcodeStats.getBarcodeCount();
+
+ for (const auto& readInfo : laneInfo.readInfos())
+ {
+ if (!readInfo.isDataRead())
+ {
+ continue;
+ }
+ data::InteropFile::ReadNumber readNumber = readInfo.getNumber();
+
+ buffer.clear();
+ // lane number
+ buffer.insert(buffer.end(), reinterpret_cast<const char *>(&laneNumber), reinterpret_cast<const char *>(&laneNumber)+sizeof(laneNumber));
+ // tile number
+ buffer.insert(buffer.end(), reinterpret_cast<const char *>(&tileNumber), reinterpret_cast<const char *>(&tileNumber)+sizeof(tileNumber));
+ // read number
+ buffer.insert(buffer.end(), reinterpret_cast<const char *>(&readNumber), reinterpret_cast<const char *>(&readNumber)+sizeof(readNumber));
+ // barcode
+ buffer.insert(buffer.end(), reinterpret_cast<const char *>(&indexNameLength), reinterpret_cast<const char *>(&indexNameLength)+sizeof(indexNameLength));
+ buffer.insert(buffer.end(), indexName.begin(), indexName.end());
+ // clusters count
+ buffer.insert(buffer.end(), reinterpret_cast<const char *>(&clustersCount), reinterpret_cast<const char *>(&clustersCount)+sizeof(clustersCount));
+ // sample name
+ buffer.insert(buffer.end(), reinterpret_cast<const char *>(&sampleIdLength), reinterpret_cast<const char *>(&sampleIdLength)+sizeof(sampleIdLength));
+ buffer.insert(buffer.end(), sampleId.begin(), sampleId.end());
+ // project name
+ buffer.insert(buffer.end(), reinterpret_cast<const char *>(&projectNameLength), reinterpret_cast<const char *>(&projectNameLength)+sizeof(projectNameLength));
+ buffer.insert(buffer.end(), projectName.begin(), projectName.end());
+
+ interopFile.write(*interopFileState, &*buffer.begin(), buffer.size());
+ }
+ }
+ }
+ }
+ }
+
+}
+
+void ConverterStats::dumpDemuxStats()
+{
+ stats::DemultiplexingStatsXml demuxStatsXml;
+ const std::string flowcellId = layout_.getFlowcellInfo().getFlowcellId();
+ for (const DemuxPair& stats : demuxStats_)
+ {
+ const layout::LaneInfo& laneInfo = stats.first;
+ unsigned int laneNumber = laneInfo.getNumber();
+ if( stats.second.empty() )
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "No demultiplexing stats for lane '" << laneNumber << "'" << std::endl;
+ continue;
+ }
+
+ std::map<std::string, std::string> uniqueSampleNameMap;
+ stats::DemuxReportGenerator::makeUniqueSampleNameMap(laneInfo, uniqueSampleNameMap);
+
+ FlowcellProjectSampleLaneBarcodeStats flowcellProjectSampleStats;
+ std::vector<stats::BarcodeStats> summaryBarcodeStats(0);
+ for (const std::vector<std::vector<bcl2fastq::stats::BarcodeStats> >& barcodeStatsForSample : stats.second)
+ {
+ for (const std::vector<bcl2fastq::stats::BarcodeStats>& barcodeStatsForBarcode : barcodeStatsForSample)
+ {
+ summaryBarcodeStats.push_back(barcodeStatsForBarcode.front().index);
+
+ for (const stats::BarcodeStats& barcodeStats : barcodeStatsForBarcode)
+ {
+ summaryBarcodeStats.back() += barcodeStats;
+ }
+ }
+ }
+
+ barcodeLaneStatsMap_.insert(std::make_pair(laneNumber, BarcodeStatsMap()));
+
+ std::string indexName,sampleId,projectName,sampleName;
+ common::SampleNumber sampleNumber;
+ BOOST_FOREACH (const stats::BarcodeStats &barcodeStats, std::make_pair(summaryBarcodeStats.begin(), summaryBarcodeStats.end()))
+ {
+ deserializeMetadata(laneInfo,
+ barcodeStats.index.first.sampleIndex_,
+ barcodeStats.index.first.barcodeIndex_,
+ indexName,
+ sampleId,
+ projectName,
+ sampleName,
+ sampleNumber);
+
+ flowcellProjectSampleStats[flowcellId][projectName][sampleId] += barcodeStats;
+ flowcellProjectSampleStats[flowcellId][projectName]["all"] += barcodeStats;
+ flowcellProjectSampleStats[flowcellId]["all"]["all"] += barcodeStats;
+ demuxStatsXml.addLaneBarcode(flowcellId, projectName, uniqueSampleNameMap[sampleId], indexName, laneNumber, barcodeStats);
+ barcodeLaneStatsMap_[laneNumber].insert(std::make_pair(indexName, barcodeStats));
+ }
+ BOOST_FOREACH(const FlowcellProjectSampleLaneBarcodeStats::value_type &flowcellStats, flowcellProjectSampleStats)
+ {
+ BOOST_FOREACH(const ProjectSampleLaneBarcodeStats::value_type &projectStats, flowcellStats.second)
+ {
+ BOOST_FOREACH(const SampleLaneBarcodeStats::value_type &sampleStats, projectStats.second)
+ {
+ demuxStatsXml.addLaneBarcode(flowcellStats.first, projectStats.first, sampleStats.first == "all" ? "all" : uniqueSampleNameMap[sampleStats.first], "all", laneNumber, sampleStats.second);
+ }
+ }
+ }
+ }
+
+ std::ofstream os(demuxStatsXmlFile_.string().c_str());
+ if (!os) {
+ BOOST_THROW_EXCEPTION(common::IoError(errno, "Unable to open file for writing: " + demuxStatsXmlFile_.string()));
+ }
+ if (!(os << demuxStatsXml)) {
+ BOOST_THROW_EXCEPTION(common::IoError(errno, "Failed to store demultiplexing statistics in : " + demuxStatsXmlFile_.string()));
+ }
+}
+
+void ConverterStats::dumpConversionStats()
+{
+ stats::JsonObject jsonStats;
+
+ stats::DemultiplexingStatsXml demuxStatsXml;
+ const std::string flowcellId = layout_.getFlowcellInfo().getFlowcellId();
+
+ jsonStats.add("Flowcell", std::shared_ptr<stats::JsonType>(new stats::JsonString(flowcellId)));
+
+ try
+ {
+ int64_t runNumber = boost::lexical_cast<int64_t>(layout_.getFlowcellInfo().getRunNumber());
+ jsonStats.add("RunNumber", std::shared_ptr<stats::JsonType>(new stats::JsonNumber(runNumber)));
+ }
+ catch (boost::bad_lexical_cast&)
+ {
+ // Unable to get the RunNumber from RunInfo.xml
+ jsonStats.add("RunNumber", std::shared_ptr<stats::JsonType>(new stats::JsonNumber()));
+ }
+
+ jsonStats.add("RunId", std::shared_ptr<stats::JsonType>(new stats::JsonString(layout_.getFlowcellInfo().getRunId())));
+
+ stats::JsonArray* jsonReadInfosForLanes = NULL;
+ jsonStats.add("ReadInfosForLanes", std::shared_ptr<stats::JsonType>(jsonReadInfosForLanes = new stats::JsonArray));
+
+ stats::JsonArray* jsonConversionResults = NULL;
+ jsonStats.add("ConversionResults", std::shared_ptr<stats::JsonType>(jsonConversionResults = new stats::JsonArray));
+
+ boost::filesystem::ofstream adapterTrimmingOstr(adapterTrimmingTxtFile_);
+ adapterTrimmingOstr << "Lane\tRead\tProject\tSample Id\tSample Name\tSample Number\tTrimmedBases\tPercentageOfBases" << std::endl;
+
+ std::ostringstream trimLengthOstr;
+ stats::ConversionStatsXml conversionStatsXml;
+ for (const auto &stats : conversionStats_)
+ {
+ const layout::LaneInfo& laneInfo = stats.first;
+
+ std::map<std::string, std::string> uniqueSampleNameMap;
+ stats::DemuxReportGenerator::makeUniqueSampleNameMap(laneInfo, uniqueSampleNameMap);
+
+ const ConversionStatsForSampleTileBarcode &conversionStats = stats.second;
+ unsigned int laneNumber = laneInfo.getNumber();
+
+ if( conversionStats.empty() )
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "No conversion stats for lane '" << laneNumber << "'" << std::endl;
+ }
+
+ stats::JsonObject* jsonReadInfosForLane = NULL;
+ jsonReadInfosForLanes->add(std::shared_ptr<stats::JsonType>(jsonReadInfosForLane = new stats::JsonObject));
+ jsonReadInfosForLane->add("LaneNumber", std::shared_ptr<stats::JsonType>(new stats::JsonNumber(laneNumber)));
+ stats::JsonArray* jsonReadInfos = NULL;
+ jsonReadInfosForLane->add("ReadInfos", std::shared_ptr<stats::JsonType>(jsonReadInfos = new stats::JsonArray));
+
+ stats::JsonObject* jsonConversionResultsForLane = NULL;
+ jsonConversionResults->add(std::shared_ptr<stats::JsonType>(jsonConversionResultsForLane = new stats::JsonObject));
+ jsonConversionResultsForLane->add("LaneNumber", std::shared_ptr<stats::JsonType>(new stats::JsonNumber(laneNumber)));
+
+ stats::JsonNumber* totalClustersRaw = NULL;
+ stats::JsonNumber* totalClustersPF = NULL;
+ stats::JsonNumber* totalYield = NULL;
+ jsonConversionResultsForLane->add("TotalClustersRaw", std::shared_ptr<stats::JsonType>(totalClustersRaw = new stats::JsonNumber(0)));
+ jsonConversionResultsForLane->add("TotalClustersPF", std::shared_ptr<stats::JsonType>(totalClustersPF = new stats::JsonNumber(0)));
+ jsonConversionResultsForLane->add("Yield", std::shared_ptr<stats::JsonType>(totalYield = new stats::JsonNumber(0)));
+
+ for (const auto& readInfo : laneInfo.getReadInfos())
+ {
+ stats::JsonObject* jsonReadInfo = NULL;
+ jsonReadInfos->add(std::shared_ptr<stats::JsonType>(jsonReadInfo = new stats::JsonObject));
+ jsonReadInfo->add("Number", std::shared_ptr<stats::JsonType>(new stats::JsonNumber(readInfo.getNumber())));
+ jsonReadInfo->add("NumCycles", std::shared_ptr<stats::JsonType>(new stats::JsonNumber(readInfo.cycleInfos().size())));
+ jsonReadInfo->add("IsIndexedRead", std::shared_ptr<stats::JsonType>(new stats::JsonBool(readInfo.isIndexRead())));
+ }
+
+ FlowcellProjectSampleBarcodeTileReadBarcodeStats flowcellProjectSampleBarcodeStats;
+ ProjectSampleReadBarcodeStats projectSampleBarcodeStats;
+
+ std::string indexName,sampleId,projectName,sampleName;
+ common::SampleNumber sampleNumber;
+
+ size_t sampleIndex = 0;
+ for (const auto& sampleStats : conversionStats)
+ {
+ size_t tileIndex = 0;
+ for (const auto& tileStats : sampleStats)
+ {
+ size_t barcodeIndex = 0;
+ for (const auto& barcodeStats : tileStats)
+ {
+ deserializeMetadata(laneInfo,
+ sampleIndex,
+ barcodeIndex,
+ indexName,
+ sampleId,
+ projectName,
+ sampleName,
+ sampleNumber);
+
+ layout::LaneInfo::TileInfosContainer::value_type tileInfo = laneInfo.getTileInfos()[tileIndex];
+ common::TileNumber tileNumber = tileInfo.getNumber();
+
+ projectSampleBarcodeStats[projectName][sampleId].first = sampleIndex;
+ projectSampleBarcodeStats[projectName][sampleId].second += barcodeStats.first;
+
+ flowcellProjectSampleBarcodeStats[flowcellId][projectName][sampleId][indexName][tileNumber].first += barcodeStats.first;
+ flowcellProjectSampleBarcodeStats[flowcellId][projectName][sampleId]["all"][tileNumber].first += barcodeStats.first;
+ flowcellProjectSampleBarcodeStats[flowcellId][projectName]["all"]["all"][tileNumber].first += barcodeStats.first;
+ flowcellProjectSampleBarcodeStats[flowcellId]["all"]["all"]["all"][tileNumber].first += barcodeStats.first;
+
+ for (const auto& readStats : barcodeStats.second)
+ {
+ common::ReadNumber readNumber = readStats.read-1;
+
+ flowcellProjectSampleBarcodeStats[flowcellId][projectName][sampleId][indexName][tileNumber].second[readNumber] += readStats;
+ flowcellProjectSampleBarcodeStats[flowcellId][projectName][sampleId]["all"][tileNumber].second[readNumber] += readStats;
+ flowcellProjectSampleBarcodeStats[flowcellId][projectName]["all"]["all"][tileNumber].second[readNumber] += readStats;
+ flowcellProjectSampleBarcodeStats[flowcellId]["all"]["all"]["all"][tileNumber].second[readNumber] += readStats;
+ }
+
+ ++barcodeIndex;
+ }
+
+ ++tileIndex;
+ }
+
+ ++sampleIndex;
+ }
+
+ std::map<std::pair<std::string, std::string>, std::vector<size_t>> trimmedCountMap;
+ writeAdapterStats(laneInfo,
+ projectSampleBarcodeStats,
+ adapterTrimmingOstr,
+ trimLengthOstr,
+ trimmedCountMap);
+
+ writeSummaryStats(flowcellProjectSampleBarcodeStats[flowcellId],
+ laneInfo);
+
+ stats::JsonArray* jsonDemuxResults = NULL;
+ jsonConversionResultsForLane->add("DemuxResults", std::shared_ptr<stats::JsonArray>(jsonDemuxResults = new stats::JsonArray()));
+
+ BOOST_FOREACH( const FlowcellProjectSampleBarcodeTileReadBarcodeStats::value_type &flowcellStats, flowcellProjectSampleBarcodeStats)
+ {
+ BOOST_FOREACH( const ProjectSampleBarcodeTileReadBarcodeStats::value_type &projectStats, flowcellStats.second)
+ {
+ BOOST_FOREACH( const SampleBarcodeTileReadBarcodeStats::value_type &sampleStats, projectStats.second)
+ {
+ stats::JsonObject* jsonDemuxSample = NULL;
+
+ stats::JsonNumber* numReads = NULL;
+ stats::JsonNumber* yield = NULL;
+ int64_t totalYieldValue = 0;
+
+ std::map<common::ReadNumber, stats::JsonNumber*> readYieldMap;
+ std::map<common::ReadNumber, stats::JsonNumber*> readYieldQ30Map;
+ std::map<common::ReadNumber, stats::JsonNumber*> readQscoreSumMap;
+
+ stats::JsonArray* jsonReadMetrics = NULL;
+ stats::JsonArray* jsonIndexMetrics = NULL;
+ std::string sampleName;
+ if (sampleStats.first != "all" && projectStats.first != "all")
+ {
+ common::SampleNumber sampleNumber;
+ getSampleNameAndNumber(laneInfo,
+ projectStats.first,
+ sampleStats.first,
+ sampleName,
+ sampleNumber);
+
+ if (sampleNumber == 0)
+ {
+ jsonConversionResultsForLane->add("Undetermined", std::shared_ptr<stats::JsonType>(jsonDemuxSample = new stats::JsonObject));
+ }
+ else
+ {
+ jsonDemuxResults->add(std::shared_ptr<stats::JsonType>(jsonDemuxSample = new stats::JsonObject()));
+
+ jsonDemuxSample->add("SampleId", std::shared_ptr<stats::JsonType>(new stats::JsonString(sampleStats.first)));
+ jsonDemuxSample->add("SampleName", std::shared_ptr<stats::JsonType>(new stats::JsonString(sampleName)));
+ }
+
+ if (sampleStats.second.size() > 1 &&
+ !(sampleStats.second.begin()->first == layout::Barcode::DEFAULT_BARCODE || (++sampleStats.second.begin())->first == layout::Barcode::DEFAULT_BARCODE))
+
+ {
+ jsonDemuxSample->add("IndexMetrics", std::shared_ptr<stats::JsonType>(jsonIndexMetrics = new stats::JsonArray));
+ }
+
+ jsonDemuxSample->add("NumberReads", std::shared_ptr<stats::JsonType>(numReads = new stats::JsonNumber(0)));
+ jsonDemuxSample->add("Yield", std::shared_ptr<stats::JsonType>(yield = new stats::JsonNumber(0)));
+
+ jsonDemuxSample->add("ReadMetrics", std::shared_ptr<stats::JsonType>(jsonReadMetrics = new stats::JsonArray));
+ }
+
+ BOOST_FOREACH( const BarcodeTileReadBarcodeStats::value_type &barcodeStats, sampleStats.second)
+ {
+ if (sampleStats.first != "all" &&
+ projectStats.first != "all" && barcodeStats.first != "all" &&
+ jsonIndexMetrics != NULL )
+ {
+ recordBarcodeStats(barcodeLaneStatsMap_[laneInfo.getNumber()],
+ barcodeStats.first,
+ *jsonIndexMetrics);
+ }
+
+
+ BOOST_FOREACH( const TileReadBarcodeStats::value_type &tileStats, barcodeStats.second)
+ {
+ if (barcodeStats.first == "all")
+ {
+ if (sampleStats.first != "all" && projectStats.first != "all")
+ {
+ totalClustersRaw->add(tileStats.second.first[stats::TileBarcodeStats::RAW].clusterCount);
+ totalClustersPF->add(tileStats.second.first[stats::TileBarcodeStats::PF].clusterCount);
+
+ numReads->add(tileStats.second.first[stats::TileBarcodeStats::PF].clusterCount);
+ }
+
+ }
+
+ conversionStatsXml.addLaneTile( flowcellStats.first,
+ projectStats.first,
+ sampleStats.first == "all" ? "all" : uniqueSampleNameMap[sampleStats.first],
+ barcodeStats.first,
+ tileStats.first,
+ laneNumber,
+ tileStats.second.first );
+
+ recordReadStats(conversionStatsXml,
+ tileStats.second.second,
+ laneNumber,
+ tileStats.first,
+ barcodeStats.first,
+ projectStats.first,
+ sampleStats.first,
+ sampleName,
+ uniqueSampleNameMap,
+ readYieldMap,
+ readYieldQ30Map,
+ readQscoreSumMap,
+ trimmedCountMap,
+ yield,
+ jsonReadMetrics);
+
+ if (sampleStats.first != "all" && projectStats.first != "all")
+ {
+ totalYieldValue = yield->getValue();
+ }
+ }
+ }
+ totalYield->add(totalYieldValue);
+ }
+ }
+ }
+ }
+
+ recordUnknownBarcodes(jsonStats,
+ conversionStatsXml);
+
+ std::ofstream os(conversionStatsXmlFile_.string().c_str());
+ if (!os) {
+ BOOST_THROW_EXCEPTION(common::IoError(errno, "Unable to open file for writing: " + conversionStatsXmlFile_.string()));
+ }
+ if (!(os << conversionStatsXml)) {
+ BOOST_THROW_EXCEPTION(common::IoError(errno, "Failed to store conversion statistics in : " + conversionStatsXmlFile_.string()));
+ }
+
+ adapterTrimmingOstr << trimLengthOstr.str();
+
+ std::ofstream jsonStatsStr(statsJsonFile_.string().c_str());
+ jsonStats.stream(jsonStatsStr);
+}
+
+void ConverterStats::recordBarcodeStats(const BarcodeStatsMap& barcodeStatsMap,
+ const std::string& barcodeName,
+ stats::JsonArray& jsonIndexMetrics)
+{
+ std::vector<stats::JsonNumber*> mismatchCountVec;
+ stats::JsonObject* mismatchCounts = NULL;
+
+ stats::JsonObject* indexMetric = NULL;
+ jsonIndexMetrics.add(std::shared_ptr<stats::JsonType>(indexMetric = new stats::JsonObject));
+ indexMetric->add("IndexSequence", std::shared_ptr<stats::JsonType>(new stats::JsonString(barcodeName)));
+ indexMetric->add("MismatchCounts", std::shared_ptr<stats::JsonType>(mismatchCounts = new stats::JsonObject));
+
+ size_t mismatches = 0;
+ for (const common::ClustersCount count : barcodeStatsMap.at(barcodeName).getBarcodeMismatchCounts())
+ {
+ if (count == 0 && mismatches > 1)
+ {
+ break;
+ }
+
+ if (mismatchCountVec.size() <= mismatches)
+ {
+ stats::JsonNumber* mismatchCount = NULL;
+ mismatchCounts->add(boost::lexical_cast<std::string>(mismatches), std::shared_ptr<stats::JsonType>(mismatchCount = new stats::JsonNumber(count)));
+ mismatchCountVec.push_back(mismatchCount);
+ }
+ else
+ {
+ mismatchCountVec[mismatches]->add(count);
+ }
+
+ ++mismatches;
+ }
+}
+
+void ConverterStats::recordReadStats(stats::ConversionStatsXml& conversionStatsXml,
+ const AnnotatedTileReadBarcodeStats& tileStats,
+ unsigned int laneNumber,
+ common::TileNumber tileNumber,
+ const std::string& barcodeName,
+ const std::string& project,
+ const std::string& sampleId,
+ const std::string& sampleName,
+ std::map<std::string, std::string>& uniqueSampleNameMap,
+ std::map<common::ReadNumber, stats::JsonNumber*>& readYieldMap,
+ std::map<common::ReadNumber, stats::JsonNumber*>& readYieldQ30Map,
+ std::map<common::ReadNumber, stats::JsonNumber*>& readQscoreSumMap,
+ std::map<std::pair<std::string, std::string>, std::vector<size_t>>& trimmedCountMap,
+ stats::JsonNumber* yield,
+ stats::JsonArray* jsonReadMetrics)
+{
+ for (const auto& readStats : tileStats)
+ {
+ if (barcodeName == "all" &&
+ sampleId != "all" &&
+ project != "all")
+ {
+ yield->add(readStats.second[stats::TileBarcodeStats::PF].yield);
+
+ stats::JsonNumber*& readYield = readYieldMap[readStats.first];
+ stats::JsonNumber*& readYieldQ30 = readYieldQ30Map[readStats.first];
+ stats::JsonNumber*& readQscoreSum = readQscoreSumMap[readStats.first];
+ if (!readYield)
+ {
+ stats::JsonObject* readMetric = NULL;
+ jsonReadMetrics->add(std::shared_ptr<stats::JsonType>(readMetric = new stats::JsonObject));
+ readMetric->add("ReadNumber", std::shared_ptr<stats::JsonType>(new stats::JsonNumber(readStats.first+1)));
+ readMetric->add("Yield", std::shared_ptr<stats::JsonType>(readYield = new stats::JsonNumber(0)));
+ readMetric->add("YieldQ30", std::shared_ptr<stats::JsonType>(readYieldQ30 = new stats::JsonNumber(0)));
+ readMetric->add("QualityScoreSum", std::shared_ptr<stats::JsonType>(readQscoreSum = new stats::JsonNumber(0)));
+ readMetric->add("TrimmedBases", std::shared_ptr<stats::JsonType>(new stats::JsonNumber(trimmedCountMap[std::make_pair(sampleId, sampleName)][readStats.first])));
+ }
+
+ readYield->add(readStats.second[stats::TileBarcodeStats::PF].yield);
+ readYieldQ30->add(readStats.second[stats::TileBarcodeStats::PF].yieldQ30);
+ readQscoreSum->add(readStats.second[stats::TileBarcodeStats::PF].qualityScoreSum);
+
+ }
+
+ conversionStatsXml.addLaneTileRead( layout_.getFlowcellInfo().getFlowcellId(),
+ project,
+ sampleId == "all" ? "all" : uniqueSampleNameMap[sampleId],
+ barcodeName,
+ tileNumber,
+ readStats.first + 1,
+ laneNumber,
+ readStats.second );
+ }
+}
+
+void ConverterStats::recordUnknownBarcodes(stats::JsonObject& jsonStats,
+ stats::ConversionStatsXml& conversionStatsXml) const
+{
+ stats::JsonArray* jsonUnknownBarcodes = NULL;
+ jsonStats.add("UnknownBarcodes", std::shared_ptr<stats::JsonType>(jsonUnknownBarcodes = new stats::JsonArray));
+
+ BOOST_FOREACH (const UnknownPair &stats, std::make_pair(topUnknownBarcodes_.begin(), topUnknownBarcodes_.end()))
+ {
+ stats::JsonObject* jsonUnknownBarcodesForLane = NULL;
+ jsonUnknownBarcodes->add(std::shared_ptr<stats::JsonType>(jsonUnknownBarcodesForLane = new stats::JsonObject));
+
+ layout::LaneInfo laneInfo = stats.first;
+ unsigned int laneNumber = laneInfo.getNumber();
+ const stats::BarcodeHits::Popular& popularBarcodes = stats.second.getPopularBarcodes();
+
+ jsonUnknownBarcodesForLane->add("Lane", std::shared_ptr<stats::JsonType>(new stats::JsonNumber(laneNumber)));
+
+ stats::JsonObject* jsonBarcodes = NULL;
+ jsonUnknownBarcodesForLane->add("Barcodes", std::shared_ptr<stats::JsonType>(jsonBarcodes = new stats::JsonObject));
+ for (const auto& barcodeHit : popularBarcodes)
+ {
+ jsonBarcodes->add(barcodeHit.first.toString(), std::shared_ptr<stats::JsonType>(new stats::JsonNumber(barcodeHit.second)));
+ }
+
+ conversionStatsXml.addFlowcellLane( layout_.getFlowcellInfo().getFlowcellId(),
+ laneNumber,
+ popularBarcodes.size() <= 10 ?
+ popularBarcodes :
+ stats::BarcodeHits::Popular(popularBarcodes.begin(), popularBarcodes.begin()+10) );
+ }
+}
+
+namespace detail
+{
+
+FastqSummaryStats::FastqSummaryStats(common::SampleNumber sampleNumber,
+ const std::string& sampleName,
+ common::TileNumber tileNumber,
+ common::ClustersCount numberReadsRaw,
+ common::ClustersCount numberReadsPF)
+: sampleNumber_(sampleNumber)
+, sampleName_(sampleName == "Undetermined" ? "None" : sampleName)
+, tileNumber_(tileNumber)
+, numberReadsRaw_(numberReadsRaw)
+, numberReadsPF_(numberReadsPF)
+{
+}
+
+bool FastqSummaryStats::operator<(const FastqSummaryStats& other) const
+{
+ if (tileNumber_ < other.tileNumber_) return true;
+ if (tileNumber_ > other.tileNumber_) return false;
+ return sampleNumber_ < other.sampleNumber_;
+}
+
+std::ostream& operator<<(std::ostream& os, const FastqSummaryStats& stats)
+{
+ os << stats.sampleNumber_ << "\t"
+ << stats.tileNumber_ << "\t"
+ << stats.numberReadsRaw_ << "\t"
+ << stats.numberReadsPF_;
+
+ return os;
+}
+
+}
+
+void ConverterStats::writeAdapterStats(const layout::LaneInfo& laneInfo,
+ const ProjectSampleReadBarcodeStats& stats,
+ std::ostream& adapterOstr,
+ std::ostream& trimLengthOstr,
+ std::map<std::pair<std::string, std::string>, std::vector<size_t>>& trimmedBaseMap) const
+{
+ BOOST_FOREACH(const ProjectSampleReadBarcodeStats::value_type& sampleBarcodeTileStats, stats)
+ {
+ BOOST_FOREACH(const SampleReadBarcodeStats::value_type& barcodeTileStats, sampleBarcodeTileStats.second)
+ {
+ const layout::SampleInfo& sampleInfo = laneInfo.getSampleInfos().at(barcodeTileStats.second.first);
+
+ writeAdapterStats(laneInfo.getNumber(),
+ sampleBarcodeTileStats.first,
+ barcodeTileStats.first,
+ sampleInfo.getName(),
+ sampleInfo.getNumber(),
+ barcodeTileStats.second.second[stats::TileBarcodeStats::PF].trimLengthCountForReads,
+ adapterOstr,
+ trimLengthOstr,
+ trimmedBaseMap);
+ }
+ }
+}
+
+void ConverterStats::writeSummaryStats(const ProjectSampleBarcodeTileReadBarcodeStats& stats,
+ const layout::LaneInfo& laneInfo) const
+{
+ detail::FastqSummaryStatsContainer fastqSummaryStats;
+
+ for (const ProjectSampleBarcodeTileReadBarcodeStats::value_type& sampleBarcodeTileStats : stats)
+ {
+ if (sampleBarcodeTileStats.first == "all") continue;
+
+ for (const SampleBarcodeTileReadBarcodeStats::value_type& barcodeTileStats : sampleBarcodeTileStats.second)
+ {
+ if (barcodeTileStats.first == "all") continue;
+
+ BarcodeTileReadBarcodeStats::const_iterator pos = barcodeTileStats.second.find("all");
+ BOOST_ASSERT(pos != barcodeTileStats.second.end());
+
+ for (const TileReadBarcodeStats::value_type& tileStats : pos->second)
+ {
+ std::string sampleName;
+ common::SampleNumber sampleNumber;
+ getSampleNameAndNumber(laneInfo,
+ sampleBarcodeTileStats.first,
+ barcodeTileStats.first,
+ sampleName,
+ sampleNumber);
+
+ fastqSummaryStats.push_back(detail::FastqSummaryStats(
+ sampleNumber,
+ sampleName,
+ tileStats.first, // tileNumber
+ tileStats.second.first[stats::TileBarcodeStats::RAW].clusterCount,
+ tileStats.second.first[stats::TileBarcodeStats::PF].clusterCount));
+ }
+ }
+ }
+
+ std::sort(fastqSummaryStats.begin(),
+ fastqSummaryStats.end());
+
+ writeFastqSummaryStats(laneInfo.getNumber(), fastqSummaryStats);
+ writeDemuxSummaryStats(laneInfo.getNumber(), fastqSummaryStats);
+}
+
+void ConverterStats::writeFastqSummaryStats(common::LaneNumber laneNumber,
+ const detail::FastqSummaryStatsContainer& fastqSummaryStats) const
+{
+ boost::filesystem::path fastqSummaryFileName(fastqSummaryPartialFileName_.string() + boost::lexical_cast<std::string>(laneNumber) + ".txt");
+ boost::filesystem::ofstream fastqSummaryOstr(fastqSummaryFileName);
+ fastqSummaryOstr << "SampleNumber\tTile\tNumberOfReadsRaw\tNumberOfReadsPF" << std::endl;
+
+ BOOST_FOREACH(const detail::FastqSummaryStats& summaryStats, fastqSummaryStats)
+ {
+ fastqSummaryOstr << summaryStats << std::endl;
+ }
+}
+
+void ConverterStats::writeDemuxSummaryStats(common::LaneNumber laneNumber,
+ const detail::FastqSummaryStatsContainer& fastqSummaryStats) const
+{
+ if (fastqSummaryStats.empty())
+ {
+ return;
+ }
+
+ boost::filesystem::path demuxSummaryFileName(demuxSummaryPartialFileName_.string() + boost::lexical_cast<std::string>(laneNumber) + ".txt");
+ boost::filesystem::ofstream demuxSummaryOstr(demuxSummaryFileName);
+
+ std::stringstream sampleNamesStr;
+ sampleNamesStr << "SampleName";
+ demuxSummaryOstr << "SampleNumber";
+
+ size_t totalReads = 0;
+ bool firstLoop = true;
+ std::vector<common::ClustersCount> rawReadsPerTile;
+ BOOST_FOREACH(const detail::FastqSummaryStats& stats, fastqSummaryStats)
+ {
+ if (stats.sampleNumber_ == fastqSummaryStats.begin()->sampleNumber_)
+ {
+ if (!firstLoop)
+ {
+ rawReadsPerTile.push_back(totalReads);
+ }
+ else
+ {
+ firstLoop = false;
+ }
+ totalReads = 0;
+ }
+
+ if (rawReadsPerTile.empty())
+ {
+ demuxSummaryOstr << "\t" << stats.sampleNumber_;
+ sampleNamesStr << "\t" << stats.sampleName_;
+ }
+ totalReads += stats.numberReadsPF_;
+ }
+ rawReadsPerTile.push_back(totalReads);
+ demuxSummaryOstr << std::endl << sampleNamesStr.str();
+
+ int tileIndex = -1;
+ BOOST_FOREACH(const detail::FastqSummaryStats& stats, fastqSummaryStats)
+ {
+ if (stats.sampleNumber_ == fastqSummaryStats.begin()->sampleNumber_)
+ {
+ ++tileIndex;
+ demuxSummaryOstr << std::endl << "L" << laneNumber << "T" << stats.tileNumber_;
+ }
+
+ demuxSummaryOstr
+ << "\t" << std::setprecision(7)
+ << (rawReadsPerTile.at(tileIndex) == 0 ? 0 : 100.0 * stats.numberReadsPF_ / rawReadsPerTile.at(tileIndex));
+ }
+
+ demuxSummaryOstr
+ << std::endl << std::endl
+ << "### Most Popular Unknown Index Sequences" << std::endl
+ << "### Columns: Index_Sequence Hit_Count" << std::endl;
+
+ BOOST_FOREACH (const UnknownPair &stats, std::make_pair(topUnknownBarcodes_.begin(), topUnknownBarcodes_.end()))
+ {
+ layout::LaneInfo laneInfo = stats.first;
+ if (laneNumber == laneInfo.getNumber())
+ {
+ BOOST_FOREACH(const stats::BarcodeHits::Popular::value_type& barcodeHits, stats.second.getPopularBarcodes())
+ {
+ demuxSummaryOstr << barcodeHits.first << "\t" << barcodeHits.second << std::endl;
+ }
+ break;
+ }
+ }
+}
+
+// This function is inefficient, but it's not important for the overall runtime.
+void ConverterStats::getSampleNameAndNumber(const layout::LaneInfo& laneInfo,
+ const std::string& project,
+ const std::string& sampleId,
+ std::string& sampleName,
+ common::SampleNumber& sampleNumber) const
+{
+ BOOST_FOREACH(const layout::SampleInfo& sampleInfo, std::make_pair(laneInfo.sampleInfosBegin(), laneInfo.sampleInfosEnd()))
+ {
+ if (project == sampleInfo.getProject() &&
+ sampleId == sampleInfo.getId())
+ {
+ sampleName = sampleInfo.getName();
+ sampleNumber = sampleInfo.getNumber();
+ return;
+ }
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::WARNING)
+ << (boost::format("Sample number was not found for project: '%s' and sample id: '%s'") % project % sampleId).str()
+ << std::endl;
+}
+
+void ConverterStats::writeAdapterStats(common::LaneNumber laneNumber,
+ const std::string& project,
+ const std::string& sampleId,
+ const std::string& sampleName,
+ common::SampleNumber sampleNumber,
+ const stats::AllReadsStats::TrimLengthCountForReads& trimLengthCountForReads,
+ std::ostream& trimBasesOstr,
+ std::ostream& trimLengthOstr,
+ std::map<std::pair<std::string, std::string>, std::vector<size_t>>& trimmedBaseMap) const
+{
+ common::ReadNumber readNumber = 0;
+ BOOST_FOREACH(const stats::AllReadsStats::TrimLengthCount& trimLengthCount, trimLengthCountForReads)
+ {
+ size_t totalTrimBasesForRead = 0;
+ size_t trimLength = 0;
+ size_t basesPerRead = trimLengthCount.size() - 1;
+ size_t numReads = 0;
+
+ BOOST_FOREACH(size_t count, trimLengthCount)
+ {
+ numReads += count;
+ totalTrimBasesForRead += trimLength++ * count;
+ }
+
+ trimmedBaseMap[std::make_pair(sampleId, sampleName)].push_back(totalTrimBasesForRead);
+
+ trimLengthOstr
+ << std::endl
+ << "Lane: " << laneNumber
+ << "\tRead: " << readNumber+1
+ << "\tProject: " << project
+ << "\tSample Id: " << sampleId
+ << "\tSample Name: " << sampleName
+ << "\tSample Number: " << sampleNumber
+ << std::endl;
+
+ trimLength = 0;
+ for (size_t count : boost::adaptors::reverse(trimLengthCount))
+ {
+ trimLengthOstr << trimLength++ << "\t" << static_cast<double>(count) / (numReads == 0 ? 1 : numReads) << std::endl;
+ }
+
+ trimBasesOstr
+ << laneNumber << "\t"
+ << ++readNumber << "\t"
+ << project << "\t"
+ << sampleId << "\t"
+ << sampleName << "\t"
+ << sampleNumber << "\t"
+ << totalTrimBasesForRead << "\t"
+ << (100.0*totalTrimBasesForRead) / (numReads == 0 ? 1 : numReads*basesPerRead)
+ << std::endl;
+ }
+}
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/conversion/Demultiplexer.cpp b/src/cxx/lib/conversion/Demultiplexer.cpp
new file mode 100644
index 0000000..657efa0
--- /dev/null
+++ b/src/cxx/lib/conversion/Demultiplexer.cpp
@@ -0,0 +1,381 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Demultiplexer.cpp
+ *
+ * \brief Implementation of demultiplexer.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#include <algorithm>
+#include <numeric>
+#include <utility>
+
+#include <boost/foreach.hpp>
+#include <boost/assign/list_of.hpp>
+
+
+#include "common/FileSystem.hh"
+#include "layout/Barcode.hh"
+#include "conversion/BclBaseConversion.hh"
+#include "conversion/Converter.hh"
+#include "conversion/Demultiplexer.hh"
+
+
+namespace bcl2fastq
+{
+namespace conversion
+{
+
+DemultiplexTask::DemuxStatsSubset::DemuxStatsSubset(DemultiplexTask::DemuxStats& summaryDemuxStats)
+: summaryDemuxStats_(summaryDemuxStats)
+, stats_(summaryDemuxStats.size())
+{
+ size_t numSamples = summaryDemuxStats.size();
+ for (size_t sampleIdx = 0; sampleIdx < numSamples; ++sampleIdx)
+ {
+ size_t numBarcodes = summaryDemuxStats[sampleIdx].size();
+ stats_[sampleIdx].resize(numBarcodes);
+ for (size_t barcodeIdx = 0; barcodeIdx < numBarcodes; ++barcodeIdx)
+ {
+ stats_[sampleIdx][barcodeIdx].resize(summaryDemuxStats[sampleIdx][barcodeIdx].size());
+ }
+ }
+}
+
+DemultiplexTask::DemuxStatsSubset::~DemuxStatsSubset()
+{
+ // Note: It is important that these are all created/destroyed on the same thread
+
+ size_t numSamples = stats_.size();
+ for (size_t sampleIdx = 0; sampleIdx < numSamples; ++sampleIdx)
+ {
+ size_t numBarcodes = stats_[sampleIdx].size();
+ for (size_t barcodeIdx = 0; barcodeIdx < numBarcodes; ++barcodeIdx)
+ {
+ size_t numTiles = stats_[sampleIdx][barcodeIdx].size();
+ for (size_t tileIdx = 0; tileIdx < numTiles; ++tileIdx)
+ {
+ summaryDemuxStats_[sampleIdx][barcodeIdx][tileIdx] += stats_[sampleIdx][barcodeIdx][tileIdx];
+ }
+ }
+ }
+}
+
+std::atomic<uint32_t> DemuxTaskManager::numTaskManagers_(0);
+std::condition_variable DemuxTaskManager::cvAllTaskManagersDone_;
+std::mutex DemuxTaskManager::mut_;
+
+DemuxTaskManager::DemuxTaskManager(std::shared_ptr<DemuxBuffer>& buffer,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBufferQueue)
+: TaskManager()
+, buffer_(buffer)
+, outputBufferQueue_(outputBufferQueue)
+{
+ ++numTaskManagers_;
+}
+
+DemuxTaskManager::~DemuxTaskManager()
+{
+ static int sequentialId = 0; ++sequentialId;
+
+ buffer_->calculateIndex();
+
+ // TODO: This can block. Do this on a separate thread. Be careful about ensuring the thread completes.
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << "Group: " << buffer_->getGroupNumber() << " Push FASTQ buffer for Create" << std::endl;
+ outputBufferQueue_.addData(buffer_);
+ --numTaskManagers_;
+
+ if (numTaskManagers_ == 0)
+ {
+ // This could be called more than once on different threads. I think
+ // this is ok.
+ cvAllTaskManagersDone_.notify_all();
+ }
+
+}
+
+void DemuxTaskManager::waitForAllTasksToFinish()
+{
+ std::unique_lock<std::mutex> lock(mut_);
+ cvAllTaskManagersDone_.wait(lock, [] { return DemuxTaskManager::numTaskManagers_ == 0; });
+}
+
+
+DemultiplexTask::DemultiplexTask(
+ std::shared_ptr<DemuxTaskManager>& taskManager,
+ const data::BclBuffer &inputBuffer,
+ const layout::ReadInfosContainer& readInfos,
+ data::BclBuffer::BclCycleContainer::size_type offsetBegin,
+ data::BclBuffer::BclCycleContainer::size_type offsetEnd,
+ const layout::BarcodeTranslationTable &barcodeTranslationTable,
+ layout::LaneInfo::TileInfosContainer::const_iterator tileInfosBegin,
+ size_t numIndexReads,
+ bool includeNonPfClusters,
+ common::NumBasesPerByte numBasesPerByte,
+ std::vector<std::shared_ptr<DemultiplexTask::DemuxStatsSubset>>& demuxStats,
+ data::BclBuffer::SamplesContainer &outputBuffer
+)
+: Task(taskManager)
+, inputBuffer_(inputBuffer)
+, readInfos_(readInfos)
+, offsetBegin_(offsetBegin)
+, offsetEnd_(offsetEnd)
+, barcodeTranslationTable_(barcodeTranslationTable)
+, tileInfosBegin_(tileInfosBegin)
+, numIndexReads_(numIndexReads)
+, includeNonPfClusters_(includeNonPfClusters)
+, numBasesPerByte_(numBasesPerByte)
+, demuxStats_(demuxStats)
+, outputBuffer_(outputBuffer)
+{
+}
+
+bool DemultiplexTask::execute(int32_t threadNum)
+{
+ switch(numBasesPerByte_)
+ {
+ case common::NumBasesPerByte::ONE:
+ return execute<common::NumBasesPerByte::ONE>(threadNum);
+ break;
+ case common::NumBasesPerByte::TWO:
+ return execute<common::NumBasesPerByte::TWO>(threadNum);
+ break;
+ case common::NumBasesPerByte::FOUR:
+ return execute<common::NumBasesPerByte::FOUR>(threadNum);
+ break;
+ default:
+ BCL2FASTQ_ASSERT_MSG(false, "Unsuported number of bases per byte. Supported values include: One, Two, or Four");
+ break;
+ };
+
+ return false;
+}
+
+Demultiplexer::Demultiplexer(
+ TaskQueue& taskQueue,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inputBuffersToUse,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& outputBuffersToSubmit,
+ const layout::Layout &layout,
+ const layout::LaneInfo &laneInfo,
+ const layout::BarcodeTranslationTable &barcodeTranslationTable,
+ bool includeNonPfClusters,
+ int32_t numThreads,
+ DemultiplexTask::DemuxStats &summaryDemuxStats
+)
+: Stage("Demultiplexing", taskQueue)
+, inputBuffersToUse_(inputBuffersToUse)
+, outputBuffersToSubmit_(outputBuffersToSubmit)
+, layout_(layout)
+, laneInfo_(laneInfo)
+, barcodeTranslationTable_(barcodeTranslationTable)
+, includeNonPfClusters_(includeNonPfClusters)
+, summaryDemuxStats_(summaryDemuxStats)
+, doDemultiplex_(false)
+, numIndexReads_(0)
+, stats_(numThreads)
+{
+ layout::LaneInfo::SampleInfosContainer::size_type sampleIndex = 0;
+ for (const layout::SampleInfo &sampleInfo : laneInfo_.getSampleInfos())
+ {
+ summaryDemuxStats_.push_back(DemultiplexTask::DemuxStats::value_type());
+
+ layout::BarcodesContainer::size_type barcodesCount = sampleInfo.getBarcodes().size();
+ if( 0==barcodesCount )
+ {
+ summaryDemuxStats_.back().push_back(DemultiplexTask::DemuxStats::value_type::value_type());
+ createTileStats( layout::BarcodeTranslationTable::SampleMetadata(sampleIndex) );
+ } else {
+ for( layout::BarcodesContainer::size_type barcodeIndex = 0;
+ barcodeIndex < barcodesCount;
+ ++barcodeIndex )
+ {
+ summaryDemuxStats_.back().push_back(DemultiplexTask::DemuxStats::value_type::value_type());
+ createTileStats( layout::BarcodeTranslationTable::SampleMetadata(sampleIndex,barcodeIndex) );
+ }
+ }
+ ++sampleIndex;
+ }
+
+ if (laneInfo.getSampleInfos().size() > 1)
+ {
+ for (const auto& readInfo : laneInfo.readInfos())
+ {
+ if ((readInfo.isIndexRead()) && !readInfo.cycleInfos().empty())
+ {
+ ++numIndexReads_;
+ doDemultiplex_ = true;
+ }
+ }
+ }
+ else
+ {
+ BCL2FASTQ_ASSERT_MSG( laneInfo.getSampleInfos().size() == 1,
+ "There must be at least default sample" );
+ // in case there is sample with barcode defined in sample sheet, there will be also default sample (i.e. at least 2 samples in total) and this branch will never get executed
+ BCL2FASTQ_ASSERT_MSG( laneInfo.sampleInfosBegin()->getBarcodes().empty(),
+ "There should be either default sample with no barcode or the only sample defined in sample sheet without barcode");
+ }
+
+ for (auto& stats : stats_)
+ {
+ stats = std::make_shared<DemultiplexTask::DemuxStatsSubset>(summaryDemuxStats_);
+ }
+}
+
+Demultiplexer::~Demultiplexer()
+{
+}
+
+void Demultiplexer::createTileStats(const layout::BarcodeTranslationTable::SampleMetadata &sampleMetadata)
+{
+ layout::LaneInfo::TileInfosContainer::size_type tilesCount = laneInfo_.getTileInfos().size();
+ for( layout::LaneInfo::TileInfosContainer::size_type tileIndex = 0;
+ tileIndex < tilesCount;
+ ++tileIndex )
+ {
+ summaryDemuxStats_.back().back().push_back(DemultiplexTask::DemuxStats::value_type::value_type::value_type(sampleMetadata,tileIndex));
+ }
+}
+
+bool Demultiplexer::startNewTasks()
+{
+ static int sequentialId = 0; ++sequentialId;
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Demultiplexer::startNewTask " << sequentialId << std::endl;
+
+ TaskQueue &taskQueue = this->getTaskQueue();
+ if (taskQueue.isTerminated())
+ {
+ outputBuffersToSubmit_.terminate();
+
+ // Only occurs if there was an error
+ return false;
+ }
+
+ auto taskManager = getNewTaskManager();
+ if (taskManager.get() == NULL)
+ {
+ DemuxTaskManager::waitForAllTasksToFinish();
+ outputBuffersToSubmit_.setFinished();
+ return false;
+ }
+
+ auto &outputBufferVec = taskManager->getBuffer().bclBuffers_;
+
+ // If no input, clear the output and return true...
+ bool hasCycleData = false;
+ for (data::BclBuffer &bclBuffer : outputBufferVec) {
+ hasCycleData = bclBuffer.hasCycleData();
+ if (hasCycleData) break;
+ }
+ if (!hasCycleData) {
+ return true;
+ }
+
+ calcFilterOffsets(outputBufferVec);
+
+ uint32_t taskNum = 0;
+
+ for (auto& outputBuffer : outputBufferVec)
+ {
+ BCL2FASTQ_ASSERT_MSG(outputBuffer.bcls_.size() > 0, "No BCL cycles to process");
+
+ if (doDemultiplex_)
+ {
+ const data::BclBuffer::BclCycleContainer::size_type dataSize = outputBuffer.bcls_.front().getNumClusters();
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Queue new demux tasks " << sequentialId << std::endl;
+ for (data::BclBuffer::BclCycleContainer::size_type offset = 0; offset < dataSize; offset += data::ClustersPerTask)
+ {
+ taskQueue.addData(std::make_shared<DemultiplexTask>(
+ taskManager,
+ outputBuffer,
+ laneInfo_.readInfos(),
+ offset,
+ std::min(offset+data::ClustersPerTask, dataSize),
+ barcodeTranslationTable_,
+ laneInfo_.getTileInfos().begin(),
+ numIndexReads_,
+ includeNonPfClusters_,
+ layout_.getNumBasesPerByte(),
+ stats_,
+ outputBuffer.samples_
+ ));
+
+ ++taskNum;
+ }
+ }
+ }
+
+ return true;
+}
+
+// TODO: Make this multithreaded
+void Demultiplexer::calcFilterOffsets(data::BclBufferVec& buffers)
+{
+ size_t bufferIndex = 0;
+ for (auto& buffer : buffers)
+ {
+ if (buffer.bcls_.back().includeNonPf_)
+ {
+ continue;
+ }
+
+ // We reserve a bit more than the necessary size in general to avoid
+ // reallocating when we find a bigger buffer.
+ if (buffer.cbclFilterOffsets_.empty())
+ {
+ buffer.cbclFilterOffsets_.reserve(buffer.filters_.capacity());
+ }
+
+ buffer.cbclFilterOffsets_.resize(buffer.filters_.size(), 0);
+
+ size_t passedFilterCount = 0;
+ size_t filterIndex = 0;
+ for (auto& filterRecord : buffer.filters_)
+ {
+ if (filterRecord.data_)
+ {
+ // Passed filter
+ buffer.cbclFilterOffsets_[filterIndex] = passedFilterCount;
+ ++passedFilterCount;
+ }
+
+ ++filterIndex;
+ }
+
+ ++bufferIndex;
+ }
+}
+
+void Demultiplexer::terminate()
+{
+ this->getTaskQueue().terminate();
+ inputBuffersToUse_.terminate();
+}
+
+std::shared_ptr<DemuxTaskManager> Demultiplexer::getNewTaskManager()
+{
+ std::shared_ptr<DemuxBuffer> inputBuffer;
+ if (!inputBuffersToUse_.tryGetData(inputBuffer))
+ {
+ // Only get here if we're done. Otherwise, tryGetData would wait.
+ return std::shared_ptr<DemuxTaskManager>();
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << "Group: " << inputBuffer->getGroupNumber() << " New DemuxBuffer" << std::endl;
+ return std::make_shared<DemuxTaskManager>(inputBuffer,
+ outputBuffersToSubmit_);
+}
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/conversion/FastqBuffer.cpp b/src/cxx/lib/conversion/FastqBuffer.cpp
new file mode 100644
index 0000000..5ee1613
--- /dev/null
+++ b/src/cxx/lib/conversion/FastqBuffer.cpp
@@ -0,0 +1,34 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FastqBuffer.cpp
+ *
+ * \brief Implementation of FASTQ buffer.
+ *
+ * \author Marek Balint
+ * \author Aaron Day
+ */
+
+
+#include "conversion/FastqBuffer.hh"
+
+
+namespace bcl2fastq {
+namespace conversion {
+
+
+FastqBuffer::FastqBuffer()
+: fastqs_()
+, groupNumber_(0)
+{
+}
+
+
+} // namespace conversion
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/conversion/FastqCreator.cpp b/src/cxx/lib/conversion/FastqCreator.cpp
new file mode 100644
index 0000000..36c1755
--- /dev/null
+++ b/src/cxx/lib/conversion/FastqCreator.cpp
@@ -0,0 +1,122 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FastqCreator.cpp
+ *
+ * \brief Implementation of FASTQ creator.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#include <string>
+#include <algorithm>
+#include <iterator>
+#include <utility>
+
+#include "conversion/FastqCreator.hh"
+
+namespace bcl2fastq
+{
+namespace conversion
+{
+
+std::atomic<uint32_t> FastqCreateTaskManager::numTaskManagers_(0);
+std::condition_variable FastqCreateTaskManager::cvAllTaskManagersDone_;
+std::mutex FastqCreateTaskManager::mut_;
+
+FastqCreateTaskManager::FastqCreateTaskManager(std::shared_ptr<DemuxBuffer>& inputBuffer,
+ ThreadSafeQueue<std::shared_ptr<DemuxBuffer>>& inputBuffersToRecycle,
+ std::shared_ptr<FastqBuffer>& outputBuffer,
+ ThreadSafeMap<uint32_t, std::shared_ptr<FastqBuffer>>& outputBufferMap)
+: TaskManager()
+, inputBuffer_(inputBuffer)
+, inputBuffersToRecycle_(inputBuffersToRecycle)
+, outputBuffer_(outputBuffer)
+, outputBufferMap_(outputBufferMap)
+{
+ ++numTaskManagers_;
+}
+
+FastqCreateTaskManager::~FastqCreateTaskManager()
+{
+ static int sequentialId = 0; ++sequentialId;
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "~FastqCreateTaskManager " << sequentialId << std::endl;
+
+ outputBuffer_->setGroupNumber(inputBuffer_->getGroupNumber());
+
+ // We're done with the inputBuffer; Recycle it.
+ inputBuffersToRecycle_.addData(inputBuffer_);
+
+ // TODO: This can block. Do this on a separate thread. Be careful about ensuring the thread completes.
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Queue new fastq-writer task " << sequentialId << std::endl;
+ outputBufferMap_.addData(outputBuffer_->getGroupNumber(), outputBuffer_);
+ --numTaskManagers_;
+
+ if (numTaskManagers_ == 0)
+ {
+ // This could be called more than once on different threads. I think
+ // this is ok.
+ cvAllTaskManagersDone_.notify_all();
+ }
+}
+
+void FastqCreateTaskManager::waitForAllTasksToFinish()
+{
+ std::unique_lock<std::mutex> lock(mut_);
+ cvAllTaskManagersDone_.wait(lock, [] { return FastqCreateTaskManager::numTaskManagers_ == 0; });
+}
+
+AllConversionStats::AllConversionStats(ConversionStatsForSampleTileBarcode& summaryConversionStats,
+ stats::BarcodeHits& summaryUnknownBarcodes)
+: summaryConversionStats_(summaryConversionStats)
+, summaryUnknownBarcodes_(summaryUnknownBarcodes)
+, conversionStats_(summaryConversionStats)
+, unknownBarcodes_(summaryUnknownBarcodes)
+{
+}
+
+AllConversionStats::~AllConversionStats()
+{
+// TODO: Ensure all of these objects are deleted on a single thread
+
+ size_t sampleIndex = 0;
+ for (const auto& statsForSample : conversionStats_)
+ {
+ size_t tileIndex = 0;
+ for (const auto& statsForTile : statsForSample)
+ {
+ size_t barcodeIndex = 0;
+ for (const auto& statsForBarcode : statsForTile)
+ {
+ summaryConversionStats_.at(sampleIndex).at(tileIndex).at(barcodeIndex).first += statsForBarcode.first;
+ size_t readIndex = 0;
+ for (const auto& statsForRead : statsForBarcode.second)
+ {
+ summaryConversionStats_.at(sampleIndex).at(tileIndex).at(barcodeIndex).second.at(readIndex) += statsForRead;
+ ++readIndex;
+ }
+
+ ++barcodeIndex;
+ }
+
+ ++tileIndex;
+ }
+
+ ++sampleIndex;
+ }
+
+ summaryUnknownBarcodes_.merge( unknownBarcodes_ );
+}
+
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/conversion/FastqWriter.cpp b/src/cxx/lib/conversion/FastqWriter.cpp
new file mode 100644
index 0000000..9eb836a
--- /dev/null
+++ b/src/cxx/lib/conversion/FastqWriter.cpp
@@ -0,0 +1,278 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FastqWriter.cpp
+ *
+ * \brief Implementation of FASTQ writer.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <utility>
+
+#include <boost/format.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/foreach.hpp>
+
+#include "common/Debug.hh"
+#include "conversion/FastqWriter.hh"
+
+
+namespace bcl2fastq {
+namespace conversion {
+
+std::atomic<uint32_t> FastqWriteTaskManager::numTaskManagers_(0);
+std::condition_variable FastqWriteTaskManager::cvAllTaskManagersDone_;
+std::mutex FastqWriteTaskManager::mut_;
+
+FastqWriteTaskManager::FastqWriteTaskManager(std::shared_ptr<FastqBuffer>& inputBuffer,
+ ThreadSafeQueue<std::shared_ptr<FastqBuffer>>& inputBuffersToRecycle)
+: TaskManager()
+, inputBuffer_(inputBuffer)
+, inputBuffersToRecycle_(inputBuffersToRecycle)
+{
+ ++numTaskManagers_;
+}
+
+FastqWriteTaskManager::~FastqWriteTaskManager()
+{
+ static int sequentialId = 0; ++sequentialId;
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "~FastqWriteTaskManager " << sequentialId << std::endl;
+
+ // We're done with the inputBuffer; Recycle it.
+ inputBuffersToRecycle_.addData(inputBuffer_);
+ --numTaskManagers_;
+
+ if (numTaskManagers_ == 0)
+ {
+ // This could be called more than once on different threads. I think
+ // this is ok.
+ cvAllTaskManagersDone_.notify_all();
+ }
+}
+
+void FastqWriteTaskManager::waitForAllTasksToFinish()
+{
+ std::unique_lock<std::mutex> lock(mut_);
+ cvAllTaskManagersDone_.wait(lock, [] { return FastqWriteTaskManager::numTaskManagers_ == 0; });
+}
+
+FastqWriteTask::FastqFileStateInfo::FastqFileStateInfo()
+: fastqFileStatePtr_()
+, nextGroupNumber_(0)
+, cvGroupNumber_()
+, mtx_()
+{
+}
+
+FastqWriteTask::FastqFileStateInfo::FastqFileStateInfo(FastqWriteTask::FastqFileStateInfo&& tmp)
+: fastqFileStatePtr_(tmp.fastqFileStatePtr_)
+, nextGroupNumber_(tmp.nextGroupNumber_)
+, cvGroupNumber_()
+, mtx_()
+{
+}
+
+FastqWriteTask::FastqFileStateInfo::FastqFileStateInfo(data::FastqFile::FastqFileStatePtr fastqFileStatePtr)
+: fastqFileStatePtr_(fastqFileStatePtr)
+, nextGroupNumber_(0)
+, cvGroupNumber_()
+, mtx_()
+{
+}
+
+void FastqWriteTask::FastqFileStateInfo::waitForGroupNumber(uint32_t groupNumber)
+{
+
+ std::unique_lock<std::mutex> lock(mtx_);
+ cvGroupNumber_.wait(lock, [this, groupNumber] { return nextGroupNumber_ == groupNumber; });
+}
+
+void FastqWriteTask::FastqFileStateInfo::signalWriteComplete()
+{
+ {
+ std::unique_lock<std::mutex> lock(mtx_);
+ ++nextGroupNumber_;
+ }
+
+ cvGroupNumber_.notify_all();
+}
+
+FastqWriteTask::FastqWriteTask(std::shared_ptr<FastqWriteTaskManager>& taskManager,
+ FastqFileStateInfo &fastqFileStateInfo,
+ const FastqBuffer::FastqsContainer::value_type::value_type &inputBuffer,
+ uint32_t groupNumber)
+: Task(taskManager)
+, fastqFileStateInfo_(fastqFileStateInfo)
+, inputBuffer_(inputBuffer)
+, groupNumber_(groupNumber)
+{
+}
+
+bool FastqWriteTask::execute(int32_t)
+{
+ fastqFileStateInfo_.waitForGroupNumber(groupNumber_);
+
+ for (const auto& read : inputBuffer_)
+ {
+ data::FastqFile::write(*fastqFileStateInfo_.fastqFileStatePtr_, &*read.begin(), read.size());
+ }
+
+ fastqFileStateInfo_.signalWriteComplete();
+
+ return true;
+}
+
+FastqWriter::FastqWriter(
+ TaskQueue& taskQueue,
+ ThreadSafeMap<uint32_t, std::shared_ptr<FastqBuffer>>& inputBuffersToUse,
+ ThreadSafeQueue<std::shared_ptr<FastqBuffer>>& inputBuffersToRecycle,
+ const layout::Layout &layout,
+ const layout::LaneInfo &laneInfo,
+ bool createFastqsForIndexReads,
+ const boost::filesystem::path& outputDir,
+ bool noLaneSplitting
+)
+: Stage("Fastq writing", taskQueue)
+, inputBuffersToUse_(inputBuffersToUse)
+, inputBuffersToRecycle_(inputBuffersToRecycle)
+, layout_(layout)
+, emptyFileDeleter_()
+, fastqFileStates_()
+, groupNumber_(0)
+{
+ layout::LaneInfo::SampleInfosContainer::size_type sampleInfosCounts = laneInfo.getSampleInfos().size();
+
+ BOOST_FOREACH (const layout::SampleInfo &sampleInfo, std::make_pair(laneInfo.sampleInfosBegin(), laneInfo.sampleInfosEnd()))
+ {
+ fastqFileStates_.push_back(FastqFileStatesContainer::value_type());
+ for (const auto& readInfo : laneInfo.readInfos())
+ {
+ if (!readInfo.shouldCreateFastqForRead(createFastqsForIndexReads))
+ {
+ continue;
+ }
+ std::ostringstream fileName;
+ fileName << (noLaneSplitting ?
+ boost::format("%s_S%d_%c%d_001.fastq.gz")
+ % sampleInfo.getName()
+ % sampleInfo.getNumber()
+ % (readInfo.isIndexRead() ? 'I' : 'R')
+ % readInfo.getNumber() :
+ boost::format("%s_S%d_%s_%c%d_001.fastq.gz")
+ % sampleInfo.getName()
+ % sampleInfo.getNumber()
+ % laneInfo.getDirName().string()
+ % (readInfo.isIndexRead() ? 'I' : 'R')
+ % readInfo.getNumber())
+ ;
+
+ boost::filesystem::path fastqPath(outputDir);
+ if (sampleInfo.hasProject())
+ {
+ fastqPath /= sampleInfo.getProject();
+ if( sampleInfo.getId() != sampleInfo.getName() )
+ {
+ fastqPath /= sampleInfo.getId();
+ }
+ }
+
+ boost::filesystem::path filePath(fastqPath / boost::filesystem::path(fileName.str()));
+ emptyFileDeleter_.addFile(filePath);
+
+ boost::filesystem::path dirPath = filePath.parent_path();
+ if (!dirPath.empty())
+ {
+ boost::filesystem::create_directories(dirPath);
+ }
+
+ fastqFileStates_.back().push_back(std::make_shared<FastqWriteTask::FastqFileStateInfo>( FastqWriteTask::FastqFileStateInfo(
+ data::FastqFile::openFile(filePath,
+ std::ios_base::out | std::ios_base::binary | (noLaneSplitting && laneInfo.getNumber() > 1 ?
+ std::ios_base::app :
+ std::ios_base::trunc )))));
+ }
+ }
+}
+
+void FastqWriter::terminate()
+{
+ this->getTaskQueue().terminate();
+ inputBuffersToUse_.terminate();
+}
+
+std::shared_ptr<FastqWriteTaskManager> FastqWriter::getNewTaskManager()
+{
+ std::shared_ptr<FastqBuffer> inputBuffer;
+ if (!inputBuffersToUse_.tryGetData(groupNumber_++, inputBuffer))
+ {
+ FastqWriteTaskManager::waitForAllTasksToFinish();
+
+ return std::shared_ptr<FastqWriteTaskManager>();
+ }
+
+ return std::make_shared<FastqWriteTaskManager>(inputBuffer,
+ inputBuffersToRecycle_);
+}
+
+bool FastqWriter::startNewTasks()
+{
+ static int sequentialId = 0; ++sequentialId;
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "FastqWriter::startNewTask " << sequentialId << std::endl;
+
+ TaskQueue &taskQueue = this->getTaskQueue();
+ if (taskQueue.isTerminated())
+ {
+ // Only occurs if there was an error
+ return false;
+ }
+
+ auto taskManager = getNewTaskManager();
+ if (taskManager.get() == NULL)
+ {
+ return false;
+ }
+
+ FastqBuffer &inputBuffer = taskManager->getInputBuffer();
+
+ FastqBuffer::FastqsContainer::value_type::size_type sampleIndex = 0;
+ for (const auto& sample : inputBuffer.fastqs_)
+ {
+ FastqBuffer::FastqsContainer::value_type::value_type::size_type readIndex = 0;
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Queue new fastq-writer tasks " << sequentialId << std::endl;
+ for (const auto& read : sample)
+ {
+ taskQueue.addData(std::make_shared<FastqWriteTask>(taskManager,
+ *fastqFileStates_.at(sampleIndex).at(readIndex),
+ read,
+ inputBuffer.getGroupNumber()));
+
+ ++readIndex;
+ }
+
+ ++sampleIndex;
+ }
+ return true;
+}
+
+FastqWriter::EmptyFileDeleter::~EmptyFileDeleter()
+{
+ for (const boost::filesystem::path& filePath : filePaths_)
+ {
+ if(boost::filesystem::exists(filePath) && boost::filesystem::file_size(filePath) == 0)
+ {
+ boost::filesystem::remove(filePath);
+ }
+ }
+}
+
+
+} // namespace conversion
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/conversion/SampleIndex.cpp b/src/cxx/lib/conversion/SampleIndex.cpp
new file mode 100644
index 0000000..dfcbf4e
--- /dev/null
+++ b/src/cxx/lib/conversion/SampleIndex.cpp
@@ -0,0 +1,311 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file SampleIndex.cpp
+ *
+ * \brief Implementation of sample index.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#include <algorithm>
+
+#include "common/Debug.hh"
+#include "conversion/SampleIndex.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace conversion {
+
+
+SampleIndex::SampleIndex(
+ layout::LaneInfo::SampleInfosContainer::size_type samplesCount,
+ SampleIndex::FastqOffsetsContainer::size_type maxFastqsCount
+)
+: state_(State::INITIALIZING)
+, offsets_(maxFastqsCount)
+, samplesMetaData_(samplesCount, SampleIndex::SampleMetaData())
+, fastqsCount_()
+{
+}
+
+void SampleIndex::reset(const data::BclBufferVec& bclBuffers)
+{
+ reset(getTotalClusters(bclBuffers));
+
+ for (const auto& inputBuffer : bclBuffers)
+ {
+ // find out how many reads in each sample
+ size_t bufferSize = inputBuffer.samples_.size();
+ for (data::BclBuffer::BclCycleContainer::size_type offset = 0; offset < bufferSize; ++offset)
+ {
+ incrementFastqCount(inputBuffer.samples_[offset].first.sampleIndex_);
+ }
+ }
+
+ reserve();
+
+ // initialize offsets
+ size_t bufferIndex = 0;
+ for (const auto& inputBuffer : bclBuffers)
+ {
+ size_t bufferSize = inputBuffer.samples_.size();
+ for (data::BclBuffer::BclCycleContainer::size_type offset = 0; offset < bufferSize; ++offset)
+ {
+ addOffset(inputBuffer.samples_[offset].first.sampleIndex_,
+ SampleIndex::FastqOffsetsContainer::value_type(bufferIndex, offset));
+ }
+
+ ++bufferIndex;
+ }
+
+ finalize();
+}
+
+size_t SampleIndex::getTotalClusters(const data::BclBufferVec& bclBuffers) const
+{
+ size_t dataSize = 0;
+
+ for (const auto& inputBuffer : bclBuffers)
+ {
+ dataSize += inputBuffer.samples_.size();
+ }
+
+ return dataSize;
+}
+
+
+void SampleIndex::reset(SampleIndex::FastqOffsetsContainer::size_type fastqsCount)
+{
+ fastqsCount_ = fastqsCount;
+ if( fastqsCount > offsets_.size() )
+ {
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Resizing offsets table to cope with " << fastqsCount << " clusters/tile" << std::endl;
+ offsets_.resize(fastqsCount + defaultFastqStep_);
+ }
+ SampleMetaData smd;
+ smd.begin = offsets_.begin();
+ smd.end = offsets_.begin();
+ smd.next = offsets_.begin();
+ std::fill(samplesMetaData_.begin(), samplesMetaData_.end(), smd);
+
+ state_ = State::CALCULATING_SIZES;
+}
+
+void SampleIndex::incrementFastqCount(
+ layout::LaneInfo::SampleInfosContainer::size_type sample,
+ SampleIndex::FastqOffsetsContainer::size_type count
+)
+{
+ BCL2FASTQ_ASSERT_MSG( sample < samplesMetaData_.size(),
+ "Invalid sample index: " << sample << "/" << (samplesMetaData_.size()-1));
+ BCL2FASTQ_ASSERT_MSG( offsets_.size() > static_cast<size_t>(samplesMetaData_[sample].end - samplesMetaData_[sample].begin) + count,
+ "Sample index #" << sample << " tries to increment FASTQ count past the end of the offsets table"
+ " (size: " << offsets_.size() << ")" );
+
+ std::advance(samplesMetaData_[sample].end, count);
+}
+
+void SampleIndex::reserveHelper(
+ std::vector<SampleIndex::SampleMetaData>::size_type &offset,
+ SampleIndex::SampleMetaData &sampleMetaData
+)
+{
+ sampleMetaData.begin += offset;
+ sampleMetaData.end += offset;
+ sampleMetaData.next = sampleMetaData.begin;
+
+ offset += sampleMetaData.end - sampleMetaData.begin;
+}
+
+void SampleIndex::reserve()
+{
+ BCL2FASTQ_ASSERT_MSG(
+ state_ == State::CALCULATING_SIZES,
+ "Sample index must be in state CALCULATING_SIZES: " << state_
+ );
+
+ std::vector<SampleIndex::SampleMetaData>::size_type offset = 0;
+ std::for_each(
+ samplesMetaData_.begin(),
+ samplesMetaData_.end(),
+ boost::bind(
+ reserveHelper,
+ boost::ref(offset),
+ _1
+ )
+ );
+
+ BCL2FASTQ_ASSERT_MSG(
+ samplesMetaData_.front().begin == offsets_.begin(),
+ "The first offset of the first sample must be at the first position of offsets array"
+ );
+ BCL2FASTQ_ASSERT_MSG(
+ samplesMetaData_.back().end == (offsets_.begin() + fastqsCount_),
+ "The last offset of the last sample must be at the position of last FASTQ offset in offsets array"
+ );
+
+ state_ = State::BUILDING_INDEX;
+}
+
+void SampleIndex::addOffset(
+ layout::LaneInfo::SampleInfosContainer::size_type sample,
+ const FastqOffsetsContainer::value_type& offset
+)
+{
+ BCL2FASTQ_ASSERT_MSG(
+ sample < samplesMetaData_.size(),
+ "Invalid sample index: " << sample << "/" << (samplesMetaData_.size()-1)
+ );
+ BCL2FASTQ_ASSERT_MSG(
+ state_ == State::BUILDING_INDEX,
+ "Sample index must be in state BUILDING_INDEX: " << state_
+ );
+ BCL2FASTQ_ASSERT_MSG(
+ samplesMetaData_[sample].next < samplesMetaData_[sample].end,
+ "Too many offsets for sample: " << sample << ":" << (samplesMetaData_[sample].end - samplesMetaData_[sample].begin)
+ );
+
+ *(samplesMetaData_[sample].next++) = offset;
+}
+
+bool SampleIndex::finalizePredicate(
+ SampleIndex::FastqOffsetsContainer::const_iterator offsetsBegin,
+ SampleIndex::FastqOffsetsContainer::const_iterator offsetsEnd,
+ const SampleIndex::SampleMetaData &sampleMetaData
+)
+{
+ // mind reversed logic of this predicate
+ return !(
+ sampleMetaData.begin >= offsetsBegin
+ &&
+ sampleMetaData.begin <= offsetsEnd
+ &&
+ sampleMetaData.end >= offsetsBegin
+ &&
+ sampleMetaData.end <= offsetsEnd
+ &&
+ sampleMetaData.next >= offsetsBegin
+ &&
+ sampleMetaData.next <= offsetsEnd
+ &&
+ sampleMetaData.begin <= sampleMetaData.end
+ &&
+ sampleMetaData.next == sampleMetaData.end
+ );
+}
+
+void SampleIndex::finalize()
+{
+ BCL2FASTQ_ASSERT_MSG(
+ state_ == State::BUILDING_INDEX,
+ "Sample index must be in state BUILDING_INDEX: " << state_
+ );
+
+ {
+ std::vector<SampleMetaData>::const_iterator invalidMetaData = find_if(
+ samplesMetaData_.begin(),
+ samplesMetaData_.end(),
+ boost::bind(
+ SampleIndex::finalizePredicate,
+ offsets_.begin(),
+ offsets_.end(),
+ _1
+ )
+ );
+
+#if 0
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << "SAMPLE INDEX: " << std::endl;
+ BOOST_FOREACH( const SampleIndex::SampleMetaData &sm, std::make_pair( samplesMetaData_.begin(), samplesMetaData_.end() ))
+ {
+ BclBuffer::BclsContainer::value_type::size_type &begin = *(sm.begin);
+ BclBuffer::BclsContainer::value_type::size_type &end = *(sm.end);
+ BclBuffer::BclsContainer::value_type::size_type &next = *(sm.next);
+
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << "\t{" << begin
+ << ", " << end
+ << ", " << next << "}" << std::endl;
+ }
+#endif
+
+ BCL2FASTQ_ASSERT_MSG(
+ invalidMetaData == samplesMetaData_.end(),
+ "Invalid index: " << (invalidMetaData - samplesMetaData_.begin())
+ );
+ }
+
+ state_ = State::INDEX_BUILT;
+}
+
+SampleIndex::FastqOffsetsContainer::const_iterator SampleIndex::offsetsBegin(layout::LaneInfo::SampleInfosContainer::size_type sample) const
+{
+ BCL2FASTQ_ASSERT_MSG(
+ sample < samplesMetaData_.size(),
+ "Invalid sample index: " << sample << "/" << (samplesMetaData_.size()-1)
+ );
+ BCL2FASTQ_ASSERT_MSG(
+ state_ == State::INDEX_BUILT,
+ "Sample index must be in state INDEX_BUILT: " << state_
+ );
+
+ return samplesMetaData_[sample].begin;
+}
+
+SampleIndex::FastqOffsetsContainer::const_iterator SampleIndex::offsetsEnd(layout::LaneInfo::SampleInfosContainer::size_type sample) const
+{
+ BCL2FASTQ_ASSERT_MSG(
+ sample < samplesMetaData_.size(),
+ "Invalid sample ID: " << sample << "/" << (samplesMetaData_.size()-1)
+ );
+ BCL2FASTQ_ASSERT_MSG(
+ state_ == State::INDEX_BUILT,
+ "Sample index must be in state INDEX_BUILT: " << state_
+ );
+
+ return samplesMetaData_[sample].end;
+}
+
+SampleIndex::FastqOffsetsContainer::iterator SampleIndex::offsetsBegin(layout::LaneInfo::SampleInfosContainer::size_type sample)
+{
+ BCL2FASTQ_ASSERT_MSG(
+ sample < samplesMetaData_.size(),
+ "Invalid sample index: " << sample << "/" << (samplesMetaData_.size()-1)
+ );
+ BCL2FASTQ_ASSERT_MSG(
+ state_ == State::INDEX_BUILT,
+ "Sample index must be in state INDEX_BUILT: " << state_
+ );
+
+ return samplesMetaData_[sample].begin;
+}
+
+SampleIndex::FastqOffsetsContainer::iterator SampleIndex::offsetsEnd(layout::LaneInfo::SampleInfosContainer::size_type sample)
+{
+ BCL2FASTQ_ASSERT_MSG(
+ sample < samplesMetaData_.size(),
+ "Invalid sample ID: " << sample << "/" << (samplesMetaData_.size()-1)
+ );
+ BCL2FASTQ_ASSERT_MSG(
+ state_ == State::INDEX_BUILT,
+ "Sample index must be in state INDEX_BUILT: " << state_
+ );
+
+ return samplesMetaData_[sample].end;
+}
+
+
+} // namespace conversion
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/conversion/Stage.cpp b/src/cxx/lib/conversion/Stage.cpp
new file mode 100644
index 0000000..03a701f
--- /dev/null
+++ b/src/cxx/lib/conversion/Stage.cpp
@@ -0,0 +1,70 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Stage.cpp
+ *
+ * \brief Implementation of a processing stage.
+ *
+ * \author Marek Balint
+ */
+
+
+//#define WITH_CPU_TIMER
+
+
+#include <boost/thread/locks.hpp>
+#include <boost/format.hpp>
+
+#include <future>
+#include "common/Timer.hh"
+#include "common/Logger.hh"
+#include "conversion/Stage.hh"
+
+#include <chrono>
+#include <thread>
+
+namespace bcl2fastq {
+namespace conversion {
+
+
+Stage::Stage(const std::string& stageName,
+ TaskQueue& taskQueue)
+: taskQueue_(taskQueue)
+, stageName_(stageName)
+{
+}
+
+Stage::~Stage()
+{
+}
+
+void Stage::run()
+{
+ try
+ {
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Starting " << stageName_ << " stage" << std::endl;
+
+ while (this->startNewTasks()) { }
+ }
+ catch (...)
+ {
+ this->terminate();
+ throw;
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << stageName_ << " stage done" << std::endl;
+}
+
+TaskQueue & Stage::getTaskQueue()
+{
+ return taskQueue_;
+}
+
+} // namespace task
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/conversion/Task.cpp b/src/cxx/lib/conversion/Task.cpp
new file mode 100644
index 0000000..6aa7ba5
--- /dev/null
+++ b/src/cxx/lib/conversion/Task.cpp
@@ -0,0 +1,38 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Task.cpp
+ *
+ * \brief Implementation of a workload item a.k.a. task.
+ *
+ * \author Marek Balint
+ */
+
+
+#include "conversion/Task.hh"
+#include "conversion/Stage.hh"
+
+#include "common/Debug.hh"
+
+namespace bcl2fastq {
+namespace conversion {
+
+Task::Task(std::shared_ptr<TaskManager> taskManager)
+ : taskManager_(taskManager)
+{
+}
+
+Task::~Task()
+{
+}
+
+
+} // namespace task
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/conversion/ThreadPool.cpp b/src/cxx/lib/conversion/ThreadPool.cpp
new file mode 100644
index 0000000..e2f64ea
--- /dev/null
+++ b/src/cxx/lib/conversion/ThreadPool.cpp
@@ -0,0 +1,102 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Task.cpp
+ *
+ * \brief Implementation of a thread pool.
+ *
+ * \author Aaron Day
+ */
+
+#include "conversion/ThreadPool.hh"
+#include "conversion/TaskQueue.hh"
+#include "conversion/Task.hh"
+#include "common/Debug.hh"
+
+namespace bcl2fastq
+{
+namespace conversion
+{
+
+ThreadPool::JoinThreads::JoinThreads(std::vector<std::thread>& threads,
+ boost::exception_ptr& firstThreadException)
+ : firstThreadException_(firstThreadException)
+ , threads_(threads)
+{
+}
+
+ThreadPool::JoinThreads::~JoinThreads()
+{
+ for (std::thread& thread : threads_)
+ {
+ if (thread.joinable()) thread.join();
+ }
+
+ if (firstThreadException_)
+ {
+ boost::rethrow_exception(firstThreadException_);
+ }
+}
+
+ThreadPool::ThreadPool(uint32_t numThreads,
+ TaskQueue& dataQueue,
+ Terminator& terminator)
+ : dataQueue_(dataQueue)
+ , terminator_(terminator)
+ , mut_()
+ , firstThreadException_()
+ , threads_()
+ , joiner_(threads_,
+ firstThreadException_)
+{
+ for (uint32_t i = 0; i < numThreads; ++i)
+ {
+ threads_.push_back(std::thread(&ThreadPool::workerThread, this, i));
+ }
+}
+
+void ThreadPool::workerThread(uint32_t threadNum)
+{
+ try
+ {
+ TaskPtr task;
+ while (dataQueue_.tryGetData(task))
+ {
+ task->execute(threadNum);
+ task.reset();
+ }
+ }
+ catch(...)
+ {
+ std::lock_guard<std::mutex> lock(mut_);
+
+ if (!firstThreadException_)
+ {
+ firstThreadException_ = boost::current_exception();
+
+ terminator_.terminate();
+ dataQueue_.setFinished();
+
+ BCL2FASTQ_LOG(common::LogLevel::ERROR_TYPE)
+ << "Thread: " << threadNum
+ << " caught an exception first: " << boost::current_exception_diagnostic_information()
+ << std::endl;
+ }
+ else
+ {
+ BCL2FASTQ_LOG(common::LogLevel::ERROR_TYPE)
+ << "Thread: " << threadNum
+ << " also caught an exception: " << boost::current_exception_diagnostic_information()
+ << std::endl;
+ }
+ }
+}
+
+} // namespace conversion
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/conversion/cppunit/CMakeLists.txt b/src/cxx/lib/conversion/cppunit/CMakeLists.txt
new file mode 100644
index 0000000..ec43d99
--- /dev/null
+++ b/src/cxx/lib/conversion/cppunit/CMakeLists.txt
@@ -0,0 +1,23 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for any cppunit subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CPPUNIT_CMAKE})
+
+
diff --git a/src/cxx/lib/conversion/cppunit/RegistryNames.txt b/src/cxx/lib/conversion/cppunit/RegistryNames.txt
new file mode 100644
index 0000000..27c6e6b
--- /dev/null
+++ b/src/cxx/lib/conversion/cppunit/RegistryNames.txt
@@ -0,0 +1 @@
+AdapterLocator
diff --git a/src/cxx/lib/conversion/cppunit/testAdapterLocator.cpp b/src/cxx/lib/conversion/cppunit/testAdapterLocator.cpp
new file mode 100644
index 0000000..96010f9
--- /dev/null
+++ b/src/cxx/lib/conversion/cppunit/testAdapterLocator.cpp
@@ -0,0 +1,192 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testAdapterLocator.cpp
+ *
+ * \brief AdapterLocator cppunit test declarations.
+ *
+ * \author Aaron Day
+ */
+
+
+#include <string>
+
+#include "RegistryName.hh"
+#include "testAdapterLocator.hh"
+
+
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(TestAdapterLocator, registryName("AdapterLocator"));
+
+void TestAdapterLocator::reset()
+{
+ expectedResults_.clear();
+ bclContainer_.bcls_.clear();
+ bclContainer_.bcls_.resize(100);
+}
+
+void TestAdapterLocator::addToBclContainer(const std::string& read,
+ size_t expectedResult)
+{
+ BOOST_ASSERT(bclContainer_.bcls_.size() == read.size());
+
+ expectedResults_.push_back(expectedResult);
+
+ for (size_t i = 0; i < read.size(); ++i)
+ {
+ switch (read[i])
+ {
+ case 'A':
+ bclContainer_.bcls_[i].bcls_.push_back('@');
+ break;
+ case 'C':
+ bclContainer_.bcls_[i].bcls_.push_back('A');
+ break;
+ case 'G':
+ bclContainer_.bcls_[i].bcls_.push_back('B');
+ break;
+ case 'T':
+ bclContainer_.bcls_[i].bcls_.push_back('C');
+ break;
+ case 'N':
+ bclContainer_.bcls_[i].bcls_.push_back('\0');
+ break;
+ default:
+ BOOST_ASSERT(false);
+ break;
+ }
+ }
+}
+
+void TestAdapterLocator::setUp()
+{
+}
+
+void TestAdapterLocator::tearDown()
+{
+}
+
+void TestAdapterLocator::testAll()
+{
+ testShortAdapter();
+
+ reset();
+
+ testLongAdapter();
+
+ reset();
+
+ testBeginningAdapter();
+
+ reset();
+
+ // This test should pass:
+ // test2Mismatch();
+}
+
+void TestAdapterLocator::testShortAdapter()
+{
+ addToBclContainer(
+ "NNNNNNNNNNNNNNNNNNNNNNNNNGGCCGGGAGTTGGGCGAGTACGGGCTGCAGGCATACACTGAAGTGAAAACTGCTGTCTCTTATTACACATCTCCG",
+ 77);
+
+ addToBclContainer(
+ "NNNNNNNNNNNNNNNNNNNNNNNNNTATCCTCTTTCTTCTCCTTTCCCTTTTCTTCCCCCTTNNNNNNNNCNNNNNNNNNNNNNNCTGTCTCTTATACAC",
+ 85);
+
+ addToBclContainer(
+ "NNNNNNNNNNNNNNNNNNNNNNNNNNACGTAGGTATTCTTTCTTTCTTCTTTTTCCTTTTCTTCCTTNCTCCTNNNNNNNNNNNNNNNNNNNNNNNNNNN",
+ 100);
+
+ addToBclContainer(
+ "NNNNNNNNNNNNNNNNNNNNNNNNNCGTCCACACCCTCCCCGGCCAGCCATGCACCGTTCTCTTATACACATCTCCGAGCCCACGAGACAGGGAGAAATT",
+ 54);
+
+ addToBclContainer(
+ "NNNNNNNNNNNNNNNNNNNNNNNNNAAGCACAGGGCAATCATATTGTGCTTTTCACTGTCTCCAGGACATGTCTCTTATACACATCTCCGAGCCCACGAG",
+ 67);
+
+ addToBclContainer(
+ "NNNNNNNNNNNNNNNNNNNNNNNGCCTCATTGCCACAGCTGTTCTCTTATACACATCTCCGAGCCCACGAGACTAAGGCGAATCTCGTATGCCGTCTTCT",
+ 38);
+
+ addToBclContainer(
+ "NNNNNNNNNNNNNNNNNNNNNNNAAAACCACTTTGTCCTCATCTCATTCCCCAAAACCGGCAGGCCCTCTCAGTTCTTTTTCCTGTCTTTACTTCTCTTA",
+ 91);
+
+ addToBclContainer(
+ "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNC",
+ 99);
+
+ bcl2fastq::conversion::AdapterLocatorWithIndels<bcl2fastq::common::NumBasesPerByte::ONE> adapterLocator(
+ "CTGTCTCTTATACACATCT",
+ .8,
+ 30);
+
+ verifyAdapterLocator(adapterLocator);
+}
+
+void TestAdapterLocator::testLongAdapter()
+{
+ addToBclContainer(
+ "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNC",
+ 99);
+
+ addToBclContainer(
+ "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNCTGTCTCTTATACACATCTACTGCTAGTCGCGTTACGGTTNNNNNNNNNNNNNNN",
+ 45);
+
+ bcl2fastq::conversion::AdapterLocatorWithIndels<bcl2fastq::common::NumBasesPerByte::ONE> adapterLocator(
+ "CTGTCTCTTATACACATCTACTGCTAGTCGCGTTACGGTT",
+ .8,
+ 30);
+
+ verifyAdapterLocator(adapterLocator);
+}
+
+void TestAdapterLocator::testBeginningAdapter()
+{
+ addToBclContainer(
+ "ATCGGAAGACACACGTCTGAACTCCAGTCACATTCCTCGATCTCGTATGCCGTTTTCTGCTTGAAAAAACCAACAACACCTGGGAACAAGCGAATCACCN",
+ 0);
+
+ bcl2fastq::conversion::AdapterLocatorWithIndels<bcl2fastq::common::NumBasesPerByte::ONE> adapterLocator(
+ "AGATCGGAAGAGCACACGTCTGAACTCCAGTCA",
+ .9,
+ 35);
+
+ verifyAdapterLocator(adapterLocator);
+}
+
+void TestAdapterLocator::test2Mismatch()
+{
+ addToBclContainer(
+ "TGTAAACATTTCTTTAGTAGAATCTGCAAGTTGATATTTAGATAGCTAGGAAGATTTCCTTGGAAACGGGAATATCTTAATATAAACTCTAGACGGAAGC",
+ 100);
+
+ bcl2fastq::conversion::AdapterLocatorWithIndels<bcl2fastq::common::NumBasesPerByte::ONE> adapterLocator(
+ "AGATCGGAAGAGCACACGTCTGAACTCCAGTCA",
+ .9,
+ 35);
+
+ verifyAdapterLocator(adapterLocator);
+}
+
+void TestAdapterLocator::verifyAdapterLocator(bcl2fastq::conversion::AdapterLocator<bcl2fastq::common::NumBasesPerByte::ONE>& adapterLocator)
+{
+ for (unsigned int i = 0; i < bclContainer_.bcls_.front().bcls_.size(); ++i)
+ {
+ bcl2fastq::conversion::FastqConstIterator<bcl2fastq::common::NumBasesPerByte::ONE> begin(bclContainer_, i);
+ bcl2fastq::conversion::FastqConstIterator<bcl2fastq::common::NumBasesPerByte::ONE> end(bclContainer_, i);
+ end += bclContainer_.bcls_.size();
+
+ bcl2fastq::conversion::FastqConstIterator<bcl2fastq::common::NumBasesPerByte::ONE> pos = adapterLocator.identifyAdapter(begin, end);
+
+ CPPUNIT_ASSERT_EQUAL(std::distance(begin, pos), expectedResults_[i]);
+ }
+}
+
diff --git a/src/cxx/lib/conversion/cppunit/testAdapterLocator.hh b/src/cxx/lib/conversion/cppunit/testAdapterLocator.hh
new file mode 100644
index 0000000..fdb52cf
--- /dev/null
+++ b/src/cxx/lib/conversion/cppunit/testAdapterLocator.hh
@@ -0,0 +1,64 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testAdapterLocator.hh
+ *
+ * \brief AdapterLocator cppunit test declarations.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_CONVERSION_TEST_ADAPTER_LOCATOR_HH
+#define BCL2FASTQ_CONVERSION_TEST_ADAPTER_LOCATOR_HH
+
+#include "data/BclBuffer.hh"
+#include "conversion/AdapterLocator.hh"
+
+#include <string>
+#include <vector>
+
+#include <cppunit/extensions/HelperMacros.h>
+
+
+/// \brief Test suite for AdapterLocaator.
+class TestAdapterLocator : public CppUnit::TestFixture
+{
+ CPPUNIT_TEST_SUITE(TestAdapterLocator);
+ CPPUNIT_TEST(testAll);
+ CPPUNIT_TEST_SUITE_END();
+
+public:
+ TestAdapterLocator() : bclContainer_(), expectedResults_() { bclContainer_.bcls_.resize(100); }
+
+ void setUp();
+ void tearDown();
+ void testAll();
+
+private:
+
+ void reset();
+
+ void addToBclContainer(const std::string& read,
+ size_t expectedResult);
+
+ void testLongAdapter();
+ void testShortAdapter();
+ void testBeginningAdapter();
+ void test2Mismatch();
+
+ void verifyAdapterLocator(bcl2fastq::conversion::AdapterLocator<bcl2fastq::common::NumBasesPerByte::ONE>& adapterLocator);
+
+ bcl2fastq::data::BclBuffer bclContainer_;
+ std::vector<long> expectedResults_;
+};
+
+
+#endif // BCL2FASTQ_CONVERSION_TEST_ADAPTER_LOCATOR_HH
+
+
diff --git a/src/cxx/lib/data/AggregatedBclFileReader.cpp b/src/cxx/lib/data/AggregatedBclFileReader.cpp
new file mode 100644
index 0000000..7a7a429
--- /dev/null
+++ b/src/cxx/lib/data/AggregatedBclFileReader.cpp
@@ -0,0 +1,165 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA^M
+ * and certain third party copyright/licenses, and any user of this^M
+ * source file is bound by the terms therein.^M
+ *
+ * \file AggregatedBclFileReader.cpp
+ *
+ * \brief Implementation of Bcl file reader for aggregated tiles.
+ *
+ * \author Aaron Day
+ */
+
+#include "data/AggregatedBclFileReader.hh"
+
+namespace bcl2fastq {
+namespace data {
+
+AggregatedBclFileReader::AggregatedBclFileReader(const::boost::filesystem::path& inputDir,
+ const layout::LaneInfo& laneInfo,
+ common::CycleNumber cycleNumber,
+ size_t cycleIndex,
+ bool ignoreMissingBcls,
+ std::shared_ptr<io::SyncFile> bclFile,
+ std::shared_ptr<data::CycleBCIFile> cycleBciFile)
+: BclFileReaderT<io::SyncFile>(inputDir,
+ laneInfo,
+ cycleNumber,
+ cycleIndex,
+ ignoreMissingBcls,
+ bclFile)
+, cycleBciFile_(cycleBciFile)
+{
+}
+
+bool AggregatedBclFileReader::read(RawBclBufferGroup& outputBuffer)
+{
+ if (outputBuffer.size() == 0) return false;
+
+ if (!openAggFileIfNeeded())
+ {
+ outputBuffer.clearClusters(cycleIndex_, bclFile_);
+ return false;
+ }
+
+ bool success = true;
+ for (auto& outputBufferForTile : outputBuffer)
+ {
+ auto tileInfoIter = outputBufferForTile.tileInfo_;
+
+ io::SyncFile::SyncFileReader bclFileReader(*bclFile_,
+ std::distance(laneInfo_.getTileInfos().begin(),
+ tileInfoIter));
+
+ outputBufferForTile.cycleData_[cycleIndex_].bcls_.path_ = bclFile_->getPath();
+
+ // 1st gzip block with data for tile
+ data::CycleBCIFile::Record firstBlockBciRecord = cycleBciFile_->getRecord(tileInfoIter->getIndex());
+ bclFileReader.seek(firstBlockBciRecord.compressedOffset, std::ios_base::beg);
+
+ outputBufferForTile.uncompressedBclOffset_ = firstBlockBciRecord.uncompressedOffset;
+
+ // Look for the next tile that is inside of the last gzip block used by the current tile.
+ auto nextTileInsideBlock = tileInfoIter + 1;
+
+ // Determine where to stop reading.
+ //
+ // nextStartPos is the start of a gzip-block containing the next tile.
+ // Unless the uncompressed offset is 0, the current tile will
+ // end in this block so we need to read this too.
+ // A value of zero is interpretted as EOF.
+ // --------------------------------
+ int64_t nextStartPos = 0;
+
+ const bool isLastTileInBciFile = (tileInfoIter->getIndex()+1 == cycleBciFile_->getTilesCount());
+ if (!isLastTileInBciFile) // && nextTileInsideBlock != laneInfo_.getTileInfos().end())
+ {
+ // Read enough gzp blocks to load tile.
+ if (nextTileInsideBlock != laneInfo_.getTileInfos().end()) {
+ data::CycleBCIFile::Record endRecord = cycleBciFile_->getRecord(nextTileInsideBlock->getIndex());
+
+ // This is start of the next tile. Some clusters will be in the next gzip block. readGzipBlocks will read this too.
+ nextStartPos = endRecord.compressedOffset;
+ }
+ }
+
+ int64_t startPos = firstBlockBciRecord.compressedOffset;
+//BCL2FASTQ_LOG(common::LogLevel::NONE) << "Tile: " << tileInfoIter->getNumber() << " startPos: " << startPos << " next: " << nextStartPos << std::endl;
+
+ if (!bclFileReader.readGzipBlocks(outputBufferForTile.cycleData_[cycleIndex_].bcls_,
+ startPos, nextStartPos))
+ {
+ success = false;
+ break;
+ }
+ }
+
+ if (!success) {
+
+ std::stringstream msg;
+ msg << "Found corrupt aggregated BCL data in cycle " << (1 + cycleIndex_)
+ << " starting at tile " << outputBuffer.begin()->tileInfo_->getNumber() << ". ";
+
+ if (ignoreMissingBcls_) {
+
+ msg << "Data for this cycle will be will be ignored. ";
+ msg << "File: '" << bclFile_->getPath() << "'" << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << msg.str();
+
+ // This will clear cluster bcl_ data for the current cycle and tile.
+ outputBuffer.clearClusters(cycleIndex_, bclFile_);
+
+ // We also want to ignore ALL remaining tiles for this cycle.
+ bclFile_.reset();
+ }
+ else
+ {
+ msg << "Aborting. ";
+ msg << "File: '" << bclFile_->getPath() << "'" << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << msg.str();
+
+ BOOST_THROW_EXCEPTION(common::InputDataError(msg.str()));
+ }
+ }
+
+ return success;
+}
+
+bool AggregatedBclFileReader::openAggFileIfNeeded()
+{
+ if (!bclFile_)
+ {
+ return false;
+ }
+
+ if (!bclFile_->isOpen())
+ {
+ if (!cycleBciFile_->openFile(inputDir_, laneInfo_.getNumber(), cycleNumber_))
+ {
+ return false;
+ }
+
+ boost::filesystem::path bclPath;
+ if (!data::BclFile::getAndVerifyFileName(inputDir_,
+ laneInfo_.getNumber(),
+ cycleNumber_,
+ ignoreMissingBcls_,
+ bclPath))
+ {
+ return false;
+ }
+
+ bclFile_->openFile(bclPath,
+ ignoreMissingBcls_,
+ 0); // There is only 1 file. It is definitely index 0.
+ }
+
+ return true;
+}
+
+} // namespace data
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/data/BclFile.cpp b/src/cxx/lib/data/BclFile.cpp
new file mode 100644
index 0000000..fb7520c
--- /dev/null
+++ b/src/cxx/lib/data/BclFile.cpp
@@ -0,0 +1,241 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BclFile.cpp
+ *
+ * \brief Implementation of BCL file.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ * \author Aaron Day
+ */
+
+
+#include "io/GzipDecompressor.hh"
+#include "data/BclFile.hh"
+#include "common/Debug.hh"
+#include "common/Logger.hh"
+#include "common/SystemCompatibility.hh"
+
+#include <boost/filesystem/operations.hpp>
+#include <boost/format.hpp>
+
+namespace bcl2fastq {
+namespace data {
+
+
+BclFile::BclFile(const common::RawDataBuffer& data,
+ io::GzipDecompressor& decompressor,
+ bool ignoreErrors,
+ bool parseHeader,
+ bool resetDecompressor, /* = true */
+ common::ClustersCount defaultClustersCount /*= 0*/)
+: FileReaderBase(data,
+ ignoreErrors)
+, BinaryFileReader(data,
+ ignoreErrors,
+ defaultClustersCount)
+, gzipDecompressor_(&decompressor)
+, isCompressed_(data.path_.extension() == ".gz" || data.path_.extension() == ".bgzf")
+{
+ if (resetDecompressor)
+ {
+ gzipDecompressor_->flush();
+ gzipDecompressor_->reset();
+ }
+
+ static const std::string errMsgBegin = "Ignoring file open failure for bcl file '";
+
+ try
+ {
+ bool headerRead = false;
+ if (parseHeader)
+ {
+ headerRead = true;
+ if (!readHeader(clustersCount_))
+ {
+ return;
+ }
+
+ /// \todo Refactoring: Do not determine endianness in runtime, let CMake to do it.
+ if (!common::isLittleEndian())
+ {
+ clustersCount_ = common::swapEndian(clustersCount_);
+ }
+ }
+
+ if (headerRead) {
+ BCL2FASTQ_LOG(common::LogLevel::INFO)
+ << "Opened BCL file '" << this->getPath().string() << "' with "
+ << clustersCount_ << " clusters" << std::endl;
+ }
+ }
+ CATCH_AND_IGNORE(std::ios_base::failure, errMsgBegin)
+ CATCH_AND_IGNORE(io::ZlibError, errMsgBegin)
+ CATCH_AND_IGNORE_ALL(errMsgBegin)
+}
+
+bool BclFile::getAndVerifyFileName(const boost::filesystem::path& inputDir,
+ common::LaneNumber laneNumber,
+ common::CycleNumber cycleNumber,
+ bool ignoreErrors,
+ boost::filesystem::path& fileName)
+{
+ fileName = boost::filesystem::path(
+ inputDir
+ /
+ boost::filesystem::path((boost::format("L%03d") % laneNumber).str())
+ /
+ boost::filesystem::path((boost::format("%04d.bcl.bgzf") % cycleNumber).str())
+ );
+
+ if (!boost::filesystem::exists(fileName))
+ {
+ std::string errStr("Unable to find BCL file: '" + fileName.string() + "'");
+ if (ignoreErrors)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << errStr << std::endl;
+ return false;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(ENOENT, errStr));
+ }
+
+ fileName = boost::filesystem::path();
+ }
+
+ return true;
+}
+
+bool BclFile::getAndVerifyFileName(const boost::filesystem::path& inputDir,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ common::CycleNumber cycleNumber,
+ bool ignoreErrors,
+ boost::filesystem::path& fileName)
+{
+ std::string sample = (boost::format("s_%d_%d") % laneNumber % tileNumber).str();
+
+ fileName = boost::filesystem::path(
+ inputDir
+ /
+ boost::filesystem::path((boost::format("L%03d") % laneNumber).str())
+ /
+ boost::filesystem::path((boost::format("C%d.1") % cycleNumber).str())
+ /
+ boost::filesystem::path((boost::format("%s.bcl.gz") % sample).str())
+ );
+
+ if (!boost::filesystem::exists(fileName))
+ {
+ fileName.replace_extension();
+
+ if (!boost::filesystem::exists(fileName))
+ {
+ std::string errStr("Unable to find BCL file for '" + sample + "' in: " + fileName.remove_filename().string());
+ if (ignoreErrors)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << errStr << std::endl;
+ return false;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(ENOENT, errStr));
+ }
+
+ fileName = boost::filesystem::path();
+ }
+ }
+
+ return true;
+}
+
+std::streamsize BclFile::read(std::istream &is, char *buffer, std::streamsize size)
+{
+ if( isCompressed_ )
+ {
+ auto ret = gzipDecompressor_->read(is, buffer, size);
+ return ret;
+ }
+ else
+ {
+ if (!is.read(buffer, size)) return -1;
+
+ return is.gcount();
+ }
+
+ return 0;
+}
+
+std::streamsize BclFile::read(char* targetBuffer, std::streamsize targetSize)
+{
+ static const std::string errMsgBegin = "Ignoring read failure for bcl file '";
+
+ if (!validateCondition(this->isOpen(), "File is not open")) { return 0; }
+
+ std::streamsize bytesRead = -1;
+ try
+ {
+ bytesRead = read(istr_, targetBuffer, targetSize);
+ while ((bytesRead < targetSize) && !istr_.eof() && bytesRead != -1)
+ {
+ std::streamsize bytesRead2 = read(istr_, targetBuffer+bytesRead, targetSize-bytesRead);
+ if (bytesRead2 != -1)
+ {
+ bytesRead += bytesRead2;
+ } else {
+ break;
+ }
+ }
+ }
+ CATCH_AND_IGNORE(io::ZlibError, errMsgBegin)
+ CATCH_AND_IGNORE(common::IoError, errMsgBegin)
+ CATCH_AND_IGNORE(std::ios_base::failure, errMsgBegin)
+ CATCH_AND_IGNORE_ALL(errMsgBegin)
+
+ return bytesRead;
+}
+
+bool BclFile::seek(std::streamsize compressedOffset, std::streamsize uncompressedOffset)
+{
+ if (!this->isOpen() && ignoreErrors_) {
+ return false;
+ }
+
+ static const std::string errMsgBegin = "Ignoring seek failure for: '";
+
+ bool success = false;
+ try
+ {
+ istr_.seekg(compressedOffset);
+ istr_.sync();
+ std::vector<char> targetBuffer(uncompressedOffset);
+
+ gzipDecompressor_->reset();
+ gzipDecompressor_->flush();
+ std::streamsize bytesRead = gzipDecompressor_->read(istr_, &*(targetBuffer.begin()), uncompressedOffset);
+
+ int errnum = errno;
+ success = (bytesRead == uncompressedOffset);
+ if (!success)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, (boost::format("Unable to seek to given position of BCL file '%s': compressed_offset=%d uncompressed_offset=%d bytes_read=%d") % this->getPath().string() % compressedOffset % uncompressedOffset % bytesRead).str()));
+ }
+ }
+ CATCH_AND_IGNORE(common::IoError, errMsgBegin)
+ CATCH_AND_IGNORE_ALL(errMsgBegin)
+
+ return success;
+}
+
+
+} // namespace data
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/data/CMakeLists.txt b/src/cxx/lib/data/CMakeLists.txt
new file mode 100644
index 0000000..26c8f9d
--- /dev/null
+++ b/src/cxx/lib/data/CMakeLists.txt
@@ -0,0 +1,23 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the cxx/lib/conversion subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CXX_LIBRARY_CMAKE})
+
+
diff --git a/src/cxx/lib/data/CbclFile.cpp b/src/cxx/lib/data/CbclFile.cpp
new file mode 100644
index 0000000..5b8d7b0
--- /dev/null
+++ b/src/cxx/lib/data/CbclFile.cpp
@@ -0,0 +1,365 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA^M
+ * and certain third party copyright/licenses, and any user of this^M
+ * source file is bound by the terms therein.^M
+ *
+ * \file CbclFile.cpp
+ *
+ * \brief Implementation of CBCL file.
+ *
+ * \author Eunho Noh
+ * \author Aaron Day
+ */
+
+
+#include "data/CbclFile.hh"
+#include "layout/TileInfo.hh"
+
+#include <boost/filesystem/operations.hpp>
+#include <boost/iostreams/stream.hpp>
+#include <boost/iostreams/device/array.hpp>
+#include <boost/lexical_cast.hpp>
+
+
+
+
+#include <boost/iostreams/filter/gzip.hpp>
+#include <boost/algorithm/string.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+
+namespace bcl2fastq {
+namespace data {
+
+
+
+CbclFileReader::Header::Header()
+: m_version(0)
+, m_header_size(0)
+, m_number_of_bits_per_basecall(0)
+, m_number_of_bits_per_qscore(0)
+, m_number_of_qscore_bins(0)
+, m_qscore_bins()
+, m_remapped_qscores()
+, m_number_of_tiles(0)
+, m_tile_ids()
+, m_number_of_clusters()
+, m_uncompressed_block_size()
+, m_compressed_block_size()
+, m_non_pf_filtered(0)
+, m_start_positions()
+{
+}
+
+bool CbclFileReader::Header::load(io::SyncFile::SyncFileReader &cbclFile, bool ignoreErrors)
+{
+ try
+ {
+ // This can throw an exception if the header is corrupt.
+ readHeader(cbclFile);
+ }
+ catch(...)
+ {
+ if (ignoreErrors)
+ {
+ static const std::string msg = "Error reading CBCL file header. All data for this cycle in unavailable and will be skipped. ";
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << msg << std::endl;
+ return false;
+ }
+ else {
+ throw;
+ }
+ }
+
+ const std::string errorMsg = validateHeaderInfo();
+ if (!errorMsg.empty())
+ {
+ if (ignoreErrors)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << errorMsg << "(" << cbclFile.getPath() << ") " << std::endl;
+ return false;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(std::runtime_error(errorMsg));
+ }
+ }
+
+ return true;
+}
+
+void CbclFileReader::Header::readHeader(io::SyncFile::SyncFileReader& in)
+{
+ read_binary(in, m_version);
+ read_binary(in, m_header_size);
+ read_binary(in, m_number_of_bits_per_basecall);
+ read_binary(in, m_number_of_bits_per_qscore);
+ read_binary(in, m_number_of_qscore_bins);
+
+ if (m_number_of_qscore_bins > 0)
+ {
+ m_qscore_bins.resize(m_number_of_qscore_bins);
+ m_remapped_qscores.resize(m_number_of_qscore_bins);
+
+ for (header_value_t i = 0; i < m_number_of_qscore_bins; ++i)
+ {
+ read_binary(in, m_qscore_bins[i]);
+ read_binary(in, m_remapped_qscores[i]);
+ }
+ }
+
+ read_binary(in, m_number_of_tiles);
+
+ if (m_number_of_tiles > 0)
+ {
+ m_tile_ids.resize(m_number_of_tiles);
+ m_number_of_clusters.resize(m_number_of_tiles);
+ m_uncompressed_block_size.resize(m_number_of_tiles);
+ m_compressed_block_size.resize(m_number_of_tiles);
+ for (header_value_t i = 0; i < m_number_of_tiles; ++i)
+ {
+ read_binary(in, m_tile_ids[i]);
+ read_binary(in, m_number_of_clusters[i]);
+ read_binary(in, m_uncompressed_block_size[i]);
+ read_binary(in, m_compressed_block_size[i]);
+ }
+ }
+
+ read_binary(in, m_non_pf_filtered);
+
+ if (m_number_of_tiles > 0)
+ {
+ m_start_positions.resize(m_number_of_tiles);
+ m_start_positions[0] = m_header_size;
+ for (size_t i = 1; i < m_number_of_tiles; ++i)
+ {
+ m_start_positions[i] = m_start_positions[i-1] + m_compressed_block_size[i-1];
+ }
+ }
+}
+
+void CbclFileReader::Header::reset()
+{
+ m_version = 0;
+ m_header_size = 0;
+ m_number_of_bits_per_basecall = 0;
+ m_number_of_bits_per_qscore = 0;
+ m_number_of_qscore_bins = 0;
+ m_qscore_bins.resize(0);
+ m_remapped_qscores.resize(0);
+ m_number_of_tiles = 0;
+ m_tile_ids.resize(0);
+ m_number_of_clusters.resize(0);
+ m_uncompressed_block_size.resize(0);
+ m_compressed_block_size.resize(0);
+ m_non_pf_filtered = 0;
+ m_start_positions.resize(0);
+}
+
+CbclFileReader::Header::header_value_t CbclFileReader::Header::calculateHeaderSize(size_t number_of_qscore_bins, size_t number_of_tiles) const
+{
+ return sizeof(header_version_t) + //version number
+ sizeof(header_value_t) + // header size
+ sizeof(header_byte_t) + // number of bits per basecall
+ sizeof(header_byte_t) + // number of bits per q-score
+ sizeof(header_value_t) + // number of q-score bins
+ number_of_qscore_bins * sizeof(header_value_t) + // q-score bins
+ number_of_qscore_bins * sizeof(header_value_t) + // q-scores remapped
+ sizeof(header_byte_t) + // non-pf clusters filtered/un-filtered
+ sizeof(header_value_t) + // number of tiles
+ number_of_tiles * sizeof(header_value_t) + // tile ids
+ number_of_tiles * sizeof(header_value_t) + // number of clusters per tile
+ number_of_tiles * sizeof(header_value_t) + // uncompressed size
+ number_of_tiles * sizeof(header_value_t); // compressed size
+}
+
+std::string CbclFileReader::Header::validateHeaderInfo() const
+{
+ static std::string badFileMsg = "A corrupt CBCL file was identified and cannot be loaded. ";;
+ if (m_version != 1)
+ {
+ return badFileMsg; // + "Unsupported CBCL file version. ";
+ }
+ else if(m_header_size != calculateHeaderSize(m_number_of_qscore_bins, m_number_of_tiles))
+ {
+ return badFileMsg; // + "The header size information does not match the expected size.";
+ }
+ else if(m_number_of_bits_per_basecall+m_number_of_bits_per_qscore != NUM_BITS_BCL_STORAGE)
+ {
+ return badFileMsg; // + "Unsupported basecall/qscore size";
+ }
+ else if(m_number_of_qscore_bins != m_qscore_bins.size() || m_number_of_qscore_bins != m_remapped_qscores.size() )
+ {
+ return badFileMsg; // + "The q-score information is invalid.";
+ }
+ else if(m_number_of_tiles != m_tile_ids.size() || m_number_of_tiles != m_number_of_clusters.size() ||
+ m_number_of_tiles != m_uncompressed_block_size.size() || m_number_of_tiles != m_compressed_block_size.size())
+ {
+ return badFileMsg; // + "The tile information is invalid.";
+ }
+ for(size_t tile = 0; tile < m_number_of_tiles; ++tile)
+ {
+ if(m_number_of_clusters[tile] > m_uncompressed_block_size[tile] * BASECALLS_PER_BYTE)
+ {
+ return badFileMsg; // + "The tile information is invalid.";
+ }
+ }
+
+ static const std::string noError;
+ return noError;
+}
+
+
+CbclFileReader::CbclFileReader(const::boost::filesystem::path& inputDir,
+ const layout::LaneInfo& laneInfo,
+ common::CycleNumber cycleNumber,
+ size_t cycleIndex,
+ bool ignoreMissingBcls,
+ const common::TileFileMap& tileFileMap,
+ std::shared_ptr<io::SyncFile> bclFile)
+: BclFileReaderT<io::SyncFile>(inputDir,
+ laneInfo,
+ cycleNumber,
+ cycleIndex,
+ ignoreMissingBcls,
+ bclFile)
+, tileFileMap_(tileFileMap)
+, header_()
+{
+}
+
+bool CbclFileReader::doesFileExist(const boost::filesystem::path& inputDir,
+ common::LaneNumber laneNumber,
+ common::CycleNumber cycleNumber,
+ common::TileNumber tileNumber,
+ const common::TileFileMap& tileFileMap,
+ bool ignoreErrors)
+{
+ boost::filesystem::path fileName = getFileName(inputDir,
+ laneNumber,
+ cycleNumber,
+ tileNumber,
+ tileFileMap,
+ ignoreErrors);
+
+ return boost::filesystem::exists(fileName);
+}
+
+void CbclFileReader::markCycleDataInvalid(PerCycleData &cycleData)
+{
+ // reset data
+ cycleData.bcls_.resize(0);
+ cycleData.uncompressedBlockSize_ = 0;
+
+ if (header_) { header_->reset(); }
+}
+
+bool CbclFileReader::read(RawBclBufferGroup& outputBuffer)
+{
+ for (auto& outputBufferForTile : outputBuffer)
+ {
+ io::SyncFile::SyncFileReader fileReader(*bclFile_,
+ std::distance(laneInfo_.getTileInfos().begin(),
+ outputBufferForTile.tileInfo_));
+
+ auto& cycleData = outputBufferForTile.cycleData_[cycleIndex_];
+
+ boost::filesystem::path fileName = getFileName(inputDir_,
+ laneInfo_.getNumber(),
+ cycleNumber_,
+ outputBufferForTile.tileInfo_->getNumber(),
+ tileFileMap_,
+ ignoreMissingBcls_);
+
+ if (ignoreMissingBcls_ && !boost::filesystem::exists(fileName))
+ {
+ // Method getFileName issues a warning.
+ cycleData.bcls_.path_ = fileName;
+ markCycleDataInvalid(cycleData);
+ continue;
+ }
+ // else let it fail below
+
+ if (header_ && header_->getHeaderSize() == 0) {
+ continue;
+ }
+
+ if (!header_ || !header_->containsTile(outputBufferForTile.tileInfo_->getNumber()))
+ {
+ // header_ = new
+ fileReader.openFile(fileName,
+ ignoreMissingBcls_);
+
+ header_ = std::unique_ptr<Header>(new Header());
+ if (!header_->load(fileReader, ignoreMissingBcls_))
+ {
+ cycleData.bcls_.path_ = fileName;
+ markCycleDataInvalid(cycleData);
+ continue;
+ }
+ }
+
+ cycleData.includeNonPf_ = !header_->isNonPfFiltered();
+
+ if (header_->isQscoreDeflated())
+ {
+ cycleData.numBitsPerQscore_ = header_->getNumberOfBitsPerQscore();
+ cycleData.remappedQscores_ = header_->getRemappedQscores();
+ }
+
+ auto tileIndex = header_->getTileIndex(outputBufferForTile.tileInfo_->getNumber());
+ const size_t startPosition = header_->getBlockStartPosition(tileIndex);
+ const uint32_t compressedBlockSize = header_->getCompressedBlockSize(tileIndex);
+
+ cycleData.uncompressedBlockSize_ = header_->getUncompressedBlockSize(tileIndex);
+
+ fileReader.seek(startPosition,
+ std::ios_base::beg);
+
+ cycleData.bcls_.resize(compressedBlockSize);
+ fileReader.read(cycleData.bcls_,
+ compressedBlockSize);
+ }
+
+ return true;
+}
+
+boost::filesystem::path CbclFileReader::getFileName(const boost::filesystem::path& inputDir,
+ common::LaneNumber laneNumber,
+ common::CycleNumber cycleNumber,
+ common::TileNumber tileNumber,
+ const common::TileFileMap& tileFileMap,
+ bool ignoreErrors)
+{
+ boost::filesystem::path cbclPath;
+ auto pos = tileFileMap.at(laneNumber).find(tileNumber);
+ if (pos != tileFileMap.at(laneNumber).end())
+ {
+ cbclPath =
+ inputDir /
+ boost::filesystem::path((boost::format("L%03d") % laneNumber).str()) /
+ boost::filesystem::path((boost::format("C%d.1") % cycleNumber).str()) /
+ pos->second;
+ }
+
+ if (pos == tileFileMap.at(laneNumber).end() || !boost::filesystem::exists(cbclPath))
+ {
+ std::string errStr("Unable to find CBCL file: '" + cbclPath.string() + "'");
+ if (ignoreErrors)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << errStr << std::endl;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(ENOENT, errStr));
+ }
+ }
+
+ return cbclPath;
+}
+
+} // namespace data
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/data/ControlFile.cpp b/src/cxx/lib/data/ControlFile.cpp
new file mode 100644
index 0000000..03ec77f
--- /dev/null
+++ b/src/cxx/lib/data/ControlFile.cpp
@@ -0,0 +1,110 @@
+/**
+* BCL to FASTQ file converter
+* Copyright (c) 2007-2017 Illumina, Inc.
+*
+* This software is covered by the accompanying EULA
+* and certain third party copyright/licenses, and any user of this
+* source file is bound by the terms therein.
+*
+* \file ControlFile.cpp
+*
+* \brief Implementation of control file.
+*
+* \author Aaron Day
+*/
+
+#include "data/ControlFile.hh"
+
+#include <boost/filesystem/operations.hpp>
+
+namespace bcl2fastq
+{
+namespace data
+{
+
+ControlFile::ControlFile(const common::RawDataBuffer& data,
+ bool ignoreErrors /*= false*/)
+ : FileReaderBase(data,
+ ignoreErrors)
+ , BinaryAllClustersFileReader(data,
+ ignoreErrors)
+{
+}
+
+bool ControlFile::readHeader()
+{
+ if (!BinaryAllClustersFileReader::readHeader(clustersCount_))
+ {
+ return false;
+ }
+
+ if (clustersCount_ == 0)
+ {
+ Header header;
+ if (!BinaryAllClustersFileReader::readHeader(header))
+ {
+ return false;
+ }
+
+ if (!validateHeader(header))
+ {
+ return false;
+ }
+
+ clustersCount_ = header.clustersCount_;
+ }
+
+ return true;
+}
+
+bool ControlFile::validateHeader(const ControlFile::Header& header)
+{
+ int errnum = errno;
+ if (header.version_ != Header::VERSION)
+ {
+ const uint32_t version = header.version_;
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, (boost::format("Corrupted header of control file '%s': header_version=%d") % getPath().string() % version).str()));
+ }
+
+ return true;
+}
+
+std::size_t ControlFile::read(
+ ControlFile::Record *targetBuffer,
+ std::size_t targetSize
+)
+{
+ if (!validateCondition(this->isOpen(), "File is not open")) { return 0; }
+
+ return readBytes(reinterpret_cast<char*>(targetBuffer), targetSize);
+}
+
+const uint32_t ControlFile::Header::VERSION = 2;
+
+bool ControlFile::doesFileExist(const boost::filesystem::path& intensitiesDir,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ boost::filesystem::path& fileName)
+{
+ boost::filesystem::path laneDir((boost::format("L%03d") % laneNumber).str());
+ boost::filesystem::path filePath = intensitiesDir / laneDir;
+
+ fileName = filePath / (boost::format("s_%d_%d.control") % laneNumber % tileNumber).str();
+ if (boost::filesystem::exists(fileName))
+ {
+ return true;
+ }
+
+ // zero padded
+ fileName = filePath / (boost::format("s_%d_%04d.control") % laneNumber % tileNumber).str();
+ if (boost::filesystem::exists(fileName))
+ {
+ return true;
+ }
+
+ return false;
+}
+
+} // namespace data
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/data/CycleBCIFile.cpp b/src/cxx/lib/data/CycleBCIFile.cpp
new file mode 100644
index 0000000..e6d4d46
--- /dev/null
+++ b/src/cxx/lib/data/CycleBCIFile.cpp
@@ -0,0 +1,173 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file CycleBCIFile.cpp
+ *
+ * \brief Implementation of Cycle BCI file.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <boost/format.hpp>
+
+#include "io/FileBufWithReopen.hh"
+#include "data/CycleBCIFile.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace data {
+
+
+CycleBCIFile::CycleBCIFile(bool ignoreMissingBcls)
+: ignoreMissingBcls_(ignoreMissingBcls)
+, path_()
+, tilesCount_()
+, buffer_()
+, bufferPosition_()
+{
+}
+
+bool CycleBCIFile::getAndVerifyFileName(const boost::filesystem::path& inputDir,
+ common::LaneNumber laneNumber,
+ common::CycleNumber cycleNumber,
+ bool ignoreErrors,
+ boost::filesystem::path& fileName)
+{
+ fileName = boost::filesystem::path(
+ inputDir
+ /
+ boost::filesystem::path((boost::format("L%03d") % laneNumber).str())
+ /
+ boost::filesystem::path((boost::format("%04d.bcl.bgzf.bci") % cycleNumber).str())
+ );
+
+ if (!boost::filesystem::exists(fileName))
+ {
+ std::string errStr("Unable to find BCI file: '" + fileName.string() + "'");
+ if (ignoreErrors)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << errStr << std::endl;
+ return false;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(ENOENT, errStr));
+ }
+
+ fileName = boost::filesystem::path();
+ }
+
+ return true;
+}
+
+bool CycleBCIFile::openFile(
+ const boost::filesystem::path &inputDir,
+ common::LaneNumber laneNumber,
+ common::CycleNumber cycleNumber
+)
+{
+ if (!getAndVerifyFileName(inputDir, laneNumber, cycleNumber, ignoreMissingBcls_, path_))
+ {
+ return false;
+ }
+
+ io::FileBufWithReopen fileBuf(std::ios_base::in | std::ios_base::binary);
+ fileBuf.reopen(path_, io::FileBufWithReopen::FadviseFlags::SEQUENTIAL_ONCE);
+ int errnum = errno;
+ if (!fileBuf.is_open())
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errnum, (boost::format("Unable to open cycle BCI file '%s' for reading") % path_.string()).str()));
+ }
+
+ Header header;
+ std::streamsize headerLength = io::read(fileBuf, reinterpret_cast<char *>(&header), sizeof(header));
+ errnum = errno;
+ if (headerLength != sizeof(header))
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, (boost::format("Unable to read header of cycle BCI file '%s': bytes_read=%d bytes_expected=%d") % path_.string() % headerLength % sizeof(header)).str()));
+ }
+ if (header.version_ != Header::VERSION)
+ {
+ const uint32_t version = header.version_;
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, (boost::format("Corrupted header of cycle BCI file '%s': header_version=%d") % path_.string() % version).str()));
+ }
+ tilesCount_ = header.tilesCount_;
+
+ std::size_t bufferSize = sizeof(Record) * tilesCount_;
+ buffer_.resize(bufferSize);
+ std::streamsize bytesRead = io::read(fileBuf, reinterpret_cast<char *>(&*buffer_.begin()), bufferSize);
+ errnum = errno;
+ if (bytesRead != static_cast<std::streamsize>(bufferSize))
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, (boost::format("Cycle BCI file '%s' truncated: bytes_read=%d bytes_expected=%d") % path_.string() % bytesRead % bufferSize).str()));
+ }
+ bufferPosition_ = buffer_.begin();
+
+ return true;
+}
+
+bool CycleBCIFile::isOpen() const
+{
+ return !path_.empty();
+}
+
+boost::filesystem::path CycleBCIFile::getPath() const
+{
+ BCL2FASTQ_ASSERT_MSG(this->isOpen(), "File is not open");
+ return path_;
+}
+
+common::TileNumber CycleBCIFile::getTilesCount() const
+{
+ BCL2FASTQ_ASSERT_MSG(this->isOpen(), "File is not open");
+ return tilesCount_;
+}
+
+std::size_t CycleBCIFile::read(
+ CycleBCIFile::Record *targetBuffer,
+ std::size_t targetSize
+)
+{
+ BCL2FASTQ_ASSERT_MSG(this->isOpen(), "File is not open");
+
+ BCL2FASTQ_ASSERT_MSG((buffer_.end() - bufferPosition_) >= 0, "Invalid buffer size.");
+ BCL2FASTQ_ASSERT_MSG((buffer_.end() - bufferPosition_) % sizeof(Record) == 0, "Invalid buffer size.");
+ const std::size_t recordsToRead = std::min(
+ static_cast<std::size_t>(buffer_.end() - bufferPosition_) / sizeof(Record),
+ targetSize
+ );
+
+ const Record * const begin = reinterpret_cast<const Record * const>(&*bufferPosition_);
+ const Record * const end = begin + recordsToRead;
+
+ std::copy(begin, end, targetBuffer);
+ bufferPosition_ += (recordsToRead * sizeof(Record));
+
+ return recordsToRead;
+}
+
+CycleBCIFile::Record CycleBCIFile::getRecord(std::size_t tileIndex) const
+{
+ BCL2FASTQ_ASSERT_MSG(this->isOpen(), "File is not open");
+ const Record * const ret = reinterpret_cast<const Record * const>(&*(buffer_.begin() + tileIndex*sizeof(Record)));
+ return *ret;
+}
+
+
+const uint32_t CycleBCIFile::Header::VERSION = 0;
+
+
+} // namespace data
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/data/FastqFile.cpp b/src/cxx/lib/data/FastqFile.cpp
new file mode 100644
index 0000000..a7dd57a
--- /dev/null
+++ b/src/cxx/lib/data/FastqFile.cpp
@@ -0,0 +1,112 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FastqFile.cpp
+ *
+ * \brief Implementation of FASTQ file.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <boost/format.hpp>
+
+#include "common/Debug.hh"
+#include "common/Logger.hh"
+#include "common/SystemCompatibility.hh"
+#include "data/FastqFile.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace data {
+
+
+FastqFileState::FastqFileState()
+: path_()
+, fileBuf_(std::ios_base::binary | std::ios_base::out | std::ios_base::app)
+{
+}
+
+FastqFileState::~FastqFileState()
+{
+}
+
+void FastqFileState::write(std::ostream &os, const char *sourceBuffer, std::streamsize sourceSize)
+{
+ os.write(sourceBuffer, sourceSize);
+}
+
+
+namespace detail {
+
+
+/// \brief Default FASTQ file internal state.
+class DefaultFastqFileState : public FastqFileState
+{
+private:
+
+ virtual void write(std::ostream &os, const char *sourceBuffer, std::streamsize sourceSize);
+};
+
+void DefaultFastqFileState::write(std::ostream &os, const char *sourceBuffer, std::streamsize sourceSize)
+{
+ FastqFileState::write(os, sourceBuffer, sourceSize);
+}
+
+
+} // namespace detail
+
+
+FastqFile::FastqFileStatePtr FastqFile::openFile(const boost::filesystem::path &path,
+ std::ios_base::openmode mode)
+{
+ FastqFileStatePtr fileState(new detail::DefaultFastqFileState);
+ fileState->path_ = path;
+
+ io::FileBufWithReopen fileBuf(std::ios_base::out | mode);
+ fileBuf.reopen(path, io::FileBufWithReopen::FadviseFlags::SEQUENTIAL_ONCE);
+ int errnum = errno;
+ if (!fileBuf.is_open())
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errnum, (boost::format("Unable to open FASTQ file '%s' for writing") % path.string()).str()));
+ }
+ fileState->fileBuf_.reopen(path, io::FileBufWithReopen::FadviseFlags::SEQUENTIAL_ONCE);
+ errnum = errno;
+ if (!fileState->fileBuf_.is_open())
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errnum, (boost::format("Unable to open FASTQ file '%s' for writing") % path.string()).str()));
+ }
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Created FASTQ file '" << path << std::endl;
+
+ return fileState;
+}
+
+const boost::filesystem::path& FastqFile::getPath(const FastqFileState &fileState)
+{
+ return fileState.path_;
+}
+
+void FastqFile::write(FastqFileState &fileState, const char *sourceBuffer, std::streamsize sourceSize)
+{
+ //fileBuf_.reopen(fileState.path_.native().c_str(), io::FileBufWithReopen::FadviseFlags::SEQUENTIAL_ONCE);
+ //BCL2FASTQ_ASSERT_MSG(fileBuf_.is_open(), "Unable to reopen FASTQ file '" << fileState.path_.native() << "' for append");
+ BCL2FASTQ_ASSERT_MSG(fileState.fileBuf_.is_open(), "FASTQ file '" << fileState.path_ << "' is not opened");
+ std::ostream os(&fileState.fileBuf_);
+
+ fileState.write(os, sourceBuffer, sourceSize);
+}
+
+
+} // namespace data
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/data/FileReader.cpp b/src/cxx/lib/data/FileReader.cpp
new file mode 100644
index 0000000..711f568
--- /dev/null
+++ b/src/cxx/lib/data/FileReader.cpp
@@ -0,0 +1,177 @@
+/**
+* BCL to FASTQ file converter
+* Copyright (c) 2007-2017 Illumina, Inc.
+*
+* This software is covered by the accompanying EULA
+* and certain third party copyright/licenses, and any user of this
+* source file is bound by the terms therein.
+*
+* \file FileReader.cpp
+*
+* \brief Implementation of FileReader.
+*
+* \author Aaron Day
+*/
+
+#include "data/FileReader.hh"
+
+#include "common/Debug.hh"
+#include "common/Exceptions.hh"
+
+
+namespace bcl2fastq
+{
+namespace data
+{
+
+FileReaderBase::FileReaderBase(const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ common::ClustersCount defaultClustersCount /*= 0*/)
+ : data_(&data)
+ , ignoreErrors_(ignoreErrors)
+ , clustersCount_(defaultClustersCount)
+ , inputSrc_(data_->data(), data_->size())
+ , istr_(inputSrc_)
+{
+}
+
+FileReaderBase::~FileReaderBase()
+{
+}
+
+const boost::filesystem::path& FileReaderBase::getPath() const
+{
+ validateCondition(this->isOpen(), "File is not open");
+ return data_->path_;
+}
+
+common::ClustersCount FileReaderBase::getClustersCount() const
+{
+ validateCondition(this->isOpen(), "File is not open");
+ return clustersCount_;
+}
+
+bool FileReaderBase::validateCondition(bool condition, const std::string& warningMsg) const
+{
+ if (!condition)
+ {
+ if (ignoreErrors_)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << warningMsg << std::endl;
+ }
+ else
+ {
+ BCL2FASTQ_ASSERT_MSG(condition, warningMsg);
+ }
+ }
+
+ return condition;
+}
+
+FileReader::FileReader(const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ common::ClustersCount defaultClustersCount /*=0*/)
+ : FileReaderBase(data,
+ ignoreErrors,
+ defaultClustersCount)
+{
+}
+
+FileReader::~FileReader()
+{
+}
+
+void FileReader::logError(std::streamsize bytesRead,
+ std::streamsize bytesExpected) const
+{
+ int errnum = errno;
+
+ boost::format errFormat(boost::format(
+ getFileTypeStr() + " file '%s' truncated: bytes_read=%d bytes_expected=%d") %
+ this->getPath().string() % bytesRead % bytesExpected);
+
+ if (ignoreErrors_)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING)
+ << errFormat
+ << ":" << std::strerror(errnum) << " (" << errnum << ")" << std::endl;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum,
+ errFormat.str()));
+ }
+}
+
+std::streamsize FileReader::readBytes(char* buffer,
+ uint32_t bytes)
+{
+ std::streamsize bytesRead = 0;
+ istr_.read(buffer, bytes);
+ bytesRead = istr_.gcount();
+ if (!istr_ || bytesRead < bytes)
+ {
+ logError(bytesRead,
+ bytes);
+ }
+
+ return bytesRead;
+}
+
+BinaryFileReader::BinaryFileReader(const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ common::ClustersCount defaultClustersCount /*=0*/)
+ : FileReaderBase(data,
+ defaultClustersCount)
+ , FileReader(data,
+ ignoreErrors,
+ defaultClustersCount)
+{
+}
+
+BinaryFileReader::~BinaryFileReader()
+{
+}
+
+std::streamsize BinaryFileReader::read(char *targetBuffer, std::streamsize targetSize)
+{
+ if (istr_.read(targetBuffer, targetSize))
+ {
+ return targetSize;
+ }
+
+ return istr_.gcount();
+}
+
+BinaryAllClustersFileReader::BinaryAllClustersFileReader(const common::RawDataBuffer& data,
+ bool ignoreErrors)
+ : FileReaderBase(data,
+ ignoreErrors)
+ , BinaryFileReader(data,
+ ignoreErrors)
+{
+}
+
+bool BinaryAllClustersFileReader::readClusters(std::vector<char>& buffer,
+ uint32_t clustersCount)
+{
+ std::streamsize bufferSize = getRecordBytes() * clustersCount;
+ std::streamsize bytesRead = readBytes(buffer.data(), bufferSize);
+
+ if (bufferSize != bytesRead)
+ {
+ buffer.resize(bytesRead > 0 ? bytesRead - bytesRead % getRecordBytes() : 0);
+ return false;
+ }
+
+ return true;
+}
+
+void BinaryAllClustersFileReader::validateHeader()
+{
+ // Let derived classes decide if they want to implement this
+}
+
+} // namespace data
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/data/FilterFile.cpp b/src/cxx/lib/data/FilterFile.cpp
new file mode 100644
index 0000000..dcec610
--- /dev/null
+++ b/src/cxx/lib/data/FilterFile.cpp
@@ -0,0 +1,158 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FilterFile.cpp
+ *
+ * \brief Implementation of filter file.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#include <boost/format.hpp>
+#include <boost/filesystem/operations.hpp>
+
+#include "io/FileBufWithReopen.hh"
+#include "data/FilterFile.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace data {
+
+
+FilterFileBase::FilterFileBase(const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ bool skipHeader)
+ : FileReaderBase(data,
+ ignoreErrors)
+ , BinaryAllClustersFileReader(data,
+ ignoreErrors)
+ , skipHeader_(skipHeader)
+{
+}
+
+bool FilterFileBase::readHeader()
+{
+ if (skipHeader_)
+ {
+ return true;
+ }
+
+ if (!BinaryAllClustersFileReader::readHeader(clustersCount_))
+ {
+ return false;
+ }
+
+ if (clustersCount_ == 0)
+ {
+ Header header;
+ if (!BinaryAllClustersFileReader::readHeader(header))
+ {
+ return false;
+ }
+
+ if (!validateHeader(header))
+ {
+ return false;
+ }
+
+ clustersCount_ = header.clustersCount_;
+ }
+
+ return true;
+}
+
+bool FilterFileBase::validateHeader(const FilterFile::Header& header)
+{
+ int errnum = errno;
+ if (header.version_ != getHeaderVersion())
+ {
+ const uint32_t version = header.version_;
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, (boost::format("Corrupted header of filter file '%s': header_version=%d") % getPath().string() % version).str()));
+
+ return false;
+ }
+
+ return true;
+}
+
+FilterFile::FilterFile(const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ bool skipHeader)
+ : FileReaderBase(data,
+ ignoreErrors)
+ , FilterFileBase(data,
+ ignoreErrors,
+ skipHeader)
+{
+ if (!skipHeader)
+ {
+ readHeader();
+ }
+}
+
+std::size_t FilterFile::read(
+ FilterFile::Record *targetBuffer,
+ std::size_t targetSize
+)
+{
+ if (!validateCondition(this->isOpen(), "File is not open")) { return 0; }
+
+ return readBytes(reinterpret_cast<char*>(targetBuffer), targetSize);
+}
+
+
+bool FilterFile::doesFileExist(const boost::filesystem::path& inputDir,
+ bool aggregateTilesFlag,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ size_t& headerSize,
+ boost::filesystem::path& filePath)
+{
+ headerSize = 12;
+ if( aggregateTilesFlag ) {
+ filePath =
+ inputDir
+ / boost::filesystem::path((boost::format("L%03d") % laneNumber).str())
+ / boost::filesystem::path((boost::format("s_%d.filter") % laneNumber).str());
+ } else {
+ boost::filesystem::path fileName((boost::format("s_%d_%d.filter") % laneNumber % tileNumber).str());
+ boost::filesystem::path fileNameZeroPadded((boost::format("s_%d_%04d.filter") % laneNumber % tileNumber).str());
+ boost::filesystem::path laneDir((boost::format("L%03d") % laneNumber).str());
+
+ filePath = inputDir / laneDir / fileName;
+
+ if (!boost::filesystem::exists(filePath))
+ {
+ filePath = inputDir / fileName;
+ headerSize = 4;
+ }
+
+ if (!boost::filesystem::exists(filePath))
+ {
+ filePath = inputDir / laneDir / fileNameZeroPadded;
+ }
+
+ if (!boost::filesystem::exists(filePath))
+ {
+ filePath = inputDir / fileNameZeroPadded;
+ headerSize = 4;
+ }
+ }
+
+ return boost::filesystem::exists(filePath);
+}
+
+
+} // namespace data
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/data/InteropFile.cpp b/src/cxx/lib/data/InteropFile.cpp
new file mode 100644
index 0000000..e0a09a5
--- /dev/null
+++ b/src/cxx/lib/data/InteropFile.cpp
@@ -0,0 +1,117 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file InteropFile.cpp
+ *
+ * \brief Implementation of InterOp file.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <boost/format.hpp>
+
+#include "common/Debug.hh"
+#include "common/Logger.hh"
+#include "common/SystemCompatibility.hh"
+#include "data/InteropFile.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace data {
+
+
+InteropFileState::InteropFileState(const boost::filesystem::path& filePath)
+: path_(filePath)
+{
+}
+
+InteropFileState::~InteropFileState()
+{
+}
+
+void InteropFileState::write(std::ostream &os, const char *sourceBuffer, std::streamsize sourceSize)
+{
+ os.write(sourceBuffer, sourceSize);
+}
+
+
+namespace detail {
+
+
+/// \brief Default InterOp file internal state.
+class DefaultInteropFileState : public InteropFileState
+{
+public:
+ DefaultInteropFileState(const boost::filesystem::path& filePath) : InteropFileState(filePath) { }
+
+private:
+ virtual void write(std::ostream &os, const char *sourceBuffer, std::streamsize sourceSize);
+};
+
+void DefaultInteropFileState::write(std::ostream &os, const char *sourceBuffer, std::streamsize sourceSize)
+{
+ InteropFileState::write(os, sourceBuffer, sourceSize);
+}
+
+
+} // namespace detail
+
+
+InteropFile::InteropFile()
+: fileBuf_(std::ios_base::out | std::ios_base::app)
+{
+}
+
+InteropFile::InteropFileStatePtr InteropFile::open(const boost::filesystem::path &path)
+{
+ io::FileBufWithReopen fileBuf(std::ios_base::out | std::ios_base::trunc);
+ // I don't like the addition of this try block. The entire interop file writing implementaion is
+ // poorly designed. All we need to do is write a number to a single file. This is over kill.
+ try
+ {
+ fileBuf.reopen(path.native().c_str(), io::FileBufWithReopen::FadviseFlags::SEQUENTIAL_ONCE);
+ }
+ catch (common::IoError)
+ {
+ //pass
+ }
+
+ if (!fileBuf.is_open())
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "Unable to open Interop file '" << path.string() << "' for writing" << std::endl;
+ return InteropFileStatePtr(NULL);
+ }
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Created InterOp file '" << path << std::endl;
+
+ return InteropFileStatePtr(new detail::DefaultInteropFileState(path));
+}
+
+boost::filesystem::path InteropFile::getPath(const InteropFileState &fileState) const
+{
+ return fileState.path_;
+}
+
+void InteropFile::write(InteropFileState &fileState, const char *sourceBuffer, std::streamsize sourceSize)
+{
+ fileBuf_.reopen(fileState.path_.native().c_str(), io::FileBufWithReopen::FadviseFlags::SEQUENTIAL_ONCE);
+ BCL2FASTQ_ASSERT_MSG(fileBuf_.is_open(), "Unable to reopen InterOp file '" << fileState.path_ << "' for append");
+ std::ostream os(&fileBuf_);
+
+ fileState.write(os, sourceBuffer, sourceSize);
+}
+
+
+} // namespace data
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/data/PositionsFile.cpp b/src/cxx/lib/data/PositionsFile.cpp
new file mode 100644
index 0000000..22ce611
--- /dev/null
+++ b/src/cxx/lib/data/PositionsFile.cpp
@@ -0,0 +1,679 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file PositionsFile.cpp
+ *
+ * \brief Implementation of positions file.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ * \author Aaron Day
+ */
+
+
+#include "data/PositionsFile.hh"
+
+#include <vector>
+#include <algorithm>
+#include <numeric>
+#include <utility>
+
+#include <boost/format.hpp>
+#include <boost/foreach.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/operations.hpp>
+#include <boost/shared_ptr.hpp>
+#include <boost/iostreams/stream.hpp>
+#include <boost/iostreams/device/array.hpp>
+#include <boost/algorithm/string.hpp>
+
+
+namespace bcl2fastq {
+namespace data {
+
+
+PositionsFile::PositionsFile(const common::RawDataBuffer& data,
+ bool ignoreErrors /*= false*/
+)
+ : FileReaderBase(data,
+ ignoreErrors)
+{
+}
+
+namespace detail {
+
+///////////////////////////////////////////////////////////////////////////////
+// positions file type: LOCS
+///////////////////////////////////////////////////////////////////////////////
+
+/// \brief LOCS file.
+class LocsFile : public PositionsFile, public BinaryAllClustersFileReader
+{
+
+public:
+
+ static const boost::filesystem::path EXTENSION;
+
+ /// \brief Constructor.
+ /// \param data Raw data from the LOCS file.
+ /// \param ignoreErrors Suppress errors opening file and/or reading it.
+ explicit LocsFile(
+ const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ bool skipHeader
+ );
+
+ /// \brief Read records from the file to buffer.
+ /// \param targetBuffer Target buffer to read to.
+ /// \param targetSize Maximum number of records to be read.
+ /// \return Number of records read.
+ virtual std::size_t read(
+ std::vector<data::PositionsFile::Record>& targetBuffer,
+ std::size_t targetSize
+ );
+
+private:
+
+ /// \brief LOCS header type definition.
+#pragma pack(push, 1)
+ struct Header
+ {
+ /// \brief Value of the first constant field.
+ static const uint32_t FIELD1;
+
+ /// \brief Value of the second constant field.
+ static const float FIELD2;
+
+ /// \brief The first constant field.
+ uint32_t field1_;
+
+ /// \brief The second constant field.
+ float field2_;
+
+ /// \brief Clusters count.
+ uint32_t clustersCount_;
+ };
+
+ /// \brief LOCS record type definition
+ struct Record
+ {
+ /// \brief X-coordinate.
+ float x_;
+
+ /// \brief y-coordinate.
+ float y_;
+
+ };
+#pragma pack(pop)
+
+ /// \brief Get the number of bytes in a record
+ /// \return Number of bytes in a record
+ virtual std::size_t getRecordBytes() const { return sizeof(Record); }
+
+ /// \brief Read the header
+ virtual bool readHeader(bool ignoreErrors);
+
+ /// \brief Validate the header. Throw an exception on failure.
+ virtual bool validateHeader(Header& header, bool ignoreErrors);
+
+ /// \brief Gets the file type string used for error messages.
+ /// \return File type string
+ virtual std::string getFileTypeStr() const { return "LOCS"; }
+
+ /// \brief Transform the Record type into a PositionsFile::Record
+ /// \param record Record to transform
+ static PositionsFile::Record transformRecord(const Record &record);
+};
+
+const boost::filesystem::path LocsFile::EXTENSION(".locs");
+
+LocsFile::LocsFile(
+ const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ bool skipHeader
+)
+: FileReaderBase(data,
+ ignoreErrors)
+, PositionsFile(data,
+ ignoreErrors)
+, BinaryAllClustersFileReader(data,
+ ignoreErrors)
+{
+ if (!skipHeader)
+ {
+ readHeader(ignoreErrors);
+ }
+}
+
+bool LocsFile::readHeader(bool ignoreErrors)
+{
+ Header header;
+ if (!BinaryAllClustersFileReader::readHeader(header) ||
+ !validateHeader(header, ignoreErrors))
+ {
+ return false;
+ }
+
+ clustersCount_ = header.clustersCount_;
+
+ return true;
+}
+
+bool LocsFile::validateHeader(LocsFile::Header& header, bool ignoreErrors)
+{
+ int errnum = errno;
+ if (header.field1_ != Header::FIELD1)
+ {
+ const uint32_t field1 = header.field1_;
+ const std::string message =
+ (boost::format("Corrupted header of LOCS file '%s': header_field_1=%d")
+ % this->getPath().string() % field1).str();
+ if (ignoreErrors) {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << message << std::endl;
+ }
+ else {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, message));
+ }
+
+ clustersCount_ = 0;
+ return false;
+ }
+ if (header.field2_ != Header::FIELD2)
+ {
+ const float field2 = header.field2_;
+ const std::string message =
+ (boost::format("Corrupted header of LOCS file '%s': header_field_2=%f")
+ % this->getPath().string() % field2).str();
+
+ if (ignoreErrors) {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << message << std::endl;
+ }
+ else {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, message));
+ }
+
+ clustersCount_ = 0;
+ return false;
+ }
+
+ clustersCount_ = header.clustersCount_;
+
+ return true;
+}
+
+std::size_t LocsFile::read(
+ std::vector<data::PositionsFile::Record>& targetBuffer,
+ std::size_t targetSize
+)
+{
+ targetBuffer.resize(targetSize);
+ std::size_t ret = readBytes(reinterpret_cast<char*>(targetBuffer.data()), targetSize*sizeof(Record));
+
+ const Record * const begin = reinterpret_cast<const Record * const>(targetBuffer.data());
+ const Record * const end = begin + targetSize;
+
+ std::transform(begin, end, targetBuffer.data(), &LocsFile::transformRecord);
+
+ return ret;
+}
+
+const uint32_t LocsFile::Header::FIELD1 = 1;
+
+const float LocsFile::Header::FIELD2 = 1.0;
+
+PositionsFile::Record LocsFile::transformRecord(const Record &record)
+{
+ PositionsFile::Record ret;
+ ret.x_ = static_cast<PositionsFile::Record::ClusterCoordinate>(record.x_ * 10.0 + 1000.5);
+ ret.y_ = static_cast<PositionsFile::Record::ClusterCoordinate>(record.y_ * 10.0 + 1000.5);
+
+ return ret;
+}
+
+
+///////////////////////////////////////////////////////////////////////////////
+// positions file type: CLOCS
+///////////////////////////////////////////////////////////////////////////////
+
+/// \brief CLOCS file.
+class ClocsFile : public PositionsFile, public BinaryAllClustersFileReader
+{
+public:
+
+ static const boost::filesystem::path EXTENSION;
+
+ /// \brief Constructor.
+ /// \param data Raw data from the CLOCS file.
+ /// \param ignoreErrors Suppress errors opening file and/or reading it.
+ ClocsFile(
+ const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ bool skipHeader
+ );
+
+ /// \brief Read records from the file to buffer.
+ /// \param targetBuffer Target buffer to read to.
+ /// \param targetSize Maximum number of records to be read.
+ /// \return Number of records read.
+ virtual std::size_t read(
+ std::vector<data::PositionsFile::Record>& targetBuffer,
+ std::size_t targetSize
+ );
+
+private:
+
+ /// \brief CLOCS header type definition.
+#pragma pack(push, 1)
+ struct Header
+ {
+ public:
+ /// \brief supported version number
+ static const uint8_t VERSION;
+
+ /// \brief Version of the format.
+ uint8_t version_;
+
+ /// \brief Number of tiles
+ uint32_t numTiles_;
+ };
+
+ struct Record
+ {
+ public:
+
+ /// \brief x-offset
+ uint8_t xOffset_;
+
+ /// \brief y-offset
+ uint8_t yOffset_;
+
+ };
+#pragma pack(pop)
+
+private:
+
+ /// \brief Get the number of bytes in a record
+ /// \return Number of bytes in a record
+ virtual std::size_t getRecordBytes() const { return sizeof(Record); }
+
+ /// \brief Validate the header. Throw an exception on failure.
+ virtual bool readHeader(bool ignoreErrors);
+
+ /// \brief Validate the header. Throw an exception on failure.
+ virtual bool validateHeader(Header& header, bool ignoreErrors);
+
+ /// \brief Read the number of clusters in the tile.
+ uint8_t readClusterCount();
+
+ /// \brief Gets the file type string used for error messages.
+ /// \return File type string
+ virtual std::string getFileTypeStr() const { return "CLOCS"; }
+
+ /// \brief Transform the Record type into a PositionsFile::Record
+ /// \param record Record to transform
+ PositionsFile::Record transformRecord(const Record &record);
+
+private:
+
+ /// \brief Number of tiles
+ uint32_t numTiles_;
+
+ uint32_t tileIndex_;
+};
+
+const boost::filesystem::path ClocsFile::EXTENSION(".clocs");
+
+ClocsFile::ClocsFile(
+ const common::RawDataBuffer& data,
+ bool ignoreErrors,
+ bool skipHeader
+)
+: FileReaderBase(data,
+ ignoreErrors)
+, PositionsFile(data,
+ ignoreErrors)
+, BinaryAllClustersFileReader(data,
+ ignoreErrors)
+, numTiles_(0)
+, tileIndex_(0)
+{
+ if (!skipHeader)
+ {
+ readHeader(ignoreErrors);
+ }
+}
+
+bool ClocsFile::readHeader(bool ignoreErrors)
+{
+ Header header;
+ if (!BinaryAllClustersFileReader::readHeader(header) ||
+ !validateHeader(header, ignoreErrors))
+ {
+ return false;
+ }
+
+ numTiles_ = header.numTiles_;
+
+ return true;
+}
+
+bool ClocsFile::validateHeader(ClocsFile::Header& header,
+ bool ignoreErrors)
+{
+ int errnum = errno;
+ if (header.version_ != Header::VERSION) {
+
+ const std::string message =
+ (boost::format("Corrupted header of CLOCS file '%s': header_version=%d")
+ % this->getPath().string() % header.version_).str();
+
+ if (ignoreErrors) {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << message << std::endl;
+ }
+ else {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, message));
+ }
+
+ return false;
+ }
+
+ return true;
+}
+
+uint8_t ClocsFile::readClusterCount()
+{
+ uint8_t clustersCount = 0;
+
+ std::streamsize clustersCountLength =
+ BinaryFileReader::read(reinterpret_cast<char *>(&clustersCount),
+ sizeof(clustersCount));
+
+ if (clustersCountLength != sizeof(clustersCount))
+ {
+ logError(clustersCountLength,
+ sizeof(clustersCount));
+ }
+
+ return clustersCount;
+}
+
+std::size_t ClocsFile::read(
+ std::vector<data::PositionsFile::Record>& targetBuffer,
+ std::size_t targetSize
+)
+{
+ targetBuffer.clear();
+
+ size_t numClustersRead = 0;
+ if (!validateCondition(numTiles_ > 0, "No tiles read from clocs file.")) { return 0; }
+
+ // Skip first 5 bytes, which are the version number and number of bins in the file
+ size_t byteIndex = 5;
+ for (uint32_t binNumber = 0; binNumber < numTiles_; ++binNumber)
+ {
+ if (!validateCondition(data_->size() > byteIndex, "Malformed clocs file.")) { return 0; }
+ unsigned char numClusters = static_cast<unsigned char>(data_->at(byteIndex++));
+
+ if (numClusters == 0)
+ {
+ continue;
+ }
+
+ targetBuffer.resize(targetBuffer.size()+numClusters);
+ if (!validateCondition(data_->size() >= byteIndex + numClusters*2, "Malformed clocs file.")) { return 0; }
+ for (uint32_t i = 0; i < numClusters; ++i)
+ {
+ auto& record = targetBuffer[numClustersRead + i];
+ record.x_ = static_cast<PositionsFile::Record::ClusterCoordinate>( static_cast<uint8_t>((*data_)[byteIndex++]) + (binNumber % 82)*250) + 1000.5;
+ record.y_ = static_cast<PositionsFile::Record::ClusterCoordinate>( static_cast<uint8_t>((*data_)[byteIndex++]) + (binNumber / 82)*250) + 1000.5;
+ }
+
+ numClustersRead += numClusters;
+ }
+
+ return numClustersRead;
+}
+
+
+const uint8_t ClocsFile::Header::VERSION = 0x01;
+
+PositionsFile::Record ClocsFile::transformRecord(const Record &record)
+{
+ PositionsFile::Record ret;
+ ret.x_ = static_cast<PositionsFile::Record::ClusterCoordinate>(record.xOffset_ + (tileIndex_ % 82)*250) + 1000.5;
+ ret.y_ = static_cast<PositionsFile::Record::ClusterCoordinate>(record.yOffset_ + (tileIndex_ / 82)*250) + 1000.5;
+
+ return ret;
+}
+
+
+class PosFile : public PositionsFile, FileReader
+{
+public:
+ static const boost::filesystem::path EXTENSION;
+
+ PosFile(const common::RawDataBuffer& data,
+ bool ignoreErrors);
+
+ /// \brief Read records from file to buffer.
+ /// \param targetBuffer Target buffer to read to.
+ /// \param targetSize Maximum number of records to be read.
+ /// \return Number of records read.
+ virtual std::size_t read(
+ std::vector<data::PositionsFile::Record>& targetBuffer,
+ std::size_t targetSize
+ );
+
+private:
+ /// \brief Gets the file type string used for error messages.
+ /// \return File type string
+ virtual std::string getFileTypeStr() const { return "POS"; }
+};
+
+const boost::filesystem::path PosFile::EXTENSION(".txt");
+
+PosFile::PosFile(const common::RawDataBuffer& data,
+ bool ignoreErrors)
+: FileReaderBase(data,
+ ignoreErrors)
+, PositionsFile(data,
+ ignoreErrors)
+, FileReader(data,
+ ignoreErrors)
+{
+}
+
+std::size_t PosFile::read(
+ std::vector<data::PositionsFile::Record>& targetBuffer,
+ std::size_t targetSize
+)
+{
+ targetBuffer.clear();
+ targetBuffer.reserve(targetSize);
+
+ std::size_t recordsRead = 0;
+ static const std::string errMsgBegin = "Ignoring read failure for POS file '";
+ try
+ {
+ std::string line;
+ while (std::getline(istr_, line) && recordsRead < targetSize)
+ {
+ std::vector<std::string> strs;
+ boost::split(strs, line, boost::is_any_of("\t "));
+
+ if (strs.size() != 2)
+ {
+ std::string errMsg = "Formatting error for POS file: " + this->getPath().string();
+ if (ignoreErrors_)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << errMsg << std::endl;
+ return recordsRead;
+ }
+ else
+ {
+ int errnum = errno;
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum, errMsg));
+ }
+ }
+
+ PositionsFile::Record record;
+ record.x_ = boost::lexical_cast<PositionsFile::Record::ClusterCoordinate>(strs[0]);
+ record.y_ = boost::lexical_cast<PositionsFile::Record::ClusterCoordinate>(strs[1]);
+ targetBuffer.push_back(record);
+
+ ++recordsRead;
+ }
+ }
+ CATCH_AND_IGNORE(common::IoError, errMsgBegin)
+ CATCH_AND_IGNORE(std::ios_base::failure, errMsgBegin)
+ CATCH_AND_IGNORE_ALL(errMsgBegin)
+
+ return recordsRead;
+}
+
+} // namespace detail
+
+
+bool PositionsFileFactory::doesFileExist(
+ const boost::filesystem::path& intensitiesDir,
+ bool aggregateTilesFlag,
+ bool isPatternedFlowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ size_t& headerSize,
+ boost::filesystem::path& positionsFilePath
+)
+{
+ if (binaryFilePathExists(intensitiesDir,
+ aggregateTilesFlag,
+ isPatternedFlowcell,
+ laneNumber,
+ tileNumber,
+ detail::LocsFile::EXTENSION,
+ positionsFilePath))
+ {
+ headerSize = 12;
+ return true;
+ }
+
+ if (binaryFilePathExists(intensitiesDir,
+ aggregateTilesFlag,
+ isPatternedFlowcell,
+ laneNumber,
+ tileNumber,
+ detail::ClocsFile::EXTENSION,
+ positionsFilePath))
+ {
+ headerSize = 5;
+ return true;
+ }
+
+ if (posFilePathExists(intensitiesDir,
+ laneNumber,
+ tileNumber,
+ positionsFilePath))
+ {
+ headerSize = 0;
+ return true;
+ }
+
+ return false;
+}
+
+bool PositionsFileFactory::binaryFilePathExists(
+ const boost::filesystem::path& intensitiesDir,
+ bool aggregateTilesFlag,
+ bool isPatternedFlowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ const boost::filesystem::path& fileExtension,
+ boost::filesystem::path& positionsFilePath
+)
+{
+ if( isPatternedFlowcell )
+ {
+ positionsFilePath = boost::filesystem::path(
+ intensitiesDir
+ /
+ boost::filesystem::path("s" + fileExtension.string())
+ );
+ } else if( aggregateTilesFlag ) {
+ positionsFilePath = boost::filesystem::path(
+ intensitiesDir
+ /
+ boost::filesystem::path((boost::format("L%03d") % laneNumber).str())
+ /
+ boost::filesystem::path((boost::format("s_%d" + fileExtension.string()) % laneNumber).str())
+ );
+ } else {
+ positionsFilePath = boost::filesystem::path(
+ intensitiesDir
+ /
+ boost::filesystem::path((boost::format("L%03d") % laneNumber).str())
+ /
+ boost::filesystem::path((boost::format("s_%d_%d" + fileExtension.string()) % laneNumber % tileNumber).str())
+ );
+ }
+
+ return boost::filesystem::exists(positionsFilePath);
+}
+
+bool PositionsFileFactory::posFilePathExists(
+ const boost::filesystem::path& intensitiesDir,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ boost::filesystem::path& positionsFilePath
+)
+{
+ positionsFilePath = boost::filesystem::path(
+ intensitiesDir
+ /
+ boost::filesystem::path((boost::format("s_%d_%03d_pos.txt") % laneNumber % tileNumber).str())
+ );
+
+ return boost::filesystem::exists(positionsFilePath);
+}
+
+std::streamsize PositionsFileFactory::read(const common::RawDataBuffer& inputData,
+ bool ignoreErrors,
+ bool skipHeader,
+ std::vector<data::PositionsFile::Record>& targetBuffer,
+ std::size_t targetSize)
+{
+ auto extension = inputData.path_.extension();
+ if (extension == detail::LocsFile::EXTENSION)
+ {
+ detail::LocsFile posFile(inputData,
+ ignoreErrors,
+ skipHeader);
+
+ return posFile.read(targetBuffer, targetSize > 0 ? targetSize : posFile.getClustersCount());
+
+ }
+ else if (extension == detail::ClocsFile::EXTENSION)
+ {
+ detail::ClocsFile posFile(inputData,
+ ignoreErrors,
+ skipHeader);
+
+ return posFile.read(targetBuffer, targetSize);
+ }
+ else if (extension == detail::PosFile::EXTENSION)
+ {
+ detail::PosFile posFile(inputData,
+ ignoreErrors);
+
+ posFile.read(targetBuffer, targetSize);
+ }
+
+ return 0;
+}
+
+} // namespace data
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/data/TileBclFileReader.cpp b/src/cxx/lib/data/TileBclFileReader.cpp
new file mode 100644
index 0000000..1805f34
--- /dev/null
+++ b/src/cxx/lib/data/TileBclFileReader.cpp
@@ -0,0 +1,78 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA^M
+ * and certain third party copyright/licenses, and any user of this^M
+ * source file is bound by the terms therein.^M
+ *
+ * \file TileBclFileReader.cpp
+ *
+ * \brief Implementation of Bcl file reader for a single tile.
+ *
+ * \author Aaron Day
+ */
+
+#include "data/TileBclFileReader.hh"
+
+namespace bcl2fastq {
+namespace data {
+
+TileBclFileReader::TileBclFileReader(const::boost::filesystem::path& inputDir,
+ const layout::LaneInfo& laneInfo,
+ common::CycleNumber cycleNumber,
+ size_t cycleIndex,
+ bool ignoreMissingBcls,
+ std::shared_ptr<io::FileReaderBase> bclFile)
+: BclFileReaderT<io::FileReaderBase>(inputDir,
+ laneInfo,
+ cycleNumber,
+ cycleIndex,
+ ignoreMissingBcls,
+ bclFile)
+{
+}
+
+bool TileBclFileReader::read(RawBclBufferGroup& outputBuffer)
+{
+ static int sequentialId = 0; ++sequentialId;
+ for (auto& outputBufferForTile : outputBuffer)
+ {
+ common::RawDataBuffer &outputBuffer = outputBufferForTile.cycleData_.at(cycleIndex_).bcls_;
+
+ boost::filesystem::path bclPath;
+ if (!data::BclFile::getAndVerifyFileName(inputDir_,
+ laneInfo_.getNumber(),
+ outputBufferForTile.tileInfo_->getNumber(),
+ cycleNumber_,
+ ignoreMissingBcls_,
+ bclPath))
+ {
+ if (ignoreMissingBcls_) {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "Unable to read tile file (" <<
+ outputBufferForTile.tileInfo_->getNumber() << "): " << bclPath.string() << std::endl;
+ outputBuffer.resize(0);
+ continue;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(ENOENT, (boost::format("Unable to open tile file '%s' for reading") % bclPath.string()).str()));
+ }
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "TileBclFileReader::read " << sequentialId << " " << bclPath.string() << std::endl;
+
+ io::UnprocessedFile bclFile(bclPath,
+ ignoreMissingBcls_);
+
+ bool status = bclFile.readEntireFile(outputBuffer);
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "... done TileBclFileReader::read " << sequentialId << std::endl;
+ return status;
+ }
+
+ return true;
+}
+
+} // namespace data
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/io/CMakeLists.txt b/src/cxx/lib/io/CMakeLists.txt
new file mode 100644
index 0000000..8aa7b58
--- /dev/null
+++ b/src/cxx/lib/io/CMakeLists.txt
@@ -0,0 +1,23 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the cxx/lib/io subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CXX_LIBRARY_CMAKE})
+
+
diff --git a/src/cxx/lib/io/GzipCompressor.cpp b/src/cxx/lib/io/GzipCompressor.cpp
new file mode 100644
index 0000000..20c9b90
--- /dev/null
+++ b/src/cxx/lib/io/GzipCompressor.cpp
@@ -0,0 +1,157 @@
+/**
+* BCL to FASTQ file converter
+* Copyright (c) 2007-2017 Illumina, Inc.
+*
+* This software is covered by the accompanying EULA
+* and certain third party copyright/licenses, and any user of this
+* source file is bound by the terms therein.
+*
+* \file GzipCompressor.cpp
+*
+* \brief Implementation of gzip compressor.
+*
+* \author Aaron Day
+*/
+
+#include "io/GzipCompressor.hh"
+
+#include "common/Debug.hh"
+#include "common/Exceptions.hh"
+
+#include <algorithm>
+
+namespace bcl2fastq
+{
+namespace io
+{
+
+GzipCompressor::GzipCompressor(std::vector<char>& outputBuffer,
+ bool useBgzf,
+ const bios::gzip_params& gzipParams)
+: outputBuffer_(outputBuffer)
+, internalBuffer_()
+, sink_(outputBuffer_)
+, compressor_(gzipParams)
+, uncompressed_in_(0)
+, currentBlockBegin_(0)
+, useBgzf_(useBgzf)
+{
+ internalBuffer_.reserve(bgzf_buffer_size_);
+ outputBuffer_.clear();
+}
+
+GzipCompressor::~GzipCompressor()
+{
+ flush();
+}
+
+void GzipCompressor::prepareHeader()
+{
+ // save room for XFIELD
+ outputBuffer_.insert(outputBuffer_.end(), sizeof(Header::XFIELD), 0);
+}
+
+std::streamsize GzipCompressor::write(const char* s, std::streamsize src_size)
+{
+ std::streamsize buffer_left(max_uncompressed_per_block_ - uncompressed_in_);
+
+ // Jump through a hoop for the basespace uploader bug
+ if (src_size > buffer_left)
+ {
+ buffer_left = 0;
+ }
+
+ const std::streamsize to_buffer(std::min<std::streamsize>(buffer_left, src_size));
+
+ if (uncompressed_in_ == 0 && useBgzf_)
+ {
+ // Start of a new block, prepare the header.
+ prepareHeader();
+ }
+
+ if (to_buffer)
+ {
+ internalBuffer_.insert(internalBuffer_.end(), s, s+to_buffer);
+ uncompressed_in_ += to_buffer;
+ }
+
+ if (src_size != to_buffer)
+ {
+ flush();
+ const std::streamsize more = src_size - to_buffer;
+ const std::streamsize written = write(s + to_buffer, more);
+ if (more != written)
+ {
+ return to_buffer + written;
+ }
+ }
+
+ return src_size;
+}
+
+void GzipCompressor::flush()
+{
+ if (uncompressed_in_)
+ {
+ compressor_.write(sink_, internalBuffer_.data(), internalBuffer_.size());
+ internalBuffer_.clear();
+ compressor_.close(sink_, BOOST_IOS::out);
+ if (useBgzf_)
+ {
+ rewriteHeader();
+ currentBlockBegin_ = outputBuffer_.size();
+ }
+ }
+
+ uncompressed_in_ = 0;
+}
+
+unsigned GzipCompressor::getBlockSize(char* buffer, bool failQuietly)
+{
+ Header *h = reinterpret_cast<Header*>(buffer);
+
+ if (h->xfield.SI1 != static_cast<unsigned char>(66) || h->xfield.SI2 != static_cast<unsigned char>(67))
+ {
+ if (failQuietly)
+ return 0;
+
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << static_cast<unsigned short>(h->xfield.SI1) << ", " << static_cast<unsigned short>(h->xfield.SI2) << ", " << static_cast<unsigned short>(h->xfield.XLEN[0]) << ", " << static_cast<unsigned short>(h->xfield.XLEN[1]) << ", " << static_cast<unsigned short>(h->XFL) << ", " << static_cast<unsigned short>(h->OS) << std::endl;
+ BOOST_THROW_EXCEPTION(common::IoError(ENOENT, "Corrupt GZIP block"));
+ }
+
+ return h->xfield.getBSIZE() + 1;
+}
+
+void GzipCompressor::rewriteHeader()
+{
+ memmove(&outputBuffer_[currentBlockBegin_], &outputBuffer_[currentBlockBegin_+sizeof(Header::XFIELD)], sizeof(Header) - sizeof(Header::XFIELD));
+ Header *h(reinterpret_cast<Header*>(&outputBuffer_[currentBlockBegin_]));
+ h->xfield = Header::XFIELD(outputBuffer_.size() - currentBlockBegin_ - 1); // why -1???
+ h->FLG |= 0x04; // tell gzip that XLEN is in effect now.
+}
+
+GzipCompressor::Header::XFIELD::XFIELD(size_t compressedSize)
+: SI1(66)
+, SI2(67)
+{
+ XLEN[0] = sizeof(XFIELD) - sizeof(short);
+ XLEN[1] = (sizeof(XFIELD) - sizeof(short)) >> 8;
+
+ SLEN[0] = SUBFIELD_LENGTH;
+ SLEN[1] = 0;
+
+ static const size_t maxSize = ~((size_t)0x0) >> ((8-SUBFIELD_LENGTH)*8);
+ BCL2FASTQ_ASSERT_MSG( compressedSize <= maxSize,
+ "Block size exceeds " << maxSize << " bytes when compressed: Got '" << compressedSize << "' bytes");
+
+ static const size_t bitMask = 0x00000000000000FF;
+ for (int i = 0; i < SUBFIELD_LENGTH; ++i)
+ {
+ BSIZE[i] = (compressedSize & (bitMask << (i*8))) >> (i*8);
+ }
+}
+
+
+} // namespace io
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/io/GzipDecompressor.cpp b/src/cxx/lib/io/GzipDecompressor.cpp
new file mode 100644
index 0000000..88dd94a
--- /dev/null
+++ b/src/cxx/lib/io/GzipDecompressor.cpp
@@ -0,0 +1,209 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file GzipDecompressor.cpp
+ *
+ * \brief Implementation of GZip decompression filter.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#include <boost/iostreams/read.hpp>
+#include <boost/lexical_cast.hpp>
+
+#include "common/Debug.hh"
+#include "common/Logger.hh"
+#include "io/GzipDecompressor.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace io {
+
+
+inline std::ostream &operator <<(std::ostream &os, const z_stream_serialization &zs)
+{
+ return os << "z_stream( " <<
+ " next_in: " << (void*)zs.next_in <<
+ ", avail_in: " << zs.avail_in <<
+ ", total_in: " << zs.total_in <<
+ ", next_out: " << (void*)zs.next_out <<
+ ", avail_out: " << zs.avail_out <<
+ ", total_out: " << zs.total_out <<
+ ", msg: '" << (zs.msg ? (const char*)zs.msg : (const char*)("null"))<<
+ "', state: " << (void*)zs.state <<
+ ", zalloc: " << zs.zalloc <<
+ ", zfree: " << zs.zfree <<
+ ", opaque: " << zs.opaque <<
+ ", data_type: " << zs.data_type <<
+ ", adler: " << zs.adler <<
+ ", reserved: " << zs.reserved <<
+ " )";
+}
+
+
+ZlibError::ZlibError(int errorNumber, z_stream &zstream, const char *msg)
+: Exception(errorNumber, (zstream.msg ? zstream.msg : msg ?
+ std::string(msg) + " unknown error " + boost::lexical_cast<std::string>(errorNumber) :
+ "unknown error " + boost::lexical_cast<std::string>(errorNumber)) + std::string("\n") +
+ boost::lexical_cast<std::string>(z_stream_serialization(zstream)))
+, std::runtime_error((zstream.msg ? zstream.msg : msg ?
+ std::string(msg) + " unknown error " + boost::lexical_cast<std::string>(errorNumber) :
+ "unknown error " + boost::lexical_cast<std::string>(errorNumber)) + std::string("\n") +
+ boost::lexical_cast<std::string>(z_stream_serialization(zstream)))
+{
+}
+
+ZlibError::ZlibError(const ZlibError &that)
+: Exception(that)
+, std::runtime_error(that)
+{
+}
+
+const char * ZlibError::what() const throw()
+{
+ return this->getMessage().c_str();
+}
+
+
+GzipDecompressor::GzipDecompressor(buffer_type::size_type bufferSize)
+: zstream_()
+, memPool_()
+, pendingBytes_(0)
+, buffer_(bufferSize, '\0')
+{
+ this->clear();
+}
+
+GzipDecompressor::GzipDecompressor(const GzipDecompressor &that)
+: zstream_()
+, memPool_()
+, pendingBytes_(that.pendingBytes_)
+, buffer_(that.buffer_)
+{
+// zstream_.state = nullptr;
+ this->reset();
+}
+
+GzipDecompressor & GzipDecompressor::operator=(const GzipDecompressor &rhs)
+{
+ if (this != &rhs)
+ {
+ this->reset();
+ pendingBytes_ = rhs.pendingBytes_;
+ buffer_ = rhs.buffer_;
+ }
+ return *this;
+}
+
+GzipDecompressor::~GzipDecompressor()
+{
+ if (zstream_.state)
+ {
+ inflateEnd(&zstream_);
+ }
+}
+
+void GzipDecompressor::init()
+{
+ zstream_.zalloc = zalloc;
+ zstream_.zfree = zfree;
+ zstream_.opaque = &memPool_;
+
+ int ret = inflateInit2(&zstream_, 16+MAX_WBITS);
+ if (ret != Z_OK)
+ {
+ BOOST_THROW_EXCEPTION(ZlibError(ret, zstream_));
+ }
+}
+
+
+void GzipDecompressor::clear()
+{
+ if (zstream_.state)
+ {
+ inflateEnd(&zstream_);
+ }
+ memset(&zstream_, 0, sizeof(zstream_));
+ this->init();
+}
+
+void GzipDecompressor::reset()
+{
+ if (zstream_.state)
+ {
+ inflateEnd(&zstream_);
+ }
+ this->init();
+}
+
+void GzipDecompressor::flush()
+{
+ zstream_.avail_in = 0;
+ zstream_.next_in = Z_NULL;
+ pendingBytes_ = 0;
+}
+
+void GzipDecompressor::processPendingBytes(bool endOfData)
+{
+ if (pendingBytes_ <= 0)
+ {
+ return;
+ }
+ zstream_.next_in = reinterpret_cast<Bytef *>(&buffer_.begin()[0]);
+ zstream_.avail_in = pendingBytes_;
+
+ BCL2FASTQ_ASSERT_MSG( PageSize >= zstream_.avail_in,
+ "There are more bytes to process than a single page can take" );
+ int err = inflate(&zstream_, Z_SYNC_FLUSH);
+ if (err != Z_OK && err != Z_STREAM_END)
+ {
+ if (endOfData)
+ {
+ BOOST_THROW_EXCEPTION(ZlibError(err, zstream_, "Premature end of compressed stream reached. "));
+ } else {
+ BOOST_THROW_EXCEPTION(ZlibError(err, zstream_));
+ }
+ }
+ const std::streamsize compressedWritten = pendingBytes_ - zstream_.avail_in;
+ std::copy(
+ buffer_.begin() + compressedWritten,
+ buffer_.begin() + pendingBytes_,
+ buffer_.begin()
+ );
+ pendingBytes_ = zstream_.avail_in;
+}
+
+voidpf GzipDecompressor::zalloc OF((voidpf opaque, uInt items, uInt size))
+{
+ voidpf ptr = static_cast<voidpf>(
+ reinterpret_cast<mempool_type *>(opaque)->allocate(items*size)
+ );
+
+ if (ptr == NULL)
+ {
+ return Z_NULL;
+ }
+ return ptr;
+}
+
+void GzipDecompressor::zfree OF((voidpf opaque, voidpf address))
+{
+ reinterpret_cast<mempool_type *>(opaque)->deallocate(address);
+}
+
+
+} // namespace io
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/io/SyncFile.cpp b/src/cxx/lib/io/SyncFile.cpp
new file mode 100644
index 0000000..29be67a
--- /dev/null
+++ b/src/cxx/lib/io/SyncFile.cpp
@@ -0,0 +1,328 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file SyncFile.cpp
+ *
+ * \brief Implementation of Synchronized file.
+ *
+ * \author Aaron Day
+ */
+
+#include "io/SyncFile.hh"
+
+namespace bcl2fastq {
+namespace io {
+
+FileReaderBase::FileReaderBase()
+: path_()
+, ignoreErrors_(false)
+{
+}
+
+FileReaderBase::FileReaderBase(const boost::filesystem::path& path,
+ bool ignoreErrors)
+: path_(path)
+, ignoreErrors_(ignoreErrors)
+{
+}
+
+UnprocessedFile::UnprocessedFile(std::ios_base::openmode openFlags)
+: FileReaderBase()
+, fileBuf_(openFlags)
+, fileSize_(0)
+{
+}
+
+UnprocessedFile::UnprocessedFile(const boost::filesystem::path& path,
+ bool ignoreErrors,
+ std::ios_base::openmode openFlags /*=std::ios_base::in | std::ios_base::binary*/)
+: FileReaderBase(path, ignoreErrors)
+, fileBuf_(openFlags)
+, fileSize_(0)
+{
+ openFileBuf();
+}
+
+UnprocessedFile::~UnprocessedFile()
+{
+}
+
+void UnprocessedFile::openFile(const boost::filesystem::path& path,
+ bool ignoreErrors,
+ size_t)
+{
+ openFileImpl(path, ignoreErrors);
+}
+
+void UnprocessedFile::openFileImpl(const boost::filesystem::path& path,
+ bool ignoreErrors)
+{
+ path_ = path;
+ ignoreErrors_ = ignoreErrors;
+
+ openFileBuf();
+}
+
+bool UnprocessedFile::openFileBuf()
+{
+ try
+ {
+ errno = 0;
+
+ fileBuf_.reopen(getPath(), io::FileBufWithReopen::FadviseFlags::SEQUENTIAL_ONCE);
+ int errnum = errno;
+ if (!fileBuf_.is_open())
+ {
+ fileBuf_.reset();
+ boost::format errFormat(boost::format("Unable to open %s file '%s' for reading")
+ % getFileTypeStr()
+ % getPath().string());
+ if (ignoreErrors_)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING)
+ << errFormat
+ << ":" << std::strerror(errnum) << " (" << errnum << ")" << std::endl;
+
+ return false;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errnum, errFormat.str()));
+ }
+ }
+ else
+ {
+ fileSize_ = fileBuf_.pubseekoff(0, std::ios_base::end);
+ fileBuf_.pubseekoff(0, std::ios_base::beg);
+ }
+ }
+ CATCH_AND_IGNORE(common::IoError, "Ignoring read failure for " + getFileTypeStr() + " file '")
+ CATCH_AND_IGNORE(std::ios_base::failure, "Ignoring read failure for " + getFileTypeStr() + " file '")
+ CATCH_AND_IGNORE_ALL("Ignoring read failure for " + getFileTypeStr() + " file '")
+
+ return true;
+}
+
+bool UnprocessedFile::readBytes(std::vector<char>& buffer,
+ uint32_t bytes)
+{
+ BCL2FASTQ_ASSERT_MSG(buffer.size() == bytes, "Incorrect buffer size");
+
+ auto bytesRead = io::read(fileBuf_, buffer.data(), bytes);
+ if (bytesRead != bytes)
+ {
+ logError(bytesRead,
+ bytes);
+
+ buffer.resize(bytesRead > 0 ? bytesRead : 0);
+
+ return false;
+ }
+
+ return true;
+}
+
+bool UnprocessedFile::readEntireFile(common::RawDataBuffer& buffer)
+{
+ bool ret = false;
+
+ buffer.path_ = path_;
+
+ auto filePos = fileBuf_.pubseekoff(0, std::ios_base::cur);
+
+ common::resizeAndRealloc(buffer, fileSize_ - filePos);
+ ret = readBytes(buffer, buffer.size());
+ if (!ret)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "Returning false!!!" << std::endl;
+ }
+
+ return ret;
+}
+
+void UnprocessedFile::logError(std::streamsize bytesRead,
+ std::streamsize bytesExpected) const
+{
+ int errnum = errno;
+
+ boost::format errFormat(boost::format(
+ getFileTypeStr() + " file '%s' truncated: bytes_read=%d bytes_expected=%d") %
+ this->getPath().string() % bytesRead % bytesExpected);
+
+ if (ignoreErrors_)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING)
+ << errFormat
+ << ":" << std::strerror(errnum) << " (" << errnum << ")" << std::endl;
+ }
+ else
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError(errnum,
+ errFormat.str()));
+ }
+}
+
+SyncFile::SyncFile(std::ios_base::openmode openFlags)
+: UnprocessedFile(openFlags)
+, mut_()
+, cv_()
+, currentTile_(0)
+, buffer_()
+{
+}
+
+SyncFile::SyncFile(const boost::filesystem::path& path,
+ bool ignoreErrors,
+ std::ios_base::openmode openFlags)
+: UnprocessedFile(path,
+ ignoreErrors,
+ openFlags)
+, mut_()
+, cv_()
+, currentTile_(0)
+, buffer_()
+{
+}
+
+SyncFile::~SyncFile()
+{
+}
+
+bool SyncFile::read(char* buffer,
+ std::streamsize bytes)
+{
+ auto bytesRead = io::read(fileBuf_, buffer, bytes);
+ if (bytesRead != bytes)
+ {
+ logError(bytesRead,
+ bytes);
+
+ return false;
+ }
+
+ return true;
+}
+
+bool SyncFile::read(std::vector<char>& buffer,
+ std::streamsize bytes)
+{
+ common::resizeAndRealloc(buffer, bytes);
+ return readBytes(buffer, bytes);
+}
+
+bool SyncFile::seek(std::streamsize offset,
+ std::ios_base::seekdir way /*= std::ios_base::cur*/)
+{
+ return fileBuf_.pubseekoff(offset, way);
+}
+
+bool SyncFile::appendBgzfBlock(std::vector<char> &gzipData)
+{
+ static const unsigned headerSize = GzipCompressor::getHeaderSize();
+ const size_t dataOffset = gzipData.size();
+
+ std::streamsize currentPos = fileBuf_.pubseekoff(0, std::ios_base::cur);
+ if (1 + currentPos + headerSize >= static_cast<int64_t>(fileSize_)) {
+ return true;
+ }
+
+ unsigned bytesToRead = headerSize;
+ common::resizeAndRealloc(gzipData, gzipData.size()+bytesToRead);
+
+ auto bytesRead = io::read(fileBuf_, gzipData.data()+dataOffset, bytesToRead);
+ if (bytesRead != bytesToRead) {
+ common::resizeAndRealloc(gzipData, dataOffset);
+ return false;
+ }
+
+ unsigned gzipBlockSize = 0;
+ try {
+ gzipBlockSize = io::GzipCompressor::getBlockSize(&gzipData[dataOffset], true);
+ }
+ catch (...) {
+ common::resizeAndRealloc(gzipData, dataOffset);
+ return false;
+ }
+
+ bytesToRead = gzipBlockSize - bytesToRead;
+ common::resizeAndRealloc(gzipData, gzipData.size()+bytesToRead);
+
+ bytesRead = io::read(fileBuf_, gzipData.data()+dataOffset+headerSize, bytesToRead);
+ if (bytesRead != bytesToRead) {
+ common::resizeAndRealloc(gzipData, dataOffset);
+ return false;
+ }
+
+ return true;
+}
+
+bool SyncFile::readGzipBlocks(std::vector<char>& gzipData,
+ std::streamsize startPos,
+ std::streamsize nextStartPos)
+{
+ BCL2FASTQ_ASSERT_MSG(nextStartPos == 0 || startPos <= nextStartPos, "Invalid aggregated read block");
+
+ fileBuf_.pubseekoff(startPos, std::ios_base::beg);
+
+ gzipData.clear();
+ const bool readToEOF = (nextStartPos == 0);
+ nextStartPos = (nextStartPos <= 0) ? fileSize_ : nextStartPos;
+ std::streamsize bytes = nextStartPos - startPos;
+
+ try {
+ common::resizeAndRealloc(gzipData, bytes);
+ }
+ catch (std::bad_alloc e) {
+ BCL2FASTQ_LOG(common::LogLevel::FATAL) << "Current tile allocation exceed memory available. " << bytes << std::endl;
+ return false;
+ }
+
+ if (bytes > 0) {
+
+ auto bytesRead = io::read(fileBuf_, gzipData.data(), bytes);
+
+ if (bytesRead != bytes)
+ {
+ logError(bytesRead, bytes);
+
+ gzipData.resize(bytesRead > 0 ? bytesRead : 0);
+
+ return false;
+ }
+
+ // Sanity check: gzipData must start with a gzip header.
+ if (gzipData.size() > 0)
+ {
+ try
+ {
+ /*unsigned gzipBlockSize =*/ io::GzipCompressor::getBlockSize(&gzipData[0]);
+ }
+ catch (common::IoError e)
+ {
+ if (ignoreErrors_)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "Identified corrupt GZIP data - " << getPath() << std::endl;
+ return false;
+ }
+
+ throw;
+ }
+ }
+ }
+
+ // Read one more gzip block
+ if (!readToEOF) {
+ appendBgzfBlock(gzipData);
+ }
+
+ return true;
+}
+
+} // namespace io
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/io/Utility.cpp b/src/cxx/lib/io/Utility.cpp
new file mode 100644
index 0000000..8014dec
--- /dev/null
+++ b/src/cxx/lib/io/Utility.cpp
@@ -0,0 +1,95 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Utility.cpp
+ *
+ * \brief Implementation of various I/O related utilities.
+ *
+ * \author Roman Petorvski
+ * \author Marek Balint
+ */
+
+
+#include <vector>
+
+#include <boost/assign/list_of.hpp>
+
+#include "io/Utility.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace io {
+
+
+namespace detail {
+
+
+static const std::vector<const char *> iosBaseToStdioOpenModesTranslationTable = boost::assign::list_of<const char*>
+ // inspired by /usr/include/c++/4.3.2/fstream
+ // | ios_base Flag combination |
+ // |binary in out trunc app |
+ (0 ) // | |
+ ("a" ) // | + |
+ (0 ) // | + |
+ (0 ) // | + + |
+ ("w" ) // | + |
+ ("a" ) // | + + |
+ ("w" ) // | + + |
+ (0 ) // | + + + |
+ ("r" ) // | + |
+ ("a+" ) // | + + |
+ (0 ) // | + + |
+ (0 ) // | + + + |
+ ("r+" ) // | + + |
+ ("a+" ) // | + + + |
+ ("w+" ) // | + + + |
+ (0 ) // | + + + + |
+ (0 ) // | + |
+ ("ab" ) // | + + |
+ (0 ) // | + + |
+ (0 ) // | + + + |
+ ("wb" ) // | + + |
+ ("ab" ) // | + + + |
+ ("wb" ) // | + + + |
+ (0 ) // | + + + + |
+ ("rb" ) // | + + |
+ ("a+b" ) // | + + + |
+ (0 ) // | + + + |
+ (0 ) // | + + + + |
+ ("r+b" ) // | + + + |
+ ("a+b" ) // | + + + + |
+ ("w+b" ) // | + + + + |
+ (0 ) // | + + + + + |
+;
+
+
+} // namespace detail
+
+
+const char * iosFlagsToStdioMode(std::ios_base::openmode mode)
+{
+ const unsigned openModeIndex =
+ !!(mode & std::ios_base::binary) << 4 |
+ !!(mode & std::ios_base::in ) << 3 |
+ !!(mode & std::ios_base::out ) << 2 |
+ !!(mode & std::ios_base::trunc ) << 1 |
+ !!(mode & std::ios_base::app ) << 0
+ ;
+
+ return detail::iosBaseToStdioOpenModesTranslationTable.at(openModeIndex);
+}
+
+
+} // namespace io
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/io/Xml.cpp b/src/cxx/lib/io/Xml.cpp
new file mode 100644
index 0000000..8069d25
--- /dev/null
+++ b/src/cxx/lib/io/Xml.cpp
@@ -0,0 +1,220 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file XmlParser.cpp
+ *
+ * \brief Basic XML parser.
+ *
+ * \author Roman Petrovski
+ * \author Marek Balint
+ */
+
+
+#include <vector>
+#include <string>
+#include <algorithm>
+#include <iterator>
+
+#include <boost/filesystem/fstream.hpp>
+#include <boost/format.hpp>
+#include <boost/foreach.hpp>
+#include <boost/property_tree/xml_parser.hpp>
+
+#include "io/Xml.hh"
+#include "common/Exceptions.hh"
+
+#include "common/Logger.hh"
+
+namespace bcl2fastq {
+namespace io {
+
+
+
+using namespace boost::property_tree;
+static const std::string indexed("<indexed>");
+
+
+/**
+ * @brief Rearranges property tree according to a few basic schema customization rules.
+ *
+ * The original property tree would produce an xml
+ * where each Read, Lane and Tile element has a number of child elements named after
+ * the number of the read, lane of tile respectively. For example:
+ *
+ * Tree:
+ * Summary.Read.1.Lane.4.Tile.5.value.Yield = 123
+ * .YieldQ30 = 12
+ * .6.value.Yield = 123
+ * .YieldQ30 = 12
+ *
+ * Xml:
+ * <Summary>
+ * <Read>
+ * <1>
+ * <Lane>
+ * <4>
+ * <Tile>
+ * <5>
+ * ...
+ * This is inconvenient and against the rest of the iSAAC metadata xml schemas.
+ * Before the tree is written out into the xml, all index nodes are collapsed into
+ * xml attribute nodes so that the resulting xml looks like:
+ *
+ * Tree:
+ * Summary.<indexed>Read.<number>1.<indexed>Lane.<number>4.<indexed>Tile.<number>5.value.Yield = 123
+ * .YieldQ30 = 12
+ * .<number>6.value.Yield = 123
+ * .YieldQ30 = 12
+ *
+ * Xml:
+ * <Summary>
+ * <Read index="1">
+ * <Lane number="4">
+ * <Tile number="5">
+ * <Yield>123</Yield>
+ * <YieldQ30>12</YieldQ30>
+ * </Tile>
+ * <Tile number="6">
+ * <Yield>123</Yield>
+ * <YieldQ30>12</YieldQ30>
+ * ...
+ */
+void unindex(const ptree::value_type &nameElementKv, ptree &resultTree)
+{
+ if (indexed == nameElementKv.first.substr(0, indexed.length()))
+ {
+ BOOST_FOREACH(const ptree::value_type &indexElementKv, nameElementKv.second)
+ {
+ BOOST_ASSERT('<' == indexElementKv.first[0]);
+ size_t attrNameEndPos = indexElementKv.first.find('>', 1);
+ BOOST_ASSERT(std::string::npos != attrNameEndPos);
+ ptree &insertedChildTree = resultTree.add_child(nameElementKv.first.substr(indexed.length()), ptree());
+
+ std::for_each(indexElementKv.second.begin(), indexElementKv.second.end(),
+ boost::bind(unindex, _1, boost::ref(insertedChildTree)));
+
+ if (insertedChildTree.empty())
+ {
+ // if child has no child elements, use its value as content
+ insertedChildTree.put_value(indexElementKv.second.get_value<std::string>());
+ }
+
+ insertedChildTree.add("<xmlattr>." + indexElementKv.first.substr(1, attrNameEndPos -1),
+ indexElementKv.first.substr(attrNameEndPos+1));
+ }
+ }
+ else
+ {
+ ptree &insertedChildTree = resultTree.add_child(nameElementKv.first, ptree());
+ std::for_each(nameElementKv.second.begin(), nameElementKv.second.end(),
+ boost::bind(unindex, _1, boost::ref(insertedChildTree)));
+ if (insertedChildTree.empty())
+ {
+ //unindexing did not find any indexed nodes. just add it as is
+ insertedChildTree.put_value(nameElementKv.second.get_value<std::string>());
+ }
+ else
+ {
+ //unindexing inserted the modified tree.
+ }
+ }
+}
+
+bool longestPathFirst(const std::string &left, const std::string &right)
+{
+ return std::count(left.begin(), left.end(), '.') > std::count(right.begin(), right.end(), '.');
+}
+
+void index(std::vector<std::string > indexNodes, ptree &tree)
+{
+ // indexing of longer paths should happen first for the shorter paths to stay valid until they get indexed
+ std::sort(indexNodes.begin(), indexNodes.end(), longestPathFirst);
+
+ BOOST_FOREACH(const std::string &idxAttrPath, indexNodes)
+ {
+ size_t attrNameDotPos(idxAttrPath.rfind('.'));
+ BOOST_ASSERT(std::string::npos != attrNameDotPos && "Could not parse index attribute name");
+ const std::string indexedNodeKey(idxAttrPath.substr(0, attrNameDotPos));
+ const std::string indexAttributeName(idxAttrPath.substr(attrNameDotPos+1));
+
+ size_t indexedNodeNameDotPos(indexedNodeKey.rfind('.'));
+ BOOST_ASSERT(std::string::npos != indexedNodeNameDotPos && "Could not parse indexed node name");
+ const std::string indexedNodeParentKey(indexedNodeKey.substr(0, indexedNodeNameDotPos));
+ const std::string indexedNodeName(indexedNodeKey.substr(indexedNodeNameDotPos+1));
+
+ ptree &indexedNodeParent(tree.get_child(indexedNodeParentKey));
+ BOOST_FOREACH(ptree::value_type &potentiallyIndexedNode, indexedNodeParent)
+ {
+ if (indexedNodeName == potentiallyIndexedNode.first)
+ {
+ std::string indexedNodeKey("<indexed>" + indexedNodeName + ".<" + indexAttributeName + ">"
+ + potentiallyIndexedNode.second.get<std::string>("<xmlattr>." + indexAttributeName));
+ BOOST_ASSERT(potentiallyIndexedNode.second.get_child("<xmlattr>").erase(indexAttributeName));
+ indexedNodeParent.add_child(indexedNodeKey, potentiallyIndexedNode.second);
+ }
+ }
+ indexedNodeParent.erase(indexedNodeName);
+ }
+}
+
+std::ostream &serializeAsXml(std::ostream &os, const boost::property_tree::ptree &tree)
+{
+ boost::property_tree::ptree treeWithIndexAttributes;
+ if (!tree.empty())
+ {
+ unindex(*tree.begin(), treeWithIndexAttributes);
+#ifndef WIN32
+ boost::property_tree::write_xml(os, treeWithIndexAttributes, boost::property_tree::xml_writer_make_settings(' ', 2));
+#else
+ boost::property_tree::write_xml(os, treeWithIndexAttributes, boost::property_tree::xml_writer_make_settings<std::string>(' ', 2));
+#endif
+ }
+ else
+ {
+#ifndef WIN32
+ boost::property_tree::write_xml(os, tree, boost::property_tree::xml_writer_make_settings(' ', 2));
+#else
+ boost::property_tree::write_xml(os, tree, boost::property_tree::xml_writer_make_settings<std::string>(' ', 2));
+#endif
+ }
+ return os;
+}
+
+std::istream &parseAsXml(std::istream &is, boost::property_tree::ptree &tree)
+{
+ boost::property_tree::read_xml<boost::property_tree::ptree>(is, tree);
+ return is;
+}
+
+boost::property_tree::ptree parseXmlFile(const boost::filesystem::path& xmlFile)
+{
+ boost::filesystem::ifstream xmlStream(xmlFile);
+ xmlStream.unsetf(std::ios::skipws);
+ int errnum = errno;
+ if (!xmlStream)
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errnum, (boost::format("Unable to open '%s' file for reading") % xmlFile.string()).str()));
+ }
+
+ typedef std::vector<char> XmlBuffer;
+ XmlBuffer xmlBuffer;
+ errnum = 0;
+ std::copy(
+ std::istream_iterator<XmlBuffer::value_type>(xmlStream),
+ std::istream_iterator<XmlBuffer::value_type>(),
+ std::back_inserter(xmlBuffer)
+ );
+
+ return parseXmlData(xmlBuffer.begin(), xmlBuffer.end());
+}
+
+
+} // namespace io
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/io/cppunit/CMakeLists.txt b/src/cxx/lib/io/cppunit/CMakeLists.txt
new file mode 100644
index 0000000..ec43d99
--- /dev/null
+++ b/src/cxx/lib/io/cppunit/CMakeLists.txt
@@ -0,0 +1,23 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for any cppunit subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CPPUNIT_CMAKE})
+
+
diff --git a/src/cxx/lib/io/cppunit/RegistryNames.txt b/src/cxx/lib/io/cppunit/RegistryNames.txt
new file mode 100644
index 0000000..1325a8d
--- /dev/null
+++ b/src/cxx/lib/io/cppunit/RegistryNames.txt
@@ -0,0 +1 @@
+GzipCompressor
diff --git a/src/cxx/lib/io/cppunit/testGzipCompressor.cpp b/src/cxx/lib/io/cppunit/testGzipCompressor.cpp
new file mode 100644
index 0000000..f901670
--- /dev/null
+++ b/src/cxx/lib/io/cppunit/testGzipCompressor.cpp
@@ -0,0 +1,178 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testGzipCompressor.cpp
+ *
+ * \brief GzipCompressor cppunit test declarations.
+ *
+ * \author Aaron Day
+ */
+
+
+#include "RegistryName.hh"
+#include "testGzipCompressor.hh"
+
+#include "io/GzipCompressor.hh"
+
+#include <boost/foreach.hpp>
+
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(TestGzipCompressor, registryName("GzipCompressor"));
+
+void TestGzipCompressor::setUp()
+{
+}
+
+void TestGzipCompressor::tearDown()
+{
+}
+
+void TestGzipCompressor::writeCompressedBlock(const std::string& uncompressedData,
+ bool useBgzf,
+ std::vector<char>& compressedData)
+{
+ bcl2fastq::io::GzipCompressor(compressedData, useBgzf).write(uncompressedData.c_str(), uncompressedData.size());
+}
+
+void TestGzipCompressor::decompressData(const std::vector<char>::const_iterator& compressedData,
+ size_t compressedSize,
+ std::vector<char>& decompressedData)
+{
+ boost::iostreams::filtering_ostream os;
+ os.push(boost::iostreams::gzip_decompressor());
+ os.push(boost::iostreams::back_inserter(decompressedData));
+ boost::iostreams::write(os, &(*compressedData), compressedSize);
+}
+
+void TestGzipCompressor::writeAllCompressedBlocks(std::vector<std::string>& uncompressedData,
+ bool useBgzf,
+ std::vector< std::vector<char> >& compressedData,
+ std::vector<char>& allCompressedBlocks)
+{
+ std::string fullString;
+ BOOST_FOREACH(const std::string& uncompressedString, std::make_pair(uncompressedData.begin(), uncompressedData.end()))
+ {
+ compressedData.push_back(std::vector<char>());
+ writeCompressedBlock(uncompressedString,
+ useBgzf,
+ compressedData.back());
+
+ allCompressedBlocks.insert(allCompressedBlocks.end(),
+ compressedData.back().begin(),
+ compressedData.back().end());
+ }
+}
+
+void TestGzipCompressor::createUncompressedData(std::vector<std::string>& uncompressedData,
+ std::string& fullString)
+{
+ uncompressedData.push_back("Hello. This is a test. ");
+ uncompressedData.push_back("Here is a second message to compress. "
+ " "
+ " Long string x" );
+ uncompressedData.push_back("And one more.");
+
+ BOOST_FOREACH(const std::string& uncompressedString, std::make_pair(uncompressedData.begin(), uncompressedData.end()))
+ {
+ fullString += uncompressedString;
+ }
+}
+
+void TestGzipCompressor::locateAndDecompressBlocks(const std::vector<std::string>& uncompressedData,
+ const std::vector< std::vector<char> >& compressedData,
+ const std::vector<char>& allCompressedBlocks)
+{
+ int offset = 0;
+ for (size_t i = 0; i < uncompressedData.size(); ++i)
+ {
+ // 4th byte should be the flags field.
+ CPPUNIT_ASSERT(allCompressedBlocks.at(offset + 3) & 0x04);
+
+ // 11th and 12th bytes are the length of the extra field.
+ CPPUNIT_ASSERT_EQUAL(static_cast<int>(allCompressedBlocks.at(offset + 10)), 4 + (int)bcl2fastq::io::GzipCompressor::SUBFIELD_LENGTH);
+ CPPUNIT_ASSERT_EQUAL(static_cast<int>(allCompressedBlocks.at(offset + 11)), 0);
+
+ // 13th and 14th bytes are the subfield identifier.
+ CPPUNIT_ASSERT_EQUAL(static_cast<int>(allCompressedBlocks.at(offset + 12)), 66);
+ CPPUNIT_ASSERT_EQUAL(static_cast<int>(allCompressedBlocks.at(offset + 13)), 67);
+
+ // 15th and 16th bytes are the length of the subfield data.
+ // This is completely redundant with 11th and 12th bytes, but that's the expected format.
+ CPPUNIT_ASSERT_EQUAL(static_cast<int>(allCompressedBlocks.at(offset + 14)), (int)bcl2fastq::io::GzipCompressor::SUBFIELD_LENGTH);
+ CPPUNIT_ASSERT_EQUAL(static_cast<int>(allCompressedBlocks.at(offset + 15)), 0);
+
+ // The next 8 bytes give the size of the compressed block.
+ size_t compressedSize = 0;
+ for (size_t j = (16 + bcl2fastq::io::GzipCompressor::SUBFIELD_LENGTH - 1); j >= 16; --j)
+ {
+ compressedSize <<= 8;
+ compressedSize |= allCompressedBlocks.at(offset + j);
+ }
+
+ // bgzf uses size-1, so we will too.
+ compressedSize += 1;
+ CPPUNIT_ASSERT_EQUAL(compressedSize, compressedData[i].size());
+
+ // Decompress a single block.
+ std::vector<char> decompressedData;
+ decompressData(allCompressedBlocks.begin()+offset,
+ compressedSize,
+ decompressedData);
+ CPPUNIT_ASSERT_EQUAL(std::string(decompressedData.begin(), decompressedData.end()), uncompressedData[i]);
+
+ offset += compressedSize;
+ }
+}
+
+void TestGzipCompressor::testBgzf()
+{
+ std::vector<std::string> uncompressedData;
+ std::string fullString;
+ createUncompressedData(uncompressedData,
+ fullString);
+
+ std::vector< std::vector<char> > compressedData;
+ std::vector<char> allCompressedBlocks;
+ writeAllCompressedBlocks(uncompressedData,
+ true,
+ compressedData,
+ allCompressedBlocks);
+
+ // Try decompressing all blocks at once
+ std::vector<char> decompressedAll;
+ decompressData(allCompressedBlocks.begin(),
+ allCompressedBlocks.size(),
+ decompressedAll);
+ CPPUNIT_ASSERT_EQUAL(std::string(decompressedAll.begin(), decompressedAll.end()), fullString);
+
+ // Test our ability to locate individual compressed blocks
+ locateAndDecompressBlocks(uncompressedData,
+ compressedData,
+ allCompressedBlocks);
+}
+
+void TestGzipCompressor::testSingleBlock()
+{
+ std::vector<std::string> uncompressedData;
+ std::string fullString;
+ createUncompressedData(uncompressedData,
+ fullString);
+
+ std::vector< std::vector<char> > compressedData;
+ std::vector<char> allCompressedBlocks;
+ writeAllCompressedBlocks(uncompressedData,
+ false,
+ compressedData,
+ allCompressedBlocks);
+
+ // Try decompressing all blocks at once
+ std::vector<char> decompressedAll;
+ decompressData(allCompressedBlocks.begin(),
+ allCompressedBlocks.size(),
+ decompressedAll);
+ CPPUNIT_ASSERT_EQUAL(std::string(decompressedAll.begin(), decompressedAll.end()), fullString);
+}
diff --git a/src/cxx/lib/io/cppunit/testGzipCompressor.hh b/src/cxx/lib/io/cppunit/testGzipCompressor.hh
new file mode 100644
index 0000000..d35eace
--- /dev/null
+++ b/src/cxx/lib/io/cppunit/testGzipCompressor.hh
@@ -0,0 +1,67 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testGzipCompressor.hh
+ *
+ * \brief GzipCompressor cppunit test declarations.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_IO_TEST_GZIP_COMPRESSOR_HH
+#define BCL2FASTQ_IO_TEST_GZIP_COMPRESSOR_HH
+
+
+#include <cppunit/extensions/HelperMacros.h>
+
+#include <string>
+#include <vector>
+
+
+/// \brief Test suite for GzipCompressor.
+class TestGzipCompressor : public CppUnit::TestFixture
+{
+ CPPUNIT_TEST_SUITE(TestGzipCompressor);
+ CPPUNIT_TEST(testBgzf);
+ CPPUNIT_TEST(testSingleBlock);
+ CPPUNIT_TEST_SUITE_END();
+
+public:
+ TestGzipCompressor() {}
+
+ void setUp();
+ void tearDown();
+ void testBgzf();
+ void testSingleBlock();
+
+private:
+ void writeCompressedBlock(const std::string& uncompressedData,
+ bool useBgzf,
+ std::vector<char>& compressedData);
+
+ void decompressData(const std::vector<char>::const_iterator& compressedData,
+ size_t compressedSize,
+ std::vector<char>& decompressedData);
+
+ void writeAllCompressedBlocks(std::vector<std::string>& uncompressedData,
+ bool useBgzf,
+ std::vector< std::vector<char> >& compressedData,
+ std::vector<char>& allCompressedBlocks);
+
+ void createUncompressedData(std::vector<std::string>& uncompressedData,
+ std::string& fullString);
+
+ void locateAndDecompressBlocks(const std::vector<std::string>& uncompressedData,
+ const std::vector< std::vector<char> >& compressedData,
+ const std::vector<char>& allCompressedBlocks);
+};
+
+
+#endif // BCL2FASTQ_IO_TEST_GZIP_COMPRESSOR_HH
+
diff --git a/src/cxx/lib/layout/BCIndex.cpp b/src/cxx/lib/layout/BCIndex.cpp
new file mode 100644
index 0000000..71170fa
--- /dev/null
+++ b/src/cxx/lib/layout/BCIndex.cpp
@@ -0,0 +1,145 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BCIndex.cpp
+ *
+ * \brief Implementation of BCI helper.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <sstream>
+#include <utility>
+
+#include <boost/filesystem/fstream.hpp>
+#include <boost/foreach.hpp>
+#include <boost/format.hpp>
+#include <boost/filesystem.hpp>
+
+#include "common/Debug.hh"
+#include "common/Exceptions.hh"
+#include "layout/BCIndex.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+namespace detail {
+
+
+/// \brief Generate path to BCI file.
+/// \param inputDir Path to input directory.
+/// \param laneNumber Lane number.
+/// \return Path to BCI file.
+boost::filesystem::path generateBciFilePath(const boost::filesystem::path& inputDir, common::LaneNumber laneNumber)
+{
+ std::ostringstream laneDirName;
+ laneDirName << boost::format("L%03d") % laneNumber;
+
+ std::ostringstream bciFileName;
+ bciFileName << boost::format("s_%d.bci") % laneNumber;
+
+ return boost::filesystem::path(inputDir / laneDirName.str() / bciFileName.str());
+}
+
+
+/// \brief BCI file record type definition.
+#pragma pack(push, 1)
+struct Record
+{
+public:
+
+ /// \brief Tile number.
+ uint32_t tileNumber_;
+
+ /// \brief Number of clusters on the tile.
+ uint32_t clustersCount_;
+
+};
+#pragma pack(pop)
+
+
+} // namespace detail
+
+
+BCIndex::BCIndex(const RawData &bciData)
+: tiles_()
+{
+ this->initTileMetadata(bciData);
+}
+
+BCIndex::TileMetadataContainer::const_iterator BCIndex::tileMetadataBegin() const
+{
+ return tiles_.begin();
+}
+
+BCIndex::TileMetadataContainer::const_iterator BCIndex::tileMetadataEnd() const
+{
+ return tiles_.end();
+}
+
+void BCIndex::initTileMetadata(const RawData &bciData)
+{
+ if (bciData.empty())
+ {
+ return;
+ }
+
+ BCL2FASTQ_ASSERT_MSG(bciData.size() % sizeof(detail::Record) == 0, "Invalid BCI data: data_size=" << bciData.size() << " record_size=" << sizeof(detail::Record));
+ BOOST_FOREACH (const detail::Record &record, std::make_pair(reinterpret_cast<const detail::Record *>(&bciData[0]), reinterpret_cast<const detail::Record *>(&bciData[0] + bciData.size())))
+ {
+ tiles_.push_back(TileMetadata());
+ TileMetadata &tileMetadata = tiles_.back();
+ tileMetadata.tileNumber_ = record.tileNumber_;
+ tileMetadata.clustersCount_ = record.clustersCount_;
+ }
+}
+
+BCIndex::RawData parseBciFile(const boost::filesystem::path& bciFile)
+{
+ int errnum;
+
+ boost::filesystem::ifstream bciStream(bciFile, std::ios_base::in | std::ios_base::binary);
+ errnum = errno;
+ if (!bciStream)
+ {
+ BOOST_THROW_EXCEPTION(common::IoError(errnum, (boost::format("Unable to open '%s' file for reading") % bciFile.string()).str()));
+ }
+
+ BCIndex::RawData bciBuffer;
+ errnum = 0;
+ std::copy(
+ std::istreambuf_iterator<BCIndex::RawData::value_type>(bciStream),
+ std::istreambuf_iterator<BCIndex::RawData::value_type>(),
+ std::back_inserter(bciBuffer)
+ );
+
+ return bciBuffer;
+}
+
+bool checkBciFile(boost::filesystem::path inputDir, common::LaneNumber laneNumber)
+{
+ return boost::filesystem::exists(detail::generateBciFilePath(inputDir, laneNumber));
+}
+
+BCIndex createBCIndex(boost::filesystem::path inputDir, common::LaneNumber laneNumber)
+{
+ return BCIndex(parseBciFile(detail::generateBciFilePath(inputDir, laneNumber)));
+}
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/layout/Barcode.cpp b/src/cxx/lib/layout/Barcode.cpp
new file mode 100644
index 0000000..d5e63cc
--- /dev/null
+++ b/src/cxx/lib/layout/Barcode.cpp
@@ -0,0 +1,168 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Barcode.cpp
+ *
+ * \brief Implementation of barcode.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <boost/format.hpp>
+#include <boost/foreach.hpp>
+
+#include "common/Debug.hh"
+#include "common/Exceptions.hh"
+#include "layout/Barcode.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+const char Barcode::COMPONENT_SEPARATOR = '+';
+
+Barcode::Component::Component(const std::string &value)
+: value_(value)
+{
+ if (value.empty())
+ {
+ BOOST_THROW_EXCEPTION(common::LogicError("Barcode component empty"));
+ }
+}
+
+const char Barcode::DEFAULT_BARCODE[] = "unknown";
+
+void Barcode::reset(size_t numIndexReads)
+{
+ // Do not clear the vector. We want to avoid reallocations for the strings.
+ components_.resize(numIndexReads);
+
+ for (auto& component : components_)
+ {
+ component.reset();
+ }
+}
+
+void Barcode::output(std::vector<char>& buffer) const
+{
+ size_t bufferSize = buffer.size();
+ const std::string& firstBarcodeString = components_.front().getString();
+ buffer.resize(bufferSize + firstBarcodeString.size());
+ std::copy(firstBarcodeString.begin(), firstBarcodeString.end(), buffer.begin() + bufferSize);
+
+ for (size_t i = 1; i < components_.size(); ++i)
+ {
+ buffer.push_back(COMPONENT_SEPARATOR);
+
+ bufferSize = buffer.size();
+ buffer.resize(bufferSize + components_[i].getString().size());
+ std::copy(components_[i].getString().begin(), components_[i].getString().end(), buffer.begin() + bufferSize);
+ }
+}
+
+void Barcode::mask(common::ReadNumber indexNumber)
+{
+ BCL2FASTQ_ASSERT_MSG(indexNumber <= components_.size(), "Attempting to mask a barcode that doesn't exist.");
+ components_.erase(components_.begin()+indexNumber-1);
+}
+
+std::ostream& operator<<(std::ostream& os, const Barcode& barcode)
+{
+ if (barcode.components_.empty())
+ {
+ os << Barcode::DEFAULT_BARCODE;
+ }
+ else
+ {
+ os << barcode.components_[0].getString();
+ for (size_t i = 1; i < barcode.components_.size(); ++i)
+ {
+ os << Barcode::COMPONENT_SEPARATOR;
+ os << barcode.components_[i].getString();
+ }
+ }
+
+ return os;
+}
+
+// The conversion operator was abused, and led to terrible memory allocations.
+// Use this method sparingly.
+std::string Barcode::toString() const
+{
+ std::string ret;
+ if( components_.empty() )
+ {
+ return DEFAULT_BARCODE;
+ }
+ BOOST_FOREACH (const Component &component, components_)
+ {
+ ret.append(component.getString());
+ ret += COMPONENT_SEPARATOR;
+ }
+ ret.resize(ret.size()-1);
+ return ret;
+}
+
+bool Barcode::operator<(const Barcode &rhs) const
+{
+ /// \todo Bug: Fix this function to have the same result as string comparison.
+ const ComponentsContainer::size_type lhsSize = components_.size();
+ const ComponentsContainer::size_type rhsSize = rhs.components_.size();
+ if (lhsSize != rhsSize)
+ {
+ return lhsSize < rhsSize;
+ }
+
+ for (ComponentsContainer::size_type i = 0; i < lhsSize; ++i)
+ {
+ if (components_[i].less(rhs.components_[i]))
+ {
+ return true;
+ }
+ if (rhs.components_[i].less(components_[i]))
+ {
+ return false;
+ }
+ }
+ return false;
+}
+
+bool Barcode::operator==(const Barcode &rhs) const
+{
+ return !(*this != rhs);
+}
+
+bool Barcode::operator!=(const Barcode &rhs) const
+{
+ const ComponentsContainer::size_type lhsSize = components_.size();
+ const ComponentsContainer::size_type rhsSize = rhs.components_.size();
+ if (lhsSize != rhsSize)
+ {
+ return true;
+ }
+
+ for (ComponentsContainer::size_type i = 0; i < lhsSize; ++i)
+ {
+ if (components_[i] != rhs.components_[i])
+ {
+ return true;
+ }
+ }
+ return false;
+}
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/layout/BarcodeCollisionDetector.cpp b/src/cxx/lib/layout/BarcodeCollisionDetector.cpp
new file mode 100644
index 0000000..f6c3be3
--- /dev/null
+++ b/src/cxx/lib/layout/BarcodeCollisionDetector.cpp
@@ -0,0 +1,210 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BarcodeCollisionDetector.cpp
+ *
+ * \brief Implementation of BarcodeCollisionDetector.
+ *
+ * \author Aaron Day
+ */
+
+#include "layout/BarcodeCollisionDetector.hh"
+#include "layout/Barcode.hh"
+#include "common/Debug.hh"
+
+namespace bcl2fastq
+{
+namespace layout
+{
+
+BarcodeCollisionError::BarcodeCollisionError(const std::string& message)
+: common::RuntimeError(0, message)
+{
+}
+
+BarcodeCollisionError::BarcodeCollisionError(const BarcodeCollisionError& other)
+: common::RuntimeError(other)
+{
+}
+
+
+BarcodeCollisionDetector::BarcodeCollisionDetector(config::BarcodeMismatchCountsContainer& componentMaxMismatches,
+ bool autoSetToZeroMismatches /*=false*/)
+ : autoSetToZeroMismatches_(autoSetToZeroMismatches)
+ , componentMaxMismatches_(componentMaxMismatches)
+ , barcodes_()
+{
+}
+
+void BarcodeCollisionDetector::validateBarcode(BarcodesContainer barcodes)
+{
+ validateBarcodeSizes(barcodes);
+
+ // Compare each of the barcodes for a single sample against all barcodes for all other samples
+ for (const auto& barcode2 : barcodes)
+ {
+ for (const auto& barcode1 : barcodes_)
+ {
+ validateBarcodes(barcode1, barcode2);
+ }
+
+ // There were no collisions, add this sample's barcodes to the list of all barcodes
+ barcodes_.push_back(barcode2);
+ }
+}
+
+void BarcodeCollisionDetector::validateBarcodeSizes(const BarcodesContainer& barcodes)
+{
+ BCL2FASTQ_ASSERT_MSG(!barcodes.empty(), "No barcodes for sample.");
+ BCL2FASTQ_ASSERT_MSG(!barcodes.front().getComponents().empty(), "Barcode has 0 components");
+
+ std::vector<std::size_t> barcodeComponentSizes;
+ for (const auto& component : barcodes.front().getComponents())
+ {
+ barcodeComponentSizes.push_back(component.size());
+ }
+
+ validateNewBarcodeSizes(barcodes,
+ barcodeComponentSizes);
+
+ validateNewBarcodeSizesAgainstExisting(barcodeComponentSizes);
+}
+
+void BarcodeCollisionDetector::validateNewBarcodeSizes(const BarcodesContainer& barcodes,
+ const std::vector<std::size_t>& barcodeComponentSizes) const
+{
+ size_t numComponents = barcodes.front().getComponents().size();
+
+ for (const auto& barcode : barcodes)
+ {
+ if (numComponents != barcode.getComponents().size())
+ {
+ BOOST_THROW_EXCEPTION(BarcodeCollisionError("Barcodes have an unequal number of components."));
+ }
+
+ std::vector<std::size_t>::const_iterator barcodeComponentSize = barcodeComponentSizes.begin();
+ for (const auto& component : barcode.getComponents())
+ {
+ if (component.size() != *barcodeComponentSize)
+ {
+ BOOST_THROW_EXCEPTION(BarcodeCollisionError("Barcodes components must have the same size."));
+ }
+ ++barcodeComponentSize;
+ }
+ }
+}
+
+void BarcodeCollisionDetector::validateNewBarcodeSizesAgainstExisting(const std::vector<std::size_t>& barcodeComponentSizes) const
+{
+ if (!barcodes_.empty())
+ {
+ if (barcodeComponentSizes.size() != barcodes_.back().getComponents().size())
+ {
+ BOOST_THROW_EXCEPTION(BarcodeCollisionError("Barcodes have an unequal number of components."));
+ }
+
+ std::vector<std::size_t>::const_iterator barcodeComponentSize = barcodeComponentSizes.begin();
+ for (const auto& component : barcodes_.back().getComponents())
+ {
+ if (component.size() != *barcodeComponentSize)
+ {
+ BOOST_THROW_EXCEPTION(BarcodeCollisionError("Barcodes components must have the same size."));
+ }
+ ++barcodeComponentSize;
+ }
+ }
+}
+
+void BarcodeCollisionDetector::validateBarcodes(const BarcodesContainer::value_type& barcode1,
+ const BarcodesContainer::value_type& barcode2)
+{
+ size_t numComponentsWithCollision = 0;
+ std::vector<std::size_t>::const_iterator componentMaxMismatchesIter = componentMaxMismatches_.begin();
+ auto barcode2Iter = barcode2.componentsBegin();
+ for (const auto& component : barcode1.getComponents())
+ {
+ // Multiply by 2 because both barcodes can have mismatches
+ if (2* (*componentMaxMismatchesIter) >= getNumMismatches(component, *barcode2Iter))
+ {
+ ++numComponentsWithCollision;
+ }
+
+ ++barcode2Iter;
+ if (componentMaxMismatchesIter + 1 != componentMaxMismatches_.end())
+ {
+ ++componentMaxMismatchesIter;
+ }
+ }
+
+ if (numComponentsWithCollision == barcode1.getComponents().size())
+ {
+ handleCollision(barcode1, barcode2);
+ }
+}
+
+void BarcodeCollisionDetector::handleCollision(const BarcodesContainer::value_type& barcode1,
+ const BarcodesContainer::value_type& barcode2)
+{
+ std::stringstream errStr;
+
+ if (barcode1 == barcode2)
+ {
+ errStr << "Identical barcode found for multiple samples in the sample sheet: " << barcode1;
+ }
+ else
+ {
+ errStr << "Barcode collision for barcodes: "
+ << barcode1
+ << ", "
+ << barcode2;
+
+ if (autoSetToZeroMismatches_)
+ {
+ for (auto& maxMismatches : componentMaxMismatches_)
+ {
+ maxMismatches = 0;
+ }
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << errStr.str()
+ << ". Setting the number of allowed mismatches to 0." << std::endl;
+
+ return;
+ }
+ else
+ {
+ errStr << std::endl << "By default, bcl2fastq allows 1 mismatch in each barcode."
+ " Barcodes with too few mismatches are ambiguous ( less than 2 times the number of mismatches plus 1)."
+ " To reduce the number of allowed mismatches, use the command line option: '--barcode-mismatches'."
+ " Note that particularly for barcodes with only 1 mismatch, there is the danger that some reads will"
+ " be written to the wrong sample due to errors in the barcode sequence.";
+ }
+ }
+
+ BOOST_THROW_EXCEPTION(BarcodeCollisionError(errStr.str()));
+}
+
+size_t BarcodeCollisionDetector::getNumMismatches(const Barcode::Component& barcodeComponent1,
+ const Barcode::Component& barcodeComponent2) const
+{
+ size_t numMismatches = 0;
+
+ size_t barcodeComponentSize = barcodeComponent1.size();
+ for (size_t i = 0; i < barcodeComponentSize; ++i)
+ {
+ // We've already verified that the sizes are equal
+ if (barcodeComponent1.getString()[i] != barcodeComponent2.getString()[i])
+ {
+ ++numMismatches;
+ }
+ }
+
+ return numMismatches;
+}
+
+} // namespace bcl2fastq
+} // namespace layout
+
diff --git a/src/cxx/lib/layout/BarcodeTranslationTable.cpp b/src/cxx/lib/layout/BarcodeTranslationTable.cpp
new file mode 100644
index 0000000..f124e95
--- /dev/null
+++ b/src/cxx/lib/layout/BarcodeTranslationTable.cpp
@@ -0,0 +1,201 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file BarcodeTranslationTable.cpp
+ *
+ * \brief Implementation of barcode translation table.
+ *
+ * \author Roman Petrovski
+ * \author Marek Balint
+ */
+
+
+#include <algorithm>
+#include <utility>
+
+#include <boost/assign/list_of.hpp>
+
+#include "common/Debug.hh"
+#include "common/Exceptions.hh"
+#include "layout/BarcodeTranslationTable.hh"
+
+
+namespace bcl2fastq
+{
+namespace layout
+{
+
+
+BarcodeTranslationTable::BarcodeTranslationTable(
+ const layout::LaneInfo::SampleInfosContainer::const_iterator samplesBegin,
+ const layout::LaneInfo::SampleInfosContainer::const_iterator samplesEnd,
+ const config::BarcodeMismatchCountsContainer& componentMaxMismatches
+)
+: barcodeSampleMappings_(this->initBarcodeSampleMappings(samplesBegin, samplesEnd, componentMaxMismatches))
+{
+}
+
+BarcodeTranslationTable::SampleMetadata BarcodeTranslationTable::translateBarcode(
+ const layout::Barcode &barcode,
+ unsigned int &mismatches
+) const
+{
+ const BarcodeSampleMapping mappingToTranslate(barcode, SampleMetadata());
+ const BarcodeSampleMappingsContainer::const_iterator mapping = std::lower_bound(
+ barcodeSampleMappings_.begin(),
+ barcodeSampleMappings_.end(),
+ mappingToTranslate,
+ &BarcodeTranslationTable::compareBarcodeSampleMappings
+ );
+
+ if (mapping == barcodeSampleMappings_.end() || mapping->barcode != mappingToTranslate.barcode)
+ {
+ return SampleMetadata();
+ }
+ else
+ {
+ //BCL2FASTQ_LOG(common::LogLevel::TRACE) << "LOOKUP:" << *(mapping.first) << std::endl;
+ mismatches = mapping->mismatches;
+ return mapping->sample;
+ }
+}
+
+const BarcodeTranslationTable::BarcodeSampleMappingsContainer BarcodeTranslationTable::initBarcodeSampleMappings(
+ const layout::LaneInfo::SampleInfosContainer::const_iterator samplesBegin,
+ const layout::LaneInfo::SampleInfosContainer::const_iterator samplesEnd,
+ const config::BarcodeMismatchCountsContainer& componentMaxMismatches
+)
+{
+ BarcodeSampleMappingsContainer barcodeMappings;
+ for (layout::LaneInfo::SampleInfosContainer::const_iterator samplesIter = samplesBegin; samplesIter != samplesEnd; ++samplesIter)
+ {
+ const auto& barcodes = samplesIter->getBarcodes();
+ for (size_t i = 0; i < barcodes.size(); ++i)
+ {
+ SampleMetadata sample;
+ sample.sampleIndex_ = (samplesIter - samplesBegin);
+ sample.barcodeIndex_ = i;
+ barcodeMappings.push_back(BarcodeSampleMappingsContainer::value_type(barcodes[i], sample));
+ }
+ }
+
+ BarcodeSampleMappingsContainer ret;
+
+ BOOST_FOREACH(const BarcodeSampleMappingsContainer::value_type &barcodeMapping, barcodeMappings)
+ {
+ /// \todo Features: Special handling for sample No.0. will be needed in case, when sample No.0 will have some value other than '********-********', which is currently hardcoded in LaneInfo.cpp.
+
+ BarcodeTranslationTable::insertBarcodeMapping(
+ barcodeMapping,
+ barcodeMapping.barcode.componentsBegin(),
+ 0,
+ componentMaxMismatches.front(),
+ componentMaxMismatches.begin(),
+ componentMaxMismatches.end(),
+ ret
+ );
+ }
+
+ std::sort(
+ ret.begin(),
+ ret.end(),
+ &BarcodeTranslationTable::compareBarcodeSampleMappings
+ );
+
+ return ret;
+}
+
+void BarcodeTranslationTable::insertBarcodeMapping(
+ const BarcodeSampleMapping &barcodeMapping,
+ layout::Barcode::ComponentsContainer::const_iterator component,
+ layout::Barcode::Component::SizeType position,
+ unsigned int allowedMismatchesLeft,
+ std::vector<std::size_t>::const_iterator componentMaxMismatchesIter,
+ std::vector<std::size_t>::const_iterator componentMaxMismatchesEnd,
+ BarcodeSampleMappingsContainer &container
+)
+{
+ BCL2FASTQ_LOG(common::LogLevel::TRACE) << "\t[barcode table] " << barcodeMapping << std::endl;
+ if(component != barcodeMapping.barcode.componentsEnd())
+ {
+ // simulate mismatches
+ if (allowedMismatchesLeft > 0) {
+ const layout::Barcode::Component::SizeType componentLength = component->getLength();
+ for (layout::Barcode::Component::SizeType mismatchedPosition = position; mismatchedPosition < componentLength; ++mismatchedPosition)
+ {
+ // magic number 4: There are 5 distinct values ('A', 'C', 'G', 'T', 'N'), therefore there are 4 distinct mismatches
+ std::vector<layout::Barcode> mismatchedBarcodes;
+ mismatchedBarcodes.reserve(4);
+ barcodeMapping.barcode.generateMismatches(component, mismatchedPosition, std::back_inserter(mismatchedBarcodes));
+
+ BOOST_FOREACH (const layout::Barcode &mismatchedBarcode, mismatchedBarcodes)
+ {
+ const BarcodeSampleMapping mismatchedBarcodeMapping(mismatchedBarcode,
+ barcodeMapping.sample,
+ *componentMaxMismatchesIter - allowedMismatchesLeft + 1);
+ BarcodeTranslationTable::insertBarcodeMapping(
+ mismatchedBarcodeMapping,
+ mismatchedBarcodeMapping.barcode.componentsBegin() + (component - barcodeMapping.barcode.componentsBegin()),
+ mismatchedPosition + 1,
+ allowedMismatchesLeft - 1,
+ componentMaxMismatchesIter,
+ componentMaxMismatchesEnd,
+ container
+ );
+ }
+ }
+ }
+
+ // move to next component (do not change current component)
+ std::vector<std::size_t>::const_iterator nextMismatchesIter = (componentMaxMismatchesIter + 1 != componentMaxMismatchesEnd
+ ?
+ componentMaxMismatchesIter + 1
+ :
+ componentMaxMismatchesIter
+ );
+ BarcodeTranslationTable::insertBarcodeMapping(
+ barcodeMapping,
+ component + 1,
+ 0,
+ *nextMismatchesIter,
+ nextMismatchesIter,
+ componentMaxMismatchesEnd,
+ container
+ );
+ }
+ else
+ {
+ // actual insertion
+ container.push_back(barcodeMapping);
+ }
+}
+
+bool BarcodeTranslationTable::compareBarcodeSampleMappings(const BarcodeSampleMapping &lhs, const BarcodeSampleMapping &rhs)
+{
+ return lhs.barcode < rhs.barcode;
+}
+
+
+std::ostream& operator<<(std::ostream& os, const BarcodeTranslationTable::BarcodeSampleMapping &bsm)
+{
+ os << bsm.barcode;
+ if( 0 != bsm.sample.sampleIndex_ )
+ {
+ os << std::string(" => sample #") << bsm.sample.sampleIndex_;
+ } else {
+ os << std::string(" : *");
+ }
+ os << " : " << bsm.mismatches;
+ return os;
+}
+
+
+} // namespace conversion
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/layout/CMakeLists.txt b/src/cxx/lib/layout/CMakeLists.txt
new file mode 100644
index 0000000..26c8f9d
--- /dev/null
+++ b/src/cxx/lib/layout/CMakeLists.txt
@@ -0,0 +1,23 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the cxx/lib/conversion subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CXX_LIBRARY_CMAKE})
+
+
diff --git a/src/cxx/lib/layout/ConfigXml.cpp b/src/cxx/lib/layout/ConfigXml.cpp
new file mode 100644
index 0000000..5f13b92
--- /dev/null
+++ b/src/cxx/lib/layout/ConfigXml.cpp
@@ -0,0 +1,127 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ConfigXml.cpp
+ *
+ * \brief Implementation of config.xml helper.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <boost/optional.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+
+#include "common/Logger.hh"
+#include "io/Xml.hh"
+#include "layout/ConfigXml.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+namespace detail {
+
+
+/// \brief Generate path to config.xml file.
+/// \param inputDir Path to input directory.
+/// \return Path to config.xml file.
+boost::filesystem::path generateConfigXmlFilePath(const boost::filesystem::path& inputDir)
+{
+ return boost::filesystem::path(inputDir / boost::filesystem::path("config.xml"));
+}
+
+
+} // namespace detail
+
+
+ConfigXml::ConfigXml(boost::property_tree::ptree configData)
+: ptree_(configData)
+, tiles_()
+{
+ this->initTileMetadata();
+}
+
+ConfigXml::TileMetadataContainer::const_iterator ConfigXml::tileMetadataBegin(common::LaneNumber laneNumber) const
+{
+ BCL2FASTQ_ASSERT_MSG(laneNumber > 0, "Invalid lane number: " << laneNumber);
+ BCL2FASTQ_ASSERT_MSG(laneNumber <= tiles_.size(), "Invalid lane number: " << laneNumber);
+ return tiles_.at(laneNumber - 1).begin();
+}
+
+ConfigXml::TileMetadataContainer::const_iterator ConfigXml::tileMetadataEnd(common::LaneNumber laneNumber) const
+{
+ BCL2FASTQ_ASSERT_MSG(laneNumber > 0, "Invalid lane number: " << laneNumber);
+ BCL2FASTQ_ASSERT_MSG(laneNumber <= tiles_.size(), "Invalid lane number: " << laneNumber);
+ return tiles_.at(laneNumber - 1).end();
+}
+
+void ConfigXml::initTileMetadata()
+{
+ boost::optional<boost::property_tree::ptree &> tileSelectionElement = ptree_.get_child_optional("BaseCallAnalysis.Run.TileSelection");
+ if (!tileSelectionElement) {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Required element 'BaseCallAnalysis.Run.TileSelection' missing in config.xml file"));
+ }
+
+ BOOST_FOREACH (boost::property_tree::ptree::value_type &laneElement, tileSelectionElement.get())
+ {
+ static const std::string laneElementName("Lane");
+ if (laneElement.first != laneElementName)
+ {
+ continue;
+ }
+
+ boost::optional<common::LaneNumber> laneNumber = laneElement.second.get_optional<common::LaneNumber>("<xmlattr>.Index");
+ if (!laneNumber)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Required attrubute 'Index' missing in element element 'BaseCallAnalysis.Run.BaseCallParameters.TileSelection.Lane' in config.xml file"));
+ }
+ if (tiles_.size() <= laneNumber.get())
+ {
+ tiles_.resize(laneNumber.get());
+ }
+ BOOST_FOREACH (const boost::property_tree::ptree::value_type &tileElement, laneElement.second)
+ {
+ static const std::string tileElementName("Tile");
+ if (tileElement.first != tileElementName)
+ {
+ continue;
+ }
+
+ TileMetadata tileMetadata;
+ tileMetadata.tileNumber_ = tileElement.second.get_value<common::TileNumber>();
+ tiles_.at(laneNumber.get() - 1).push_back(tileMetadata);
+ }
+ }
+}
+
+
+bool checkConfigXml(const boost::filesystem::path& inputDir)
+{
+ boost::filesystem::path configXmlFileName(detail::generateConfigXmlFilePath(inputDir));
+ return boost::filesystem::ifstream(configXmlFileName).good() && !boost::filesystem::is_directory(configXmlFileName);
+}
+
+ConfigXml createConfigXml(const boost::filesystem::path& inputDir)
+{
+ boost::filesystem::path file = detail::generateConfigXmlFilePath(inputDir);
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "config.xml: " << file << std::endl;
+ return ConfigXml(io::parseXmlFile(file));
+}
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/layout/CycleInfo.cpp b/src/cxx/lib/layout/CycleInfo.cpp
new file mode 100644
index 0000000..5061164
--- /dev/null
+++ b/src/cxx/lib/layout/CycleInfo.cpp
@@ -0,0 +1,45 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file CycleInfo.cpp
+ *
+ * \brief Implementation of cycle metadata.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <sstream>
+
+#include <boost/format.hpp>
+
+#include "layout/CycleInfo.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+CycleInfo::CycleInfo(common::CycleNumber number)
+: number_(number)
+{
+}
+
+common::CycleNumber CycleInfo::getNumber() const
+{
+ return number_;
+}
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/layout/FileExistenceVerifier.cpp b/src/cxx/lib/layout/FileExistenceVerifier.cpp
new file mode 100644
index 0000000..f708761
--- /dev/null
+++ b/src/cxx/lib/layout/FileExistenceVerifier.cpp
@@ -0,0 +1,217 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FileExistenceVerifier.cpp
+ *
+ * \brief Implementation of FileExistenceVerifier.
+ *
+ * \author Aaron Day
+ */
+
+#include "layout/FileExistenceVerifier.hh"
+
+#include "layout/LaneInfo.hh"
+#include "data/FilterFile.hh"
+#include "data/PositionsFile.hh"
+#include "data/BclFile.hh"
+#include "data/CbclFile.hh"
+
+namespace bcl2fastq
+{
+namespace layout
+{
+
+void FileExistenceVerifier::verifyAllFilesExist(const boost::filesystem::path& inputDir,
+ const boost::filesystem::path& intensitiesDir,
+ const LaneInfo& laneInfo,
+ const common::TileFileMap& tileFileNameMap,
+ common::TileAggregationMode aggregateTilesMode,
+ bool isPatternedFlowcell,
+ bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions)
+{
+ if (aggregateTilesMode == common::TileAggregationMode::AGGREGATED)
+ {
+ verifyFilesExist(inputDir,
+ intensitiesDir,
+ laneInfo,
+ tileFileNameMap,
+ aggregateTilesMode,
+ isPatternedFlowcell,
+ ignoreMissingBcls,
+ ignoreMissingFilters,
+ ignoreMissingPositions);
+ }
+ else
+ {
+ for (const auto& tileInfo : laneInfo.getTileInfos())
+ {
+ verifyFilesExist(inputDir,
+ intensitiesDir,
+ laneInfo,
+ tileFileNameMap,
+ aggregateTilesMode,
+ isPatternedFlowcell,
+ ignoreMissingBcls,
+ ignoreMissingFilters,
+ ignoreMissingPositions,
+ tileInfo.getNumber());
+ }
+ }
+}
+
+void FileExistenceVerifier::verifyFilesExist(const boost::filesystem::path& inputDir,
+ const boost::filesystem::path& intensitiesDir,
+ const LaneInfo& laneInfo,
+ const common::TileFileMap& tileFileNameMap,
+ common::TileAggregationMode aggregateTilesMode,
+ bool isPatternedFlowcell,
+ bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions,
+ const common::TileNumber tileNumber /*= 0*/)
+{
+ for (const auto& readInfo : laneInfo.readInfos())
+ {
+ if (!ignoreMissingBcls)
+ {
+ for (const layout::CycleInfo &cycleInfo : readInfo.cycleInfos())
+ {
+ verifyBclFileExists(inputDir,
+ aggregateTilesMode,
+ laneInfo.getNumber(),
+ cycleInfo.getNumber(),
+ tileNumber,
+ tileFileNameMap);
+ }
+ }
+
+ if (!ignoreMissingFilters)
+ {
+ verifyFilterFileExists(inputDir,
+ aggregateTilesMode,
+ laneInfo.getNumber(),
+ tileNumber,
+ readInfo.getNumber());
+ }
+ }
+
+ if (!ignoreMissingPositions)
+ {
+ verifyPositionsFileExists(intensitiesDir,
+ aggregateTilesMode,
+ isPatternedFlowcell,
+ laneInfo.getNumber(),
+ tileNumber);
+ }
+}
+
+void FileExistenceVerifier::verifyBclFileExists(const boost::filesystem::path& inputDir,
+ common::TileAggregationMode aggregateTilesMode,
+ common::LaneNumber laneNumber,
+ common::CycleNumber cycleNumber,
+ common::TileNumber tileNumber,
+ const common::TileFileMap& tileFileNameMap)
+{
+ boost::filesystem::path fileName;
+ switch (aggregateTilesMode)
+ {
+ case common::TileAggregationMode::AGGREGATED:
+ data::BclFile::getAndVerifyFileName(inputDir,
+ laneNumber,
+ cycleNumber,
+ false,
+ fileName);
+ break;
+ case common::TileAggregationMode::NON_AGGREGATED:
+ data::BclFile::getAndVerifyFileName(inputDir,
+ laneNumber,
+ tileNumber,
+ cycleNumber,
+ false,
+ fileName);
+ break;
+ case common::TileAggregationMode::CBCL:
+ fileName = data::CbclFileReader::getFileName(inputDir,
+ laneNumber,
+ cycleNumber,
+ tileNumber,
+ tileFileNameMap,
+ false);
+ break;
+ default:
+ BCL2FASTQ_ASSERT_MSG(false, "Unexpected aggregation mode type");
+ }
+}
+
+void FileExistenceVerifier::verifyFilterFileExists(const boost::filesystem::path& inputDir,
+ common::TileAggregationMode aggregateTilesMode,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ common::ReadNumber readNumber)
+{
+ size_t headerSize = 0;
+ boost::filesystem::path filePath;
+ if (!(data::FilterFile::doesFileExist(inputDir,
+ aggregateTilesMode == common::TileAggregationMode::AGGREGATED,
+ laneNumber,
+ tileNumber,
+ headerSize,
+ filePath)))
+ {
+ throwException("filter",
+ aggregateTilesMode,
+ laneNumber,
+ tileNumber);
+ }
+}
+
+void FileExistenceVerifier::verifyPositionsFileExists(const boost::filesystem::path& intensitiesDir,
+ common::TileAggregationMode aggregateTilesMode,
+ bool isPatternedFlowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber)
+{
+ size_t headerSize = 0;
+ boost::filesystem::path posFilePath;
+ if (!data::PositionsFileFactory::doesFileExist(intensitiesDir,
+ aggregateTilesMode == common::TileAggregationMode::AGGREGATED ||
+ aggregateTilesMode == common::TileAggregationMode::CBCL,
+ isPatternedFlowcell,
+ laneNumber,
+ tileNumber,
+ headerSize,
+ posFilePath))
+ {
+ throwException("positions",
+ aggregateTilesMode,
+ laneNumber,
+ tileNumber);
+ }
+}
+
+void FileExistenceVerifier::throwException(const std::string& fileType,
+ common::TileAggregationMode aggregateTilesMode,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber)
+{
+ std::stringstream str;
+ str << "Unable to find " << fileType << " file for lane: " << laneNumber;
+ if (aggregateTilesMode == common::TileAggregationMode::NON_AGGREGATED)
+ {
+ str << " and tile: " << tileNumber;
+ }
+ str << std::endl;
+
+ BOOST_THROW_EXCEPTION(common::IoError(ENOENT, str.str()));
+}
+
+} // namespace layout
+} // namespace bcl2fastq
+
diff --git a/src/cxx/lib/layout/FlowcellInfo.cpp b/src/cxx/lib/layout/FlowcellInfo.cpp
new file mode 100644
index 0000000..baf3d88
--- /dev/null
+++ b/src/cxx/lib/layout/FlowcellInfo.cpp
@@ -0,0 +1,77 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file FlowcellInfo.cpp
+ *
+ * \brief Implementation of flowcell metadata.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#include <sstream>
+
+#include <boost/format.hpp>
+
+#include "layout/FlowcellInfo.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+void FlowcellInfo::setInstrument(const std::string &instrument)
+{
+ instrument_ = instrument;
+}
+
+void FlowcellInfo::setRunNumber(const std::string &runNumber)
+{
+ runNumber_ = runNumber;
+}
+
+void FlowcellInfo::setFlowcellId(const std::string &flowcellId)
+{
+ flowcellId_ = flowcellId;
+}
+
+common::TileAggregationMode FlowcellInfo::getAggregateTilesMode() const
+{
+ return aggregateTilesFlag_;
+}
+
+void FlowcellInfo::setAggregateTilesFlag(common::TileAggregationMode aggregateTilesFlag)
+{
+ aggregateTilesFlag_ = aggregateTilesFlag;
+}
+
+bool FlowcellInfo::isPatternedFlowcell() const
+{
+ return patternedFlowcell_;
+}
+
+void FlowcellInfo::setPatternedFlowcell()
+{
+ patternedFlowcell_ = true;
+}
+
+void FlowcellInfo::resetPatternedFlowcell()
+{
+ patternedFlowcell_ = false;
+}
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/layout/LaneInfo.cpp b/src/cxx/lib/layout/LaneInfo.cpp
new file mode 100644
index 0000000..556be99
--- /dev/null
+++ b/src/cxx/lib/layout/LaneInfo.cpp
@@ -0,0 +1,112 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file LaneInfo.cpp
+ *
+ * \brief Implementation of lane metadata.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <sstream>
+#include <iterator>
+#include <numeric>
+#include <utility>
+
+#include <boost/assign.hpp>
+#include <boost/format.hpp>
+#include <boost/foreach.hpp>
+
+#include "common/Debug.hh"
+#include "layout/LaneInfo.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+LaneInfo::LaneInfo(common::LaneNumber number)
+: number_(number)
+, sampleInfos_()
+, tileInfos_()
+, readInfos_()
+, minimumTrimmedReadLength_(0)
+{
+}
+
+void LaneInfo::maskBarcode(common::ReadNumber indexNumber)
+{
+ for (auto& sampleInfo : sampleInfos_)
+ {
+ sampleInfo.maskBarcode(indexNumber);
+ }
+}
+
+void LaneInfo::removeDefaultSample()
+{
+ sampleInfos_.erase(
+ std::remove_if(sampleInfos_.begin(),
+ sampleInfos_.end(),
+ [](const SampleInfo& sample) { return sample.getNumber() == 0; }));
+}
+
+void LaneInfo::sortSampleInfos()
+{
+ std::sort(sampleInfos_.begin(),
+ sampleInfos_.end(),
+ [] (const SampleInfo& sample1,
+ const SampleInfo& sample2)
+ { return sample1.getNumber() < sample2.getNumber(); });
+}
+
+common::ClustersCount LaneInfo::getClustersCount() const
+{
+ return std::accumulate(
+ tileInfos_.begin(),
+ tileInfos_.end(),
+ common::ClustersCount(0),
+ boost::bind(
+ std::plus<common::ClustersCount>(),
+ _1,
+ boost::bind(&TileInfo::getClustersCount, _2)
+ )
+ );
+}
+
+bool LaneInfo::haveClustersCount() const
+{
+ for (const auto &tileInfo : tileInfos_)
+ {
+ if (!tileInfo.haveClustersCount())
+ {
+ return false;
+ }
+ }
+ return true;
+}
+
+common::CycleNumber LaneInfo::getNumCyclesToLoad() const
+{
+ return std::accumulate(
+ readInfos_.begin(),
+ readInfos_.end(),
+ common::CycleNumber(0),
+ [](common::CycleNumber sum, const layout::ReadInfo& elem) { return sum + elem.getNumCyclesToLoad(); });
+}
+
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/layout/Layout.cpp b/src/cxx/lib/layout/Layout.cpp
new file mode 100644
index 0000000..77baa78
--- /dev/null
+++ b/src/cxx/lib/layout/Layout.cpp
@@ -0,0 +1,1246 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file Layout.cpp
+ *
+ * \brief Implementation of data layout.
+ *
+ * \author Marek Balint
+ * \author Mauricio Varea
+ */
+
+
+#include <utility>
+#include <numeric>
+#include <sstream>
+
+#include <boost/bind.hpp>
+#include <boost/foreach.hpp>
+#include <boost/format.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/regex.hpp>
+#include <boost/algorithm/string.hpp>
+#include <boost/filesystem/operations.hpp>
+#include <boost/filesystem/fstream.hpp>
+
+#include "common/Logger.hh"
+#include "common/Types.hh"
+#include "layout/Layout.hh"
+#include "layout/UseBasesMask.hh"
+#include "layout/RunInfoXml.hh"
+#include "config/SampleSheetCsv.hh"
+#include "layout/ConfigXml.hh"
+#include "layout/BCIndex.hh"
+#include "common/DirectoryValidator.hh"
+#include "layout/BarcodeCollisionDetector.hh"
+#include "layout/FileExistenceVerifier.hh"
+#include "data/CbclFile.hh"
+#include "io/SyncFile.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+typedef std::map<std::string, common::SampleNumber> SampleIdNumberMap;
+
+const FlowcellInfo & Layout::getFlowcellInfo() const
+{
+ return flowcellInfo_;
+}
+
+LaneInfosContainer::const_iterator Layout::laneInfosBegin() const
+{
+ return laneInfos_.begin();
+}
+
+LaneInfosContainer::const_iterator Layout::laneInfosEnd() const
+{
+ return laneInfos_.end();
+}
+
+bool includeTile(const std::string& flowcell,
+ common::LaneNumber laneNumber,
+ common::TileNumber tileNumber,
+ const std::vector<boost::regex>& tileRegexps,
+ const config::SampleSheetCsv& sampleSheet)
+{
+ if (tileRegexps.empty())
+ {
+ return !sampleSheet.isTileExcluded(flowcell, laneNumber, tileNumber);
+ }
+ else
+ {
+ std::string tileString((boost::format("s_%d_%04d") % laneNumber % tileNumber).str());
+ BOOST_FOREACH(const boost::regex &re, tileRegexps)
+ {
+ if (boost::regex_search(tileString, re))
+ {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+bool checkExistence(const boost::filesystem::path& laneDir,
+ const std::string& fileExt,
+ std::vector<boost::filesystem::path>& files,
+ bool findOne = true)
+{
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Checking if there are any " << fileExt << " files in: " << laneDir << std::endl;
+
+ boost::regex pattern(".*\\." + fileExt + "$", boost::regex_constants::mod_x);
+ if( !boost::filesystem::is_directory( laneDir )) return false;
+ for( boost::filesystem::directory_iterator it(laneDir), end; it != end; ++it )
+ {
+ boost::smatch match;
+ if( boost::filesystem::is_regular_file( it->status() )
+ && boost::regex_match( it->path().leaf().string(), match, pattern ))
+ {
+ if (findOne)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "\t... found " << it->path() << std::endl;
+ return true;
+ }
+
+ files.push_back(it->path());
+ }
+ }
+
+ if (!files.empty())
+ {
+ return true;
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "\t... no " << fileExt << " files found!" << std::endl;
+ return false;
+}
+
+bool checkBciExistence(const boost::filesystem::path& laneDir)
+{
+ std::vector<boost::filesystem::path> unused;
+ return checkExistence(laneDir, "bci", unused);
+}
+
+bool checkCbclExistence(const boost::filesystem::path& laneDir)
+{
+ std::vector<boost::filesystem::path> unused;
+ return checkExistence(laneDir / "C1.1", "cbcl", unused);
+}
+
+enum AdapterType
+{
+ TRIM,
+ MASK
+};
+
+unsigned int addAdaptersToRead( config::SampleSheetCsv::AdaptersContainer::const_iterator adaptersBegin,
+ config::SampleSheetCsv::AdaptersContainer::const_iterator adaptersEnd,
+ ReadInfo &readInfo,
+ AdapterType adapterType)
+{
+ BCL2FASTQ_ASSERT_MSG(adapterType == TRIM || adapterType == MASK, "Unrecognized adapter type");
+
+ unsigned int adapterCount = 0;
+ bool haveNonZeroReadAdapter = false;
+ BOOST_FOREACH (const config::SampleSheetCsv::AdaptersContainer::value_type &adapter, std::make_pair(adaptersBegin, adaptersEnd))
+ {
+ if (adapter.first != 0)
+ {
+ haveNonZeroReadAdapter = true;
+ }
+ }
+ BOOST_FOREACH (const config::SampleSheetCsv::AdaptersContainer::value_type &adapter, std::make_pair(adaptersBegin, adaptersEnd))
+ {
+ bool acceptAdapter = false;
+ if(!haveNonZeroReadAdapter)
+ {
+ acceptAdapter = true;
+ }
+ else if (adapter.first == readInfo.getNumber())
+ {
+ acceptAdapter = true;
+ }
+ else if((adapter.first == 0) && (readInfo.getNumber() == 1))
+ {
+ acceptAdapter = true;
+ }
+
+ if (acceptAdapter)
+ {
+ std::string adapterTypeStr;
+ if (adapterType == MASK)
+ {
+ readInfo.addMaskAdapter(adapter.second);
+ adapterTypeStr = "mask";
+ }
+ else if (adapterType == TRIM)
+ {
+ readInfo.addTrimAdapter(adapter.second);
+ adapterTypeStr = "trim";
+ }
+
+ ++adapterCount;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << " Adapter: " << adapter.second << (boost::format(" (%s adapter)") % adapterTypeStr).str() << std::endl;
+ }
+ }
+ return adapterCount;
+}
+
+void validateBarcodes(const LaneInfo& laneInfo,
+ BarcodeCollisionDetector& barcodeCollisionDetector)
+{
+ if (laneInfo.getSampleInfos().size() == 1)
+ {
+ BCL2FASTQ_ASSERT_MSG(laneInfo.getSampleInfos().front().getBarcodes().empty(),
+ "Internal programming error. Barcode exists for lane with only 1 sample");
+
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << " Sample: " << laneInfo.getSampleInfos().front() << std::endl;
+
+ return;
+ }
+
+ for (const auto& sampleInfo : laneInfo.getSampleInfos())
+ {
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << " Sample: " << sampleInfo << std::endl;
+
+ if (sampleInfo.getNumber() == 0)
+ {
+ // No barcodes for the default sample
+ continue;
+ }
+
+ for (const auto& barcode : sampleInfo.getBarcodes())
+ {
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << " Barcode: '" << barcode << "'" << std::endl;
+ }
+
+ barcodeCollisionDetector.validateBarcode(sampleInfo.getBarcodes());
+ }
+}
+
+void Layout::detectFlowcellInfo(const boost::filesystem::path& intensitiesDir,
+ const RunInfoXml& runInfoXml,
+ bool hasBci,
+ bool hasCbcl)
+{
+ flowcellInfo_.setInstrument(runInfoXml.getInstrument());
+ flowcellInfo_.setRunNumber(runInfoXml.getRunNumber());
+ flowcellInfo_.setFlowcellId(runInfoXml.getFlowcellId());
+ flowcellInfo_.setRunId(runInfoXml.getRunId());
+
+ if( boost::filesystem::exists( intensitiesDir / "s.locs" ) )
+ {
+ flowcellInfo_.setPatternedFlowcell();
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Patterned flowcell detected" << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "This may be a 'HiSeq X Ten' run" << std::endl;
+ } else {
+ flowcellInfo_.resetPatternedFlowcell();
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "NO patterned flowcell detected" << std::endl;
+ }
+
+ if( hasBci )
+ {
+ flowcellInfo_.setAggregateTilesFlag(common::TileAggregationMode::AGGREGATED);
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "This may be a 'NextSeq 500' run" << std::endl;
+ }
+ else if ( hasCbcl )
+ {
+ flowcellInfo_.setAggregateTilesFlag(common::TileAggregationMode::CBCL);
+ }
+ else
+ {
+ flowcellInfo_.setAggregateTilesFlag(common::TileAggregationMode::NON_AGGREGATED);
+ }
+}
+
+void parseUseBasesMasks(const std::vector<std::string>& useBasesMaskStrs,
+ std::map<common::LaneNumber, std::string>& useBasesMasksForLane,
+ std::string& defaultUseBasesMask)
+{
+ BOOST_FOREACH(const std::string& useBasesMask, useBasesMaskStrs)
+ {
+ std::vector<std::string> useBasesMaskAndLane;
+ boost::split(useBasesMaskAndLane, useBasesMask, boost::is_any_of(":"));
+
+ if (useBasesMaskAndLane.size() == 1)
+ {
+ BCL2FASTQ_ASSERT_MSG(defaultUseBasesMask.empty(), "Multiple use-bases-mask paramters entered without specifying lane number.");
+
+ defaultUseBasesMask = useBasesMask;
+ }
+ else if (useBasesMaskAndLane.size() == 2)
+ {
+ boost::algorithm::trim<std::string>(useBasesMaskAndLane[0]);
+ boost::algorithm::trim<std::string>(useBasesMaskAndLane[1]);
+
+ useBasesMasksForLane.insert(std::make_pair(boost::lexical_cast<unsigned int>(useBasesMaskAndLane[0]),
+ useBasesMaskAndLane[1]));
+ }
+ else
+ {
+ BCL2FASTQ_ASSERT_MSG(false, "Invalid use-bases-mask: " + useBasesMask);
+ }
+ }
+}
+
+void setReadMetadata(common::CycleNumber startCycle,
+ common::CycleNumber endCycle,
+ layout::ReadMetadata& readMetadata)
+{
+ if (endCycle >= 1)
+ {
+ readMetadata.lastCycle_ = readMetadata.firstCycle_ + (endCycle-1);
+ }
+ if (startCycle >= 1)
+ {
+ readMetadata.firstCycle_ += (startCycle-1);
+ }
+}
+
+void setReadMetadata(common::CycleNumber startCycle,
+ common::CycleNumber endCycle,
+ common::CycleNumber umiStartFromCycle,
+ common::CycleNumber umiLength,
+ bool trimUmi,
+ layout::ReadMetadata& readMetadata)
+{
+ common::CycleNumber cycleRange = readMetadata.lastCycle_ - readMetadata.firstCycle_ + 1;
+ if (startCycle == 0)
+ {
+ startCycle = 1;
+ }
+ if (endCycle == 0)
+ {
+ endCycle = cycleRange;
+ }
+
+ if (startCycle < 1 ||
+ startCycle > endCycle ||
+ startCycle > cycleRange ||
+ endCycle > cycleRange )
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Specified start/end cycles in read are outside of the range found in RunInfo.xml"));
+ }
+
+
+ if (umiStartFromCycle > startCycle &&
+ endCycle + 1 > umiStartFromCycle + umiLength)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError(
+ "A UMI in the center of a read is not supported."));
+ }
+
+ if (umiStartFromCycle > readMetadata.lastCycle_)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError(
+ "UMI start cycle is after the last cycle in the read."));
+ }
+
+ if (trimUmi)
+ {
+ common::CycleNumber umiEndCycle = umiStartFromCycle + umiLength - 1;
+ if (umiEndCycle >= startCycle &&
+ umiStartFromCycle <= endCycle)
+ {
+ if (umiStartFromCycle <= startCycle)
+ {
+ startCycle = umiEndCycle + 1;
+ }
+ else
+ {
+ endCycle = umiStartFromCycle - 1;
+ }
+ }
+ }
+
+ setReadMetadata(startCycle,
+ endCycle,
+ readMetadata);
+}
+
+void setIndexReadMetadata(const std::vector<size_t>& barcodeLengths,
+ layout::ReadMetadata& readMetadata,
+ size_t indexNumber)
+{
+ if (barcodeLengths.size() > indexNumber)
+ {
+ if (barcodeLengths[indexNumber] > 1 + readMetadata.lastCycle_ - readMetadata.firstCycle_)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError((boost::format(
+ "Barcodes in sample sheet are longer than the index length found in RunInfo.xml.")).str()));
+ }
+
+ if (barcodeLengths[indexNumber] == 0)
+ {
+ // Mask the read
+ readMetadata.firstCycle_ = 1;
+ readMetadata.lastCycle_ = 0;
+ }
+ else
+ {
+ setReadMetadata(0,
+ barcodeLengths[indexNumber],
+ readMetadata);
+ }
+ }
+}
+
+void validateIndexLengths(const layout::ReadMetadataContainer& readMetadataContainer,
+ const std::vector<size_t>& barcodeLengths,
+ layout::LaneInfo& laneInfo)
+{
+ std::vector<std::pair<size_t,size_t>> indexNumberLengthPairs;
+ for (const auto& readMetadata : readMetadataContainer)
+ {
+ if (readMetadata.readType_ == common::ReadType::INDEX)
+ {
+ indexNumberLengthPairs.push_back(std::make_pair(readMetadata.lastCycle_ - readMetadata.firstCycle_ + 1, readMetadata.readNumber_));
+ }
+ }
+
+ for (const std::pair<size_t,size_t> indexNumberLengthPair : indexNumberLengthPairs)
+ {
+ if (indexNumberLengthPair.second > 2)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Greater than 2 index reads encountered in use-bases-mask. This is not allowed."));
+ }
+
+ if ((indexNumberLengthPairs.size() == 2 && barcodeLengths.at(indexNumberLengthPair.second - 1) != indexNumberLengthPair.first) ||
+ (barcodeLengths[0] != indexNumberLengthPair.first && barcodeLengths[1] != indexNumberLengthPair.first))
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Barcode lengths in the sample sheet do not match those in --use-bases-mask"));
+ }
+ }
+
+ if (indexNumberLengthPairs.empty())
+ {
+ // There are no index sequences
+ if (barcodeLengths[0] != 0 && barcodeLengths[1] != 0)
+ {
+ laneInfo.maskBarcode(2);
+ laneInfo.maskBarcode(1);
+ }
+ else if (barcodeLengths[0] != 0 || barcodeLengths[1] != 0)
+ {
+ laneInfo.maskBarcode(1);
+ }
+
+ laneInfo.removeDefaultSample();
+ if (laneInfo.getSampleInfos().size() > 1)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError((boost::format(
+ "Multiple samples with all barcodes masked for lane: %d") % laneInfo.getNumber()).str()));
+ }
+ }
+ else if ((indexNumberLengthPairs.size() == 1 && barcodeLengths.size() == 2) &&
+ (barcodeLengths[0] != 0 && barcodeLengths[1] != 0))
+ {
+ laneInfo.maskBarcode(indexNumberLengthPairs[0].second == 1 ? 2 : 1);
+ }
+}
+
+void getReadMetadata(const RunInfoXml& runInfoXml,
+ const std::vector<size_t>& barcodeLengths,
+ const std::string& useBasesMaskStr,
+ common::CycleNumber read1StartCycle,
+ common::CycleNumber read2StartCycle,
+ common::CycleNumber read1EndCycle,
+ common::CycleNumber read2EndCycle,
+ common::CycleNumber read1UmiLength,
+ common::CycleNumber read2UmiLength,
+ common::CycleNumber read1UmiStartFromCycle,
+ common::CycleNumber read2UmiStartFromCycle,
+ bool trimUmi,
+ layout::LaneInfo& laneInfo,
+ layout::ReadMetadataContainer& readMetadataContainer)
+{
+ readMetadataContainer =
+ layout::ReadMetadataContainer(runInfoXml.readMetadataBegin(),
+ runInfoXml.readMetadataEnd());
+
+ size_t indexNumber = 0;
+ common::ReadNumber readNumber = 1;
+ for (ReadMetadata& readMetadata : readMetadataContainer)
+ {
+ if (readMetadata.readType_ == common::ReadType::INDEX)
+ {
+ setIndexReadMetadata(barcodeLengths,
+ readMetadata,
+ indexNumber++);
+ }
+ else
+ {
+ if (readNumber == 1)
+ {
+ setReadMetadata(read1StartCycle,
+ read1EndCycle,
+ read1UmiStartFromCycle,
+ read1UmiLength,
+ trimUmi,
+ readMetadata);
+ }
+ else if (readNumber == 2)
+ {
+ setReadMetadata(read2StartCycle,
+ read2EndCycle,
+ read2UmiStartFromCycle,
+ read2UmiLength,
+ trimUmi,
+ readMetadata);
+ }
+
+ ++readNumber;
+ }
+ }
+
+ if (!useBasesMaskStr.empty())
+ {
+ if (read1UmiLength != 0 || read2UmiLength != 0)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Use of --use-bases-mask is not supported with a UMI sequence entered in the sample sheet."));
+ }
+
+ // useBasesMask overrides the data from RunInfoXml
+ layout::UseBasesMask useBasesMask(useBasesMaskStr,
+ runInfoXml.readMetadataBegin(),
+ runInfoXml.readMetadataEnd());
+
+ if (!useBasesMask.isEmpty())
+ {
+ readMetadataContainer = layout::ReadMetadataContainer(useBasesMask.readMetadataBegin(),
+ useBasesMask.readMetadataEnd());
+
+ if (!barcodeLengths.empty())
+ {
+ validateIndexLengths(readMetadataContainer,
+ barcodeLengths,
+ laneInfo);
+ }
+ }
+ }
+}
+
+void detectReadLayout(const RunInfoXml& runInfoXml,
+ const config::SampleSheetCsv& sampleSheetCsv,
+ const std::vector<std::string>& useBasesMaskStrs,
+ size_t minimumTrimmedReadLength,
+ common::CycleNumber read1StartCycle,
+ common::CycleNumber read2StartCycle,
+ common::CycleNumber read1EndCycle,
+ common::CycleNumber read2EndCycle,
+ common::CycleNumber read1UmiLength,
+ common::CycleNumber read2UmiLength,
+ common::CycleNumber read1UmiStartFromCycle,
+ common::CycleNumber read2UmiStartFromCycle,
+ bool trimUmi,
+ bool autoSetToZeroMismatches,
+ config::BarcodeMismatchCountsContainer& componentMaxMismatches,
+ LaneInfosContainer& lanes,
+ const std::vector<std::vector<size_t>>& barcodeLengthsForLane)
+{
+ std::map<common::LaneNumber, std::string> useBasesMasksForLane;
+ std::string defaultUseBasesMask;
+ parseUseBasesMasks(useBasesMaskStrs,
+ useBasesMasksForLane,
+ defaultUseBasesMask);
+
+ for (LaneInfo& laneInfo : lanes)
+ {
+ BarcodeCollisionDetector barcodeCollisionDetector(componentMaxMismatches,
+ autoSetToZeroMismatches);
+
+ std::string useBasesMaskStr(defaultUseBasesMask);
+ std::map<common::LaneNumber, std::string>::const_iterator pos = useBasesMasksForLane.find(laneInfo.getNumber());
+ if (pos != useBasesMasksForLane.end())
+ {
+ useBasesMaskStr = pos->second;
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Lane: " << laneInfo.getNumber() << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Mask: " << useBasesMaskStr << std::endl;
+
+ unsigned int totalAdapters = 0;
+ size_t minimumNonIndexReadLength = minimumTrimmedReadLength;
+
+ layout::ReadMetadataContainer readMetadataContainer;
+ getReadMetadata(runInfoXml,
+ barcodeLengthsForLane[laneInfo.getNumber()-1],
+ useBasesMaskStr,
+ read1StartCycle,
+ read2StartCycle,
+ read1EndCycle,
+ read2EndCycle,
+ read1UmiLength,
+ read2UmiLength,
+ read1UmiStartFromCycle,
+ read2UmiStartFromCycle,
+ trimUmi,
+ laneInfo,
+ readMetadataContainer);
+
+ validateBarcodes(laneInfo, barcodeCollisionDetector);
+
+ for (const auto& readMetadata : readMetadataContainer)
+ {
+ // read
+ ReadInfo readInfo(readMetadata.readNumber_,
+ readMetadata.readType_,
+ common::CycleRange(readMetadata.firstCycle_, readMetadata.lastCycle_),
+ common::CycleRange(readMetadata.firstUnmaskedCycle_, readMetadata.lastUnmaskedCycle_),
+ readMetadata.readNumber_ == 1 ?
+ common::CycleRange(read1UmiStartFromCycle, read1UmiLength) :
+ common::CycleRange(read2UmiStartFromCycle, read2UmiLength));
+
+ // adapters
+ if (readInfo.isDataRead())
+ {
+ size_t readLength = readMetadata.lastCycle_ - readMetadata.firstCycle_ + 1;
+
+ if (readLength < minimumNonIndexReadLength && readLength != 0)
+ {
+ minimumNonIndexReadLength = readLength;
+ }
+
+ totalAdapters += addAdaptersToRead(sampleSheetCsv.maskAdaptersBegin(), sampleSheetCsv.maskAdaptersEnd(), readInfo, MASK);
+ totalAdapters += addAdaptersToRead(sampleSheetCsv.trimAdaptersBegin(), sampleSheetCsv.trimAdaptersEnd(), readInfo, TRIM);
+ }
+
+ laneInfo.addRead(readInfo);
+ }
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Total number of adapters: " << totalAdapters << std::endl;
+
+ if (minimumTrimmedReadLength > minimumNonIndexReadLength)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::WARNING) << "Option: '--minimum-trimmed-read-length' with value: "
+ << minimumTrimmedReadLength
+ << " is being overwritten by the shortest non-index read length: "
+ << minimumNonIndexReadLength << std::endl;
+
+ minimumTrimmedReadLength = minimumNonIndexReadLength;
+ }
+
+ laneInfo.setMinimumTrimmedReadLength(minimumTrimmedReadLength);
+ }
+}
+
+typedef std::map<std::string, SampleInfo> SampleMap;
+
+const std::string& getSampleName(const std::string& sampleName, const std::string& sampleId)
+{
+ return sampleName.empty() ?
+ (sampleId.empty() ? SampleInfo::defaultName : sampleId) :
+ sampleName;
+}
+
+void checkForDuplicateSample(const std::string& sampleId,
+ const SampleInfo& sample1,
+ const SampleInfo& sample2)
+{
+ if (sample1.getProject() != sample2.getProject())
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError((boost::format(
+ "Only 1 project is allowed for a given sampleId. SampleId: '" + sampleId + "'.")).str()));
+ }
+ if (sample1.getName() != sample2.getName())
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError((boost::format(
+ "Only 1 sample name is allowed for a given sampleId. SampleId: '" + sampleId + "'.")).str()));
+ }
+}
+
+void handleDuplicateSampleId(SampleMap& sampleMap,
+ SampleInfo& sample)
+{
+ std::pair<SampleMap::iterator, bool> pos = sampleMap.insert(std::make_pair(sample.getId(), sample));
+ if (!pos.second)
+ {
+ // Duplicate sample id.
+ sample.setNumber(pos.first->second.getNumber());
+
+ checkForDuplicateSample(sample.getId(), sample, pos.first->second);
+ }
+}
+
+void handleDefaultSample(bool hasBarcodes,
+ bool hasNoSamples,
+ size_t numSamplesInLane,
+ LaneInfo& laneInfo,
+ bool& sampleWithoutBarcodesInLane)
+{
+ if (hasBarcodes)
+ {
+ if (hasNoSamples)
+ {
+ SampleInfo defaultSample;
+ laneInfo.addSample(defaultSample);
+ }
+ }
+ else
+ {
+ sampleWithoutBarcodesInLane = true;
+ }
+
+ if (sampleWithoutBarcodesInLane && numSamplesInLane != 1)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError((boost::format("Missing a barcode in lane %d") % laneInfo.getNumber()).str()));
+ }
+}
+
+void addBarcodes(SampleInfo& sampleInfo,
+ const common::SampleMetadata::BarcodesContainer& barcodes,
+ std::vector<size_t>& barcodeLengths)
+{
+ for (const auto& barcodeComponents : barcodes)
+ {
+ common::SampleMetadata::BarcodesContainer::value_type nonEmptyBarcodeComponents;
+ for (const auto& component : barcodeComponents)
+ {
+ if (!component.empty())
+ {
+ nonEmptyBarcodeComponents.push_back(component);
+ }
+ }
+ Barcode barcode (nonEmptyBarcodeComponents.begin(), nonEmptyBarcodeComponents.end());
+ sampleInfo.addBarcode(barcode);
+ }
+
+ if (barcodeLengths.empty() && !barcodes.empty())
+ {
+ for (const auto& component : barcodes.front())
+ {
+ barcodeLengths.push_back(component.size());
+ }
+ }
+}
+
+void addSamplesForLane(const common::SampleMetadataContainer& sampleMetadataContainer,
+ SampleMap& sampleMap,
+ SampleIdNumberMap& sampleIdNumberMap,
+ LaneInfo& laneInfo,
+ std::vector<size_t>& barcodeLengths)
+{
+ common::LaneNumber laneNumber = laneInfo.getNumber();
+
+ size_t numSamplesInLane = 0;
+ bool sampleWithoutBarcodesInLane = false;
+ std::map<common::SampleNumber, size_t> laneInfoSampleMap;
+ SampleMap sampleMapForLane;
+ for (const auto& sampleMetadata : sampleMetadataContainer)
+ {
+ if (!sampleMetadata.lanes_.empty() &&
+ std::find(sampleMetadata.lanes_.begin(), sampleMetadata.lanes_.end(), laneNumber) == sampleMetadata.lanes_.end())
+ {
+ // SampleSheet.csv specified a lane for this sample, and this lane is not it.
+ continue;
+ }
+
+ ++numSamplesInLane;
+
+ bool hasBarcodes = !sampleMetadata.barcodes_.empty();
+ std::string sampleId = sampleMetadata.id_.empty() ? SampleInfo::defaultId : sampleMetadata.id_;
+ std::string sampleName = getSampleName(sampleMetadata.name_, sampleMetadata.id_);
+
+ handleDefaultSample(hasBarcodes,
+ laneInfo.sampleInfosBegin() == laneInfo.sampleInfosEnd(),
+ numSamplesInLane,
+ laneInfo,
+ sampleWithoutBarcodesInLane);
+
+ SampleInfo sample(sampleIdNumberMap[sampleId],
+ sampleId,
+ sampleName,
+ sampleMetadata.project_.empty() ? SampleInfo::defaultProject : sampleMetadata.project_);
+
+ handleDuplicateSampleId(sampleMap,
+ sample);
+
+ auto pos = sampleMapForLane.insert(std::make_pair(sample.getId(), sample));
+ addBarcodes(pos.first->second,
+ sampleMetadata.barcodes_,
+ barcodeLengths);
+ }
+
+ for (const auto& keyValue : sampleMapForLane)
+ {
+ laneInfo.addSample(keyValue.second);
+ }
+
+ // Sort by sample number to improve readability for output
+ laneInfo.sortSampleInfos();
+}
+
+void getSampleNumbers(const common::SampleMetadataContainer& sampleMetadataContainer,
+ SampleIdNumberMap& sampleIdNumberMap)
+{
+ common::SampleNumber sampleNumber = 0;
+ for (const auto& sampleMetadata : sampleMetadataContainer)
+ {
+ common::SampleNumber& sampleNumberForId = sampleIdNumberMap[sampleMetadata.id_];
+ if (sampleNumberForId == 0)
+ {
+ sampleNumberForId = ++sampleNumber;
+ }
+ }
+}
+void detectLaneLayout(const layout::RunInfoXml& runInfoXml,
+ const config::SampleSheetCsv& sampleSheetCsv,
+ const boost::filesystem::path& inputDir,
+ bool hasConfigXml,
+ bool includeNonPfClusters,
+ bool& hasBci,
+ bool& hasCbcl,
+ LaneInfosContainer& lanes,
+ std::vector<std::vector<size_t>>& barcodeLengthsForLane)
+{
+ SampleIdNumberMap sampleIdNumberMap;
+ getSampleNumbers(sampleSheetCsv.getSampleMetadata(),
+ sampleIdNumberMap);
+
+ SampleMap sampleMap;
+
+ const common::LaneNumber lanesCount = runInfoXml.getLanesCount();
+ for (common::LaneNumber laneNumber = 1; laneNumber <= lanesCount; ++laneNumber)
+ {
+ // lane
+ LaneInfo laneInfo(laneNumber);
+
+ barcodeLengthsForLane.resize(barcodeLengthsForLane.size() + 1);
+ if (sampleSheetCsv.getSampleMetadata().empty())
+ {
+ // Add a default sample
+ laneInfo.addSample(SampleInfo());
+ }
+ else
+ {
+ addSamplesForLane(sampleSheetCsv.getSampleMetadata(),
+ sampleMap,
+ sampleIdNumberMap,
+ laneInfo,
+ barcodeLengthsForLane.back());
+
+ }
+
+ if (laneInfo.sampleInfosBegin() != laneInfo.sampleInfosEnd())
+ {
+ hasBci = hasBci || checkBciExistence( inputDir / laneInfo.getDirName() );
+ hasCbcl = hasCbcl || checkCbclExistence( inputDir / laneInfo.getDirName() );
+
+ if (hasCbcl && includeNonPfClusters)
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Cannot specify '--with-failed-reads' with cbcl input data. Failed reads are not stored in cbcl files."));
+ }
+
+ lanes.push_back(laneInfo);
+ }
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Has BCI: " << hasBci << std::endl;
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "Has 'config.xml': " << hasConfigXml << std::endl;
+}
+
+void readTilesFromRunInfoXmlForLane(const layout::RunInfoXml& runInfoXml,
+ const config::SampleSheetCsv& sampleSheet,
+ const std::vector<boost::regex>& tileRegexps,
+ common::LaneNumber laneNumber,
+ LaneInfo& laneInfo)
+{
+ size_t skippedTiles = 0;
+ size_t tileIndex = 0;
+ BOOST_FOREACH (const common::TileNumber &tileNumber,
+ std::make_pair( runInfoXml.tileNumbersBegin( laneNumber ),
+ runInfoXml.tileNumbersEnd( laneNumber )) )
+ {
+ if ( includeTile(runInfoXml.getFlowcellId(), laneNumber, tileNumber, tileRegexps, sampleSheet) )
+ {
+ TileInfo tileInfo(tileNumber, tileIndex, skippedTiles);
+ laneInfo.addTile(tileInfo);
+ skippedTiles = 0;
+ }
+ else
+ {
+ ++skippedTiles;
+ }
+ ++tileIndex;
+ }
+}
+
+void readTilesFromRunInfoXml(const layout::RunInfoXml& runInfoXml,
+ const config::SampleSheetCsv& sampleSheet,
+ const std::vector<boost::regex>& tileRegexps,
+ std::vector<LaneInfo>& lanes)
+{
+ BOOST_FOREACH (LaneInfo &laneInfo, std::make_pair(lanes.begin(), lanes.end()))
+ {
+ common::LaneNumber laneNumber = laneInfo.getNumber();
+ BCL2FASTQ_ASSERT_MSG( 1 <= laneNumber && laneNumber <= runInfoXml.getLanesCount(), "Lane '" << laneNumber << "' is out of range");
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Lane: " << laneNumber << std::endl;
+
+ readTilesFromRunInfoXmlForLane(runInfoXml,
+ sampleSheet,
+ tileRegexps,
+ laneNumber,
+ laneInfo);
+ }
+}
+
+void readTilesFromBciFile(const boost::filesystem::path& inputDir,
+ const config::SampleSheetCsv& sampleSheet,
+ const std::vector<boost::regex>& tileRegexps,
+ const std::string& flowcellId,
+ std::vector<LaneInfo>& lanes)
+{
+ BOOST_FOREACH (LaneInfo &laneInfo, std::make_pair(lanes.begin(), lanes.end()))
+ {
+ size_t skippedTiles = 0;
+ size_t skippedClusters = 0;
+ size_t tileIndex = 0;
+ common::LaneNumber laneNumber = laneInfo.getNumber();
+ if (checkBciFile(inputDir, laneNumber))
+ {
+ const BCIndex bcIndex = createBCIndex(inputDir, laneInfo.getNumber());
+ BOOST_FOREACH (const BCIndex::TileMetadata &tileMetadata, std::make_pair(bcIndex.tileMetadataBegin(), bcIndex.tileMetadataEnd()))
+ {
+ if ( includeTile(flowcellId, laneNumber, tileMetadata.tileNumber_, tileRegexps, sampleSheet) &&
+ tileMetadata.clustersCount_ > 0)
+ {
+ TileInfo tileInfo(tileMetadata.tileNumber_, tileIndex, tileMetadata.clustersCount_, skippedTiles, skippedClusters);
+ laneInfo.addTile(tileInfo);
+ skippedTiles = 0;
+ skippedClusters = 0;
+ }
+ else
+ {
+ ++skippedTiles;
+ skippedClusters += tileMetadata.clustersCount_;
+ }
+ ++tileIndex;
+ }
+ }
+ }
+}
+
+void readTilesFromConfigXmlOrRunInfoXml(const boost::filesystem::path& inputDir,
+ const layout::RunInfoXml& runInfoXml,
+ const config::SampleSheetCsv& sampleSheet,
+ const std::vector<boost::regex>& tileRegexps,
+ std::vector<LaneInfo>& lanes)
+{
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "This may be a 'HiSeq 2000' / 'MiSeq' run" << std::endl;
+ const ConfigXml configXml = createConfigXml(inputDir);
+ BOOST_FOREACH (LaneInfo &laneInfo, std::make_pair(lanes.begin(), lanes.end()))
+ {
+ size_t skippedTiles = 0;
+ size_t tileIndex = 0;
+ common::LaneNumber laneNumber = laneInfo.getNumber();
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Lane: " << laneNumber << std::endl;
+
+ if (configXml.containsLaneNumber(laneNumber))
+ {
+ BOOST_FOREACH (const ConfigXml::TileMetadata &tileMetadata,
+ std::make_pair(configXml.tileMetadataBegin(laneInfo.getNumber()),
+ configXml.tileMetadataEnd(laneInfo.getNumber())) )
+ {
+ if ( includeTile(runInfoXml.getFlowcellId(), laneNumber, tileMetadata.tileNumber_, tileRegexps, sampleSheet) )
+ {
+ TileInfo tileInfo(tileMetadata.tileNumber_, tileIndex, skippedTiles);
+ laneInfo.addTile(tileInfo);
+ skippedTiles = 0;
+ }
+ else
+ {
+ ++skippedTiles;
+ }
+ ++tileIndex;
+ }
+ }
+ else
+ {
+ readTilesFromRunInfoXmlForLane(runInfoXml,
+ sampleSheet,
+ tileRegexps,
+ laneNumber,
+ laneInfo);
+ }
+ }
+}
+
+void readTilesFromCbclHeaders(const boost::filesystem::path& inputDir,
+ const config::SampleSheetCsv& sampleSheet,
+ const std::vector<boost::regex>& tileRegexps,
+ const std::string& flowcellId,
+ std::vector<LaneInfo>& lanes,
+ common::TileFileMap& tileFileMap,
+ common::NumBasesPerByte& numBasesPerByte,
+ bool ignoreMissingBcls)
+{
+ tileFileMap.resize(lanes.back().getNumber() + 1);
+
+ for (auto& lane : lanes)
+ {
+ std::vector<boost::filesystem::path> files;
+ checkExistence(inputDir / lane.getDirName() / "C1.1",
+ "cbcl",
+ files,
+ false);
+
+ auto laneNumber = lane.getNumber();
+
+ std::vector<std::pair<common::TileNumber, boost::filesystem::path>> tileNumbers;
+ for (const auto& fileName : files)
+ {
+ io::SyncFile cbclFile(fileName,
+ false, // Can't ignore errors here. Need this data.
+ std::ios_base::in | std::ios_base::binary);
+
+ // Synchronization isn't really necessary here, but its safer to
+ // always require it.
+ io::SyncFile::SyncFileReader cbclFileReader(cbclFile,
+ 0);
+
+ data::CbclFileReader::Header cbclHeader;
+ if (!cbclHeader.load(cbclFileReader, false))
+ {
+ cbclHeader.reset();
+ }
+
+ // 8 bits per byte, 2 bits per basecall
+ numBasesPerByte = static_cast<common::NumBasesPerByte>(8 / (cbclHeader.getNumberOfBitsPerQscore() + 2));
+
+ for (auto tileNumber : cbclHeader.getTileIds())
+ {
+ tileNumbers.push_back(std::make_pair(tileNumber, fileName.leaf()));
+ }
+ }
+
+ std::sort(tileNumbers.begin(), tileNumbers.end());
+
+ size_t skippedTiles = 0;
+ size_t tileIndex = 0;
+ for (const auto& tileNumberFilePair : tileNumbers)
+ {
+ if (includeTile(flowcellId, laneNumber, tileNumberFilePair.first, tileRegexps, sampleSheet) )
+ {
+ TileInfo tileInfo(tileNumberFilePair.first, tileIndex, skippedTiles);
+ lane.addTile(tileInfo);
+ skippedTiles = 0;
+
+ tileFileMap[laneNumber].insert(tileNumberFilePair);
+ }
+ else
+ {
+ ++skippedTiles;
+ }
+
+ ++tileIndex;
+ }
+ }
+}
+
+void detectTileLayout(const std::vector<std::string>& tilesFilterList,
+ const layout::RunInfoXml& runInfoXml,
+ const config::SampleSheetCsv& sampleSheet,
+ const boost::filesystem::path& inputDir,
+ std::vector<LaneInfo>& lanes,
+ common::TileFileMap& tileFileMap,
+ common::NumBasesPerByte& numBasesPerByte,
+ common::TileAggregationMode flowcellAggregateTilesFlag,
+ bool hasConfigXml,
+ bool ignoreMissingBcls)
+{
+ std::vector<boost::regex> tileRegexps;
+ BOOST_FOREACH(const std::string &tileRegex, tilesFilterList)
+ {
+ tileRegexps.push_back(boost::regex(tileRegex));
+ }
+
+ switch (flowcellAggregateTilesFlag)
+ {
+ case common::TileAggregationMode::NON_AGGREGATED:
+ if( hasConfigXml )
+ {
+ readTilesFromConfigXmlOrRunInfoXml(inputDir,
+ runInfoXml,
+ sampleSheet,
+ tileRegexps,
+ lanes);
+ } else { // !hasConfigXml
+ readTilesFromRunInfoXml(runInfoXml,
+ sampleSheet,
+ tileRegexps,
+ lanes);
+ }
+ break;
+
+ case common::TileAggregationMode::AGGREGATED:
+ readTilesFromBciFile(inputDir,
+ sampleSheet,
+ tileRegexps,
+ runInfoXml.getFlowcellId(),
+ lanes);
+ break;
+ case common::TileAggregationMode::CBCL:
+ readTilesFromCbclHeaders(inputDir,
+ sampleSheet,
+ tileRegexps,
+ runInfoXml.getFlowcellId(),
+ lanes,
+ tileFileMap,
+ numBasesPerByte,
+ ignoreMissingBcls);
+ break;
+ default:
+ BCL2FASTQ_ASSERT_MSG(false, "Unrecognized tile aggregation mode");
+ }
+
+ bool hasTiles = false;
+ for (const auto& laneInfo : lanes)
+ {
+ if (!laneInfo.getTileInfos().empty())
+ {
+ hasTiles = true;
+ break;
+ }
+ }
+
+ if (!hasTiles)
+ {
+ bool first = true;
+ std::string tileFilterStr;
+ for (const auto& filter : tilesFilterList)
+ {
+ if (first)
+ {
+ first = false;
+ }
+ else
+ {
+ tileFilterStr += ", ";
+ }
+ tileFilterStr += "'" + filter + "'";
+ }
+ BOOST_THROW_EXCEPTION(common::InputDataError("No tiles were found to process." +
+ (tilesFilterList.empty() ? "" : (" Tile regular expressions used: (" + tileFilterStr + ")"))));
+ }
+}
+
+Layout::Layout(
+ const boost::filesystem::path& intensitiesDir,
+ const boost::filesystem::path& inputDir,
+ const boost::filesystem::path& outputDir,
+ const boost::filesystem::path& reportsDir,
+ const boost::filesystem::path& statsDir,
+ const config::SampleSheetCsv& sampleSheet,
+ const RunInfoXml& runInfoXml,
+ const std::vector<std::string> & tilesFilterList,
+ const std::vector<std::string>& useBasesMasks,
+ size_t minimumTrimmedReadLength,
+ bool autoSetToZeroMismatches,
+ config::BarcodeMismatchCountsContainer& componentMaxMismatches,
+ bool ignoreMissingBcls,
+ bool ignoreMissingFilters,
+ bool ignoreMissingPositions,
+ common::CycleNumber read1StartCycle,
+ common::CycleNumber read2StartCycle,
+ common::CycleNumber read1EndCycle,
+ common::CycleNumber read2EndCycle,
+ common::CycleNumber read1UmiLength,
+ common::CycleNumber read2UmiLength,
+ common::CycleNumber read1UmiStartFromCycle,
+ common::CycleNumber read2UmiStartFromCycle,
+ bool trimUmi,
+ bool includeNonPfClusters
+)
+: flowcellInfo_()
+, laneInfos_()
+, tileFileMap_()
+, numBasesPerByte_(common::NumBasesPerByte::ONE)
+{
+ common::DirectoryValidator::getSingleton().addUniquePath(reportsDir, "Reports");
+ common::DirectoryValidator::getSingleton().addUniquePath(statsDir, "Stats");
+
+ bool hasConfigXml = checkConfigXml(inputDir);
+
+ bool hasBci = false;
+ bool hasCbcl = false;
+ std::vector<LaneInfo> lanes;
+ std::vector<std::vector<size_t>> barcodeLengthsForLane;
+ detectLaneLayout(runInfoXml,
+ sampleSheet,
+ inputDir,
+ hasConfigXml,
+ includeNonPfClusters,
+ hasBci,
+ hasCbcl,
+ lanes,
+ barcodeLengthsForLane);
+
+ detectReadLayout(runInfoXml,
+ sampleSheet,
+ useBasesMasks,
+ minimumTrimmedReadLength,
+ read1StartCycle,
+ read2StartCycle,
+ read1EndCycle,
+ read2EndCycle,
+ read1UmiLength,
+ read2UmiLength,
+ read1UmiStartFromCycle,
+ read2UmiStartFromCycle,
+ trimUmi,
+ autoSetToZeroMismatches,
+ componentMaxMismatches,
+ lanes,
+ barcodeLengthsForLane);
+
+ detectFlowcellInfo(intensitiesDir,
+ runInfoXml,
+ hasBci,
+ hasCbcl);
+
+ detectTileLayout(tilesFilterList,
+ runInfoXml,
+ sampleSheet,
+ inputDir,
+ lanes,
+ tileFileMap_,
+ numBasesPerByte_,
+ flowcellInfo_.getAggregateTilesMode(),
+ hasConfigXml,
+ ignoreMissingBcls);
+
+ for (const auto& laneInfo : lanes)
+ {
+ if (laneInfo.getTileInfos().empty())
+ {
+ continue;
+ }
+
+ FileExistenceVerifier::verifyAllFilesExist(inputDir,
+ intensitiesDir,
+ laneInfo,
+ tileFileMap_,
+ flowcellInfo_.getAggregateTilesMode(),
+ flowcellInfo_.isPatternedFlowcell(),
+ ignoreMissingBcls,
+ ignoreMissingFilters,
+ ignoreMissingPositions);
+
+ laneInfos_.push_back(laneInfo);
+ }
+}
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/layout/ReadInfo.cpp b/src/cxx/lib/layout/ReadInfo.cpp
new file mode 100644
index 0000000..30e7a68
--- /dev/null
+++ b/src/cxx/lib/layout/ReadInfo.cpp
@@ -0,0 +1,157 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ReadInfo.cpp
+ *
+ * \brief Implementation of read metadata.
+ *
+ * \author Marek Balint
+ */
+
+
+#include "layout/ReadInfo.hh"
+#include "common/Exceptions.hh"
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+ReadInfo::ReadInfo(common::ReadNumber number,
+ common::ReadType readType,
+ const common::CycleRange& cyclesToUse,
+ const common::CycleRange& unmaskedCycles,
+ const common::CycleRange& umiCycles)
+: number_(number)
+, readType_(readType)
+, cycleInfos_()
+, unmaskedCycleInfos_()
+, cyclesToLoad_()
+, maskAdapters_()
+, trimAdapters_()
+, umiCycleRange_(0, 0)
+, bclBufferOffset_(0)
+{
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Read: " << number_ << " " << getReadType() << std::endl;
+
+ std::ostringstream cycles;
+ for (common::CycleNumber cycleNumber = cyclesToUse.first; cycleNumber <= cyclesToUse.second; ++cycleNumber)
+ {
+ CycleInfo cycleInfo(cycleNumber);
+ addCycle(cycleInfo);
+
+ if (!cycles.str().empty())
+ {
+ cycles << ",";
+ }
+ cycles << cycleNumber;
+ }
+ for (common::CycleNumber cycleNumber = unmaskedCycles.first; cycleNumber <= unmaskedCycles.second; ++cycleNumber)
+ {
+ addUnmaskedCycle(CycleInfo(cycleNumber));
+ }
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << " Cycles: " << cycles.str() << std::endl;
+
+ if (readType == common::ReadType::DATA)
+ {
+ addUmiBases(umiCycles);
+ }
+
+ if (!cycleInfos_.empty() && ! cyclesToLoad_.empty())
+ {
+ common::CycleNumber firstCycleToUse = cycleInfos_.front().getNumber();
+
+ for (const auto& cycleToLoad : cyclesToLoad_)
+ {
+ if (cycleToLoad.getNumber() == firstCycleToUse)
+ {
+ break;
+ }
+ ++bclBufferOffset_;
+ }
+ }
+}
+
+void ReadInfo::addUmiBases(common::CycleRange umiCycleRange)
+{
+ if (umiCycleRange.second == 0)
+ {
+ return;
+ }
+
+ // Now holds relative start position and length
+ umiCycleRange_ = umiCycleRange;
+
+ if (!unmaskedCycleInfos_.empty())
+ {
+ // The CycleRange is the first and last cycle. Both read 1 and read 2
+ // use 1 as the start number. Move to the actual cycle number for read 2.
+ umiCycleRange.first += unmaskedCycleInfos_.front().getNumber() - 1;
+ umiCycleRange.second += umiCycleRange.first - 1;
+ }
+
+ if (unmaskedCycleInfos_.empty() ||
+ umiCycleRange.first < unmaskedCycleInfos_.front().getNumber() ||
+ umiCycleRange.second > unmaskedCycleInfos_.back().getNumber())
+ {
+ BOOST_THROW_EXCEPTION(common::InputDataError("UMI bases are outside of the range of the bases for read " + boost::lexical_cast<std::string>(number_)));
+ }
+
+ if (umiCycleRange.second != 0)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "Read: " << number_ << " " << "UMI" << std::endl;
+
+ std::ostringstream cycles;
+ for (common::CycleNumber cycle = umiCycleRange.first; cycle <= umiCycleRange.second; ++cycle)
+ {
+ if (!cycles.str().empty())
+ {
+ cycles << ",";
+ }
+ cycles << cycle;
+ cyclesToLoad_.insert(layout::CycleInfo(cycle));
+ }
+
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << " Cycles: " << cycles.str() << std::endl;
+ }
+
+ // Everything up to here was validation and display.
+ // Now we store the index into the bcl buffer
+ common::CycleNumber firstUmiCycle = umiCycleRange_.first + unmaskedCycleInfos_.front().getNumber() - 1;
+ umiCycleRange_.first = 0;
+ for (const auto& cycleToLoad : cyclesToLoad_)
+ {
+ if (cycleToLoad.getNumber() == firstUmiCycle)
+ {
+ break;
+ }
+ ++umiCycleRange_.first;
+ }
+
+ umiCycleRange_.second += umiCycleRange_.first;
+
+ // umiCycleRange now holds the absolute index into the bcl buffer
+}
+
+void ReadInfo::addUnmaskedCycle(const CycleInfo &cycleInfo)
+{
+ unmaskedCycleInfos_.push_back(cycleInfo);
+ if (isIndexRead())
+ {
+ // We need to load all the cycles from BCL file for index reads
+ cyclesToLoad_.insert(cycleInfo);
+ }
+}
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/layout/RunInfoXml.cpp b/src/cxx/lib/layout/RunInfoXml.cpp
new file mode 100644
index 0000000..ad366f3
--- /dev/null
+++ b/src/cxx/lib/layout/RunInfoXml.cpp
@@ -0,0 +1,223 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file RunInfoXml.cpp
+ *
+ * \brief Implementation of RunInfo.xml helper.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <fstream>
+
+#include <boost/property_tree/xml_parser.hpp>
+#include <boost/optional.hpp>
+#include <boost/format.hpp>
+
+#include "common/Logger.hh"
+#include "common/Exceptions.hh"
+#include "io/Xml.hh"
+#include "layout/RunInfoXml.hh"
+
+
+namespace bcl2fastq {
+
+
+namespace layout {
+
+
+namespace detail {
+
+
+static const std::string runInfoXmlFileName("RunInfo.xml");
+
+
+} // namespace detail
+
+
+RunInfoXml::RunInfoXml(boost::property_tree::ptree runInfoData)
+: ptree_(runInfoData)
+, reads_()
+{
+ this->initReadMetadata();
+ this->initTileNumbers();
+}
+
+std::string RunInfoXml::getInstrument() const
+{
+ boost::optional<std::string> ret = ptree_.get_optional<std::string>("RunInfo.Run.Instrument");
+ return ret ? ret.get() : "";
+}
+
+std::string RunInfoXml::getRunNumber() const
+{
+ boost::optional<std::string> ret = ptree_.get_optional<std::string>("RunInfo.Run.<xmlattr>.Number");
+ return ret ? ret.get() : "";
+}
+
+std::string RunInfoXml::getRunId() const
+{
+ boost::optional<std::string> ret = ptree_.get_optional<std::string>("RunInfo.Run.<xmlattr>.Id");
+ return ret ? ret.get() : "";
+}
+
+std::string RunInfoXml::getFlowcellId() const
+{
+ boost::optional<std::string> ret = ptree_.get_optional<std::string>("RunInfo.Run.Flowcell");
+ return ret ? ret.get() : "";
+}
+
+common::LaneNumber RunInfoXml::getLanesCount() const
+{
+ boost::optional<common::LaneNumber> ret = ptree_.get_optional<common::LaneNumber>("RunInfo.Run.FlowcellLayout.<xmlattr>.LaneCount");
+ if(!ret) {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Required attribute 'LaneCount' of element 'RunInfo.Run.FlowcellLayout' missing in RunInfo.xml file"));
+ }
+ return ret.get();
+}
+
+ReadMetadataContainer::const_iterator RunInfoXml::readMetadataBegin() const
+{
+ return reads_.begin();
+}
+
+ReadMetadataContainer::const_iterator RunInfoXml::readMetadataEnd() const
+{
+ return reads_.end();
+}
+
+RunInfoXml::TileNumbersContainer::const_iterator RunInfoXml::tileNumbersBegin( common::LaneNumber lane ) const
+{
+ BCL2FASTQ_ASSERT_MSG( 0 < lane && lane <= getLanesCount(), "Lane '" << lane << "' is out of range" );
+ return tiles_[lane-1].begin();
+}
+
+RunInfoXml::TileNumbersContainer::const_iterator RunInfoXml::tileNumbersEnd( common::LaneNumber lane ) const
+{
+ BCL2FASTQ_ASSERT_MSG( 0 < lane && lane <= getLanesCount(), "Lane '" << lane << "' is out of range" );
+ return tiles_[lane-1].end();
+}
+
+void RunInfoXml::initReadMetadata()
+{
+ boost::optional<boost::property_tree::ptree &> readsElement = ptree_.get_child_optional("RunInfo.Run.Reads");
+ if (!readsElement) {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Required element 'RunInfo.Run.Reads' missing in RunInfo.xml file"));
+ }
+
+ // magic number 1: read number is 1-based
+ common::ReadNumber dataReadsCounter = 1;
+ common::ReadNumber indexReadsCounter = 1;
+ // magic number 1: cycle number is 1-based
+ common::CycleNumber cyclesCounter = 1;
+
+ BOOST_FOREACH (const boost::property_tree::ptree::value_type &readElement, readsElement.get())
+ {
+ static const std::string readsElementName("Read");
+ if (readElement.first != readsElementName)
+ {
+ continue;
+ }
+
+ boost::optional<const boost::property_tree::ptree &> indexElement = readElement.second.get_child_optional("Index");
+ boost::optional<std::string> indexAttribute = readElement.second.get_optional<std::string>("<xmlattr>.IsIndexedRead");
+ common::ReadNumber readNumber;
+ bool indexReadFlag;
+ if (indexElement || (indexAttribute && indexAttribute.get() == "Y"))
+ {
+ readNumber = indexReadsCounter++;
+ indexReadFlag = true;
+ }
+ else
+ {
+ readNumber = dataReadsCounter++;
+ indexReadFlag = false;
+ }
+
+ boost::optional<common::CycleNumber> firstCycle = readElement.second.get_optional<common::CycleNumber>("<xmlattr>.FirstCycle");
+ boost::optional<common::CycleNumber> lastCycle = readElement.second.get_optional<common::CycleNumber>("<xmlattr>.LastCycle");
+ if (!firstCycle)
+ {
+ firstCycle = cyclesCounter;
+ }
+ if (!lastCycle)
+ {
+ boost::optional<common::CycleNumber> numCycles = readElement.second.get_optional<common::CycleNumber>("<xmlattr>.NumCycles");
+ if(!numCycles) {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Either 'FirstCycle' and 'LastCycle' or 'NumCycles' attribute in element 'RunInfo.Run.Reads.Read' is required in RunInfo.xml file"));
+ }
+ // magic number -1: lastCycle needs to reference last existing cycle as opposed to one-past-last cycle
+ lastCycle = firstCycle.get() + numCycles.get() - 1;
+ }
+ // magic nubmer +1: lastCycle refers to last existing cycle in this read, next read needs to start with the next cycle
+ cyclesCounter = lastCycle.get() + 1;
+
+ reads_.push_back(ReadMetadata(ReadMetadata::Range(firstCycle.get(),
+ lastCycle.get()),
+ readNumber,
+ indexReadFlag ? common::ReadType::INDEX : common::ReadType::DATA));
+ }
+}
+
+void RunInfoXml::initTileNumbers()
+{
+ boost::optional<boost::property_tree::ptree &> flowcellLayout = ptree_.get_child_optional("RunInfo.Run.FlowcellLayout");
+ if (!flowcellLayout) {
+ BOOST_THROW_EXCEPTION(common::InputDataError("Required element 'RunInfo.Run.FlowcellLayout' missing in RunInfo.xml file"));
+ }
+ const unsigned laneCount = flowcellLayout.get().get<unsigned>("<xmlattr>.LaneCount");
+ const unsigned surfaceCount = flowcellLayout.get().get<unsigned>("<xmlattr>.SurfaceCount");
+ const unsigned swathCount = flowcellLayout.get().get<unsigned>("<xmlattr>.SwathCount");
+ const unsigned tileCount = flowcellLayout.get().get<unsigned>("<xmlattr>.TileCount");
+ const unsigned sectionPerLane = flowcellLayout.get().get<unsigned>("<xmlattr>.SectionPerLane", 0);
+ const unsigned lanePerSection = flowcellLayout.get().get<unsigned>("<xmlattr>.LanePerSection", 0);
+
+ for (unsigned lane = 0; lane < laneCount; ++lane)
+ {
+ TileNumbersContainer tmp;
+ for (unsigned surface = 1; surface <= surfaceCount; ++surface)
+ {
+ for (unsigned swath = 1; swath <= swathCount; ++swath)
+ {
+ for (unsigned tile = 1; tile <= tileCount; ++tile)
+ {
+ if (sectionPerLane > 1 || lanePerSection > 1)
+ {
+ BCL2FASTQ_ASSERT_MSG(lanePerSection, "Unexpected Zero lanePerSection");
+ const unsigned firstSection = 1 + sectionPerLane * (lane/lanePerSection);
+ for (unsigned section = firstSection; section <= firstSection + sectionPerLane; ++section)
+ {
+ tmp.push_back( common::TileNumber( tile + section * 100 + swath * 1000 + surface * 10000 ));
+ }
+ }
+ else
+ {
+ tmp.push_back( common::TileNumber( tile + swath * 100 + surface * 1000 ));
+ }
+ }
+ }
+ }
+ tiles_.push_back( tmp );
+ }
+}
+
+RunInfoXml createRunInfoXml(boost::filesystem::path runfolderDir)
+{
+ boost::filesystem::path file(runfolderDir / boost::filesystem::path(detail::runInfoXmlFileName));
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << "RunInfo.xml: '" << file << "'" << std::endl;
+ return RunInfoXml(io::parseXmlFile(file));
+}
+
+
+} // namespace layout
+
+
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/layout/SampleInfo.cpp b/src/cxx/lib/layout/SampleInfo.cpp
new file mode 100644
index 0000000..38389fc
--- /dev/null
+++ b/src/cxx/lib/layout/SampleInfo.cpp
@@ -0,0 +1,110 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file SampleInfo.cpp
+ *
+ * \brief Implementation of sample metadata.
+ *
+ * \author Marek Balint
+ */
+
+
+#include "layout/SampleInfo.hh"
+
+
+namespace bcl2fastq
+{
+namespace layout
+{
+
+
+const std::string SampleInfo::defaultProject = "default";
+const std::string SampleInfo::defaultId = "unknown";
+const std::string SampleInfo::defaultName = "Undetermined";
+
+SampleInfo::SampleInfo(
+ common::SampleNumber number,
+ SampleId sampleId,
+ SampleName sampleName,
+ Project project
+)
+: number_(number)
+, sampleId_(sampleId)
+, sampleName_(sampleName)
+, project_(project)
+, barcodes_()
+{
+}
+
+common::SampleNumber SampleInfo::getNumber() const
+{
+ return number_;
+}
+
+const Barcode& SampleInfo::getBarcode(BarcodesContainer::size_type idx) const
+{
+ return barcodes_[idx];
+}
+
+bool SampleInfo::hasId() const
+{
+ return SampleInfo::defaultId != sampleId_;
+}
+
+bool SampleInfo::hasName() const
+{
+ return SampleInfo::defaultName != sampleName_;
+}
+
+bool SampleInfo::hasProject() const
+{
+ return SampleInfo::defaultProject != project_;
+}
+
+bool SampleInfo::hasBarcodes() const
+{
+ return !barcodes_.empty();
+}
+
+void SampleInfo::addBarcode(const Barcode &barcode)
+{
+ barcodes_.push_back(barcode);
+}
+
+void SampleInfo::maskBarcode(common::ReadNumber indexNumber)
+{
+ bool noComponents = false;
+ for (auto& barcode : barcodes_)
+ {
+ barcode.mask(indexNumber);
+ if (barcode.getComponents().empty())
+ {
+ noComponents = true;
+ break;
+ }
+ }
+
+ if (noComponents)
+ {
+ barcodes_.clear();
+ }
+}
+
+std::ostream& operator<<(std::ostream& os, const SampleInfo &sampleInfo)
+{
+ os << "#" << sampleInfo.number_
+ << " '" << std::string(sampleInfo.sampleId_) << "' '" << std::string(sampleInfo.sampleName_)
+ << "' [" << std::string(sampleInfo.project_) << "]";
+ return os;
+}
+
+
+} // namespace layout
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/layout/TileInfo.cpp b/src/cxx/lib/layout/TileInfo.cpp
new file mode 100644
index 0000000..6dd467e
--- /dev/null
+++ b/src/cxx/lib/layout/TileInfo.cpp
@@ -0,0 +1,85 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file TileInfo.cpp
+ *
+ * \brief Implementation of tile metadata.
+ *
+ * \author Marek Balint
+ */
+
+
+#include "layout/TileInfo.hh"
+
+#include "common/Logger.hh"
+
+#include <boost/format.hpp>
+
+namespace bcl2fastq
+{
+namespace layout
+{
+
+
+TileInfo::TileInfo()
+: number_()
+, index_(0)
+, clustersCount_()
+, haveClustersCount_(false)
+, skippedTiles_(0)
+, skippedClusters_(0)
+{}
+
+TileInfo::TileInfo(
+ common::TileNumber number,
+ size_t index,
+ size_t skippedTiles
+)
+: number_(number)
+, index_(index)
+, clustersCount_()
+, haveClustersCount_(false)
+, skippedTiles_(skippedTiles)
+, skippedClusters_(0)
+{
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << " Tile: " << this->getNumber()
+ << " (index: " << this->getIndex()
+ << (this->haveClustersCount() ? ( boost::format(", clusters count: %d")
+ % this->getClustersCount() ).str()
+ : std::string(""))
+ << ", skipped tiles: " << this->getSkippedTilesCount() << ")" << std::endl;
+}
+
+TileInfo::TileInfo(
+ common::TileNumber number,
+ size_t index,
+ common::ClustersCount clustersCount,
+ size_t skippedTiles,
+ size_t skippedClusters
+)
+: number_(number)
+, index_(index)
+, clustersCount_(clustersCount)
+, haveClustersCount_(true)
+, skippedTiles_(skippedTiles)
+, skippedClusters_(skippedClusters)
+{
+ BCL2FASTQ_LOG(common::LogLevel::INFO) << " Tile: " << this->getNumber()
+ << " (index: " << this->getIndex()
+ << (this->haveClustersCount() ? ( boost::format(", clusters count: %d")
+ % this->getClustersCount() ).str()
+ : std::string(""))
+ << ", skipped tiles: " << this->getSkippedTilesCount()
+ << ", skipped clusters: " << this->getSkippedClustersCount() << ")" << std::endl;
+}
+
+
+} // namespace layout
+} // namespace bcl2fastq
+
+
diff --git a/src/cxx/lib/layout/UseBasesMask.cpp b/src/cxx/lib/layout/UseBasesMask.cpp
new file mode 100644
index 0000000..453be53
--- /dev/null
+++ b/src/cxx/lib/layout/UseBasesMask.cpp
@@ -0,0 +1,350 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file UseBasesMask.cpp
+ *
+ * \brief Implementation of UseBasesMask.
+ *
+ * \author Aaron Day
+ */
+
+#include "layout/UseBasesMask.hh"
+
+#include <boost/lexical_cast.hpp>
+#include <boost/algorithm/string.hpp>
+#include <boost/foreach.hpp>
+#include <vector>
+#include <functional>
+
+namespace bcl2fastq {
+
+namespace layout {
+
+UseBasesMaskFormatError::UseBasesMaskFormatError(const std::string& message)
+: common::RuntimeError(0, message)
+{
+}
+
+UseBasesMaskFormatError::UseBasesMaskFormatError(const UseBasesMaskFormatError& other)
+: common::RuntimeError(other)
+{
+}
+
+UseBasesMask::UseBasesMask(std::string baseMasks,
+ ReadMetadataContainer::const_iterator readsBegin,
+ ReadMetadataContainer::const_iterator readsEnd) :
+ reads_()
+{
+ if (baseMasks.empty()) return;
+
+ try
+ {
+ boost::algorithm::to_lower(baseMasks);
+
+ std::vector<std::string> masks;
+ boost::split(masks, baseMasks, boost::is_any_of(","));
+
+ common::ReadNumber indexReadsCounter = 1;
+ common::ReadNumber dataReadsCounter = 1;
+ common::CycleNumber cyclesCounter = 1;
+
+ if (masks.size() != static_cast<size_t>(std::distance(readsBegin, readsEnd)))
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError(
+ "UseBasesMask formatting error. A mask must be specified for each read."
+ " Number of reads: " + boost::lexical_cast<std::string>(std::distance(readsBegin, readsEnd))
+ + " Base masks: '" + baseMasks + "'"));
+ }
+
+ size_t numCyclesRunInfoAllReads = 0;
+ bool doesMatchRunInfo = true;
+ ReadMetadataContainer::const_iterator readIter = readsBegin;
+ BOOST_FOREACH(std::string& mask, std::make_pair(masks.begin(), masks.end()))
+ {
+ unsigned int totalCyclesRunInfoXml = readIter->lastCycle_ - readIter->firstCycle_ + 1;
+ numCyclesRunInfoAllReads += totalCyclesRunInfoXml;
+
+ // Get rid of white space
+ boost::trim(mask);
+
+ basicValidate(mask);
+
+ expandNumbers(mask);
+
+ parseAsteriskAndValidateSize(mask,
+ doesMatchRunInfo,
+ totalCyclesRunInfoXml);
+
+ std::pair<int, int> numIgnoreCycles = parseIgnoreCycles(mask);
+ cyclesCounter += numIgnoreCycles.first;
+ addCycles(mask,
+ readIter->readType_,
+ dataReadsCounter,
+ indexReadsCounter,
+ cyclesCounter);
+ cyclesCounter += numIgnoreCycles.second;
+
+ // Go to the next read in RunInfo.xml
+ ++readIter;
+ }
+
+ if (cyclesCounter-1 != numCyclesRunInfoAllReads)
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError("The total number of cycles specified by"
+ " the use-bases-mask does not match the number found in RunInfo.xml. Masks: " + baseMasks + "'"));
+ }
+ }
+ catch(boost::bad_lexical_cast&)
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError("UseBasesMask formatting error. Failed attempt at reading number of cycles"));
+ }
+}
+
+ReadMetadataContainer::const_iterator UseBasesMask::readMetadataBegin() const
+{
+ return reads_.begin();
+}
+
+ReadMetadataContainer::const_iterator UseBasesMask::readMetadataEnd() const
+{
+ return reads_.end();
+}
+
+bool UseBasesMask::isMaskCharInvalid(char maskChar) const
+{
+ return maskChar != 'i' &&
+ maskChar != 'y' &&
+ maskChar != 'n';
+}
+
+// Throw an exception if the mask is not in a valid format
+void UseBasesMask::basicValidate(const std::string& mask) const
+{
+ if (mask.size() < 2)
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError(
+ "UseBasesMask formatting error. Length too small."
+ " Use base mask: '" + mask + "'"));
+ }
+
+ size_t invalidCharPos = mask.find_first_not_of("*yni0123456789");
+ if (invalidCharPos != std::string::npos)
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError(
+ std::string("UseBasesMask formatting error. Invalid character: '")
+ + mask[invalidCharPos] + std::string("' in mask: '") + mask + "'"));
+ }
+
+ if (isMaskCharInvalid(mask[0]))
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError(
+ "UseBasesMask formatting error. Use base mask must start with either a 'y', 'i', or 'n'."
+ " Use base mask: '" + mask + "'"));
+ }
+
+ size_t asteriskPos = mask.find_first_of('*');
+ if (asteriskPos != std::string::npos)
+ {
+ if (mask.find('*', asteriskPos+1) != std::string::npos)
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError(
+ "UseBasesMask formatting error. Use base mask must contain at most 1 '*'."
+ " Use base mask: '" + mask + "'"));
+ }
+
+ if (isMaskCharInvalid(mask[asteriskPos-1]))
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError(
+ "UseBasesMask formatting error. An asterisk must be preceded by either a 'y', 'i', or 'n'."
+ " Use base mask: '" + mask + "'"));
+ }
+
+ }
+}
+
+void UseBasesMask::expandNumbers(std::string& mask) const
+{
+ while (true)
+ {
+ size_t beginDigitPos = mask.find_first_of("0123456789");
+ if (beginDigitPos == std::string::npos) return;
+
+ if (beginDigitPos == 0 || mask[beginDigitPos] == '0')
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError(
+ "UseBasesMask formatting error. Error parsing number from mask."
+ " Base mask: '" + mask + "'"));
+ }
+
+ size_t endDigitPos = mask.find_first_not_of("0123456789", beginDigitPos);
+ size_t digitsLength = endDigitPos==std::string::npos ?
+ mask.size()-beginDigitPos :
+ endDigitPos-beginDigitPos;
+
+ unsigned int numExpand = boost::lexical_cast<unsigned int>(mask.substr(beginDigitPos, digitsLength));
+
+ mask.erase(beginDigitPos, digitsLength);
+
+ // numExpand-1 because y2 should be replaced by yy (just 1 y added)
+ mask.insert(beginDigitPos, numExpand-1, mask[beginDigitPos-1]);
+ }
+}
+
+void UseBasesMask::parseAsteriskAndValidateSize(std::string& mask,
+ bool& doesMatchRunInfo,
+ unsigned int totalCycles) const
+{
+ size_t asteriskPos = mask.find('*');
+
+ if (asteriskPos == std::string::npos)
+ {
+ if (mask.size() != totalCycles)
+ {
+ doesMatchRunInfo = false;
+ }
+
+ return;
+ }
+
+ if (!doesMatchRunInfo)
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError("One or more read masks do not match"
+ " the number of cycles specified in RunInfo.xml, and a '*' was specified."
+ " There is no way to determine the number of cycles implied by the '*'."
+ " Base mask: " + mask + "'"));
+ }
+
+ if (mask.size() > totalCycles + 2)
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError(
+ "UseBasesMask formatting error. Mask size does not match number of cycles in RunInfo.xml."
+ " RunInfo.xml cycles: " + boost::lexical_cast<std::string>(totalCycles) +
+ " Base mask: '" + mask + "'"));
+ }
+
+ if (asteriskPos == 0 ||
+ (mask[asteriskPos - 1] != 'i' &&
+ mask[asteriskPos - 1] != 'n' &&
+ mask[asteriskPos - 1] != 'y'))
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError(
+ "UseBasesMask formatting error. Invalid use of '*'."
+ " Base mask: '" + mask + "'"));
+ }
+
+ // If we got here, the format is valid.
+ mask.erase(asteriskPos, 1);
+
+ int numExpandCycles = totalCycles - mask.size();
+ if (numExpandCycles >= 0)
+ {
+ mask.insert(asteriskPos, numExpandCycles, mask[asteriskPos-1]);
+ }
+ else
+ {
+ BCL2FASTQ_ASSERT_MSG(numExpandCycles == -1, "Use bases mask internal error");
+ mask.resize(mask.size()-1);
+ }
+}
+
+// Cut the n's off the mask. Return (# n's at beginning, #n's at end)
+std::pair<int, int> UseBasesMask::parseIgnoreCycles(std::string& mask) const
+{
+ size_t firstNotN = mask.find_first_not_of('n');
+
+ if (firstNotN == std::string::npos)
+ {
+ std::pair<int, int> ret(mask.size(), 0);
+ mask = "";
+ return ret;
+ }
+
+ size_t lastNotN = mask.find_last_not_of('n');
+
+ std::string maskWithoutN = mask.substr(firstNotN, lastNotN-firstNotN+1);
+
+ if (maskWithoutN.find('n') != std::string::npos)
+ {
+ BOOST_THROW_EXCEPTION(UseBasesMaskFormatError(
+ "UseBasesMask formatting error. Masking out cycles in the middle of the read is not allowed."
+ " Base mask: '" + mask + "'"));
+ }
+
+ int numIgnoreAtEnd = mask.size() - lastNotN - 1;
+ mask = maskWithoutN;
+
+ return std::make_pair(firstNotN, numIgnoreAtEnd);
+}
+
+
+void UseBasesMask::addCycles(const std::string& mask,
+ common::ReadType readType,
+ common::ReadNumber& dataReadsCounter,
+ common::ReadNumber& indexReadsCounter,
+ common::CycleNumber& cyclesCounter)
+{
+ size_t nextReadBeginPos = 0;
+
+ if (mask.find_first_not_of('n') == std::string::npos)
+ {
+ // The read is all 'n's, all bases are masked.
+ switch (readType)
+ {
+ case common::ReadType::DATA:
+ ++dataReadsCounter;
+ break;
+ case common::ReadType::INDEX:
+ ++indexReadsCounter;
+ break;
+ default:
+ BCL2FASTQ_ASSERT_MSG(false, "Unrecognized read type");
+ break;
+ }
+ return;
+ }
+
+ while (nextReadBeginPos != std::string::npos)
+ {
+ size_t readBeginPos = nextReadBeginPos;
+
+ reads_.push_back(ReadMetadata());
+ ReadMetadata& readMetadata = reads_.back();
+
+ size_t firstNotI = mask.find_first_not_of('i', readBeginPos);
+
+ if (firstNotI == readBeginPos)
+ {
+ readMetadata.readNumber_ = dataReadsCounter++;
+ readMetadata.readType_ = common::ReadType::DATA;
+ nextReadBeginPos = mask.find_first_not_of('y', readBeginPos);
+ }
+ else
+ {
+ readMetadata.readNumber_ = indexReadsCounter++;
+ readMetadata.readType_ = common::ReadType::INDEX;
+ nextReadBeginPos = mask.find_first_not_of('i', readBeginPos);
+ }
+
+ readMetadata.firstCycle_ = cyclesCounter;
+ readMetadata.lastCycle_ = readMetadata.firstCycle_ +
+ (nextReadBeginPos == std::string::npos ?
+ mask.size() - readBeginPos :
+ nextReadBeginPos - readBeginPos) - 1;
+
+ // I have no idea how to figure out which cycles are masked/unmasked
+ // when the UseBasesMask could alter the read layout entirely.
+ readMetadata.firstUnmaskedCycle_ = readMetadata.firstCycle_;
+ readMetadata.lastUnmaskedCycle_ = readMetadata.lastCycle_;
+
+ cyclesCounter += readMetadata.lastCycle_ - readMetadata.firstCycle_ + 1;
+ }
+}
+
+}
+
+}
diff --git a/src/cxx/lib/layout/cppunit/CMakeLists.txt b/src/cxx/lib/layout/cppunit/CMakeLists.txt
new file mode 100644
index 0000000..ec43d99
--- /dev/null
+++ b/src/cxx/lib/layout/cppunit/CMakeLists.txt
@@ -0,0 +1,23 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for any cppunit subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CPPUNIT_CMAKE})
+
+
diff --git a/src/cxx/lib/layout/cppunit/RegistryNames.txt b/src/cxx/lib/layout/cppunit/RegistryNames.txt
new file mode 100644
index 0000000..66b5027
--- /dev/null
+++ b/src/cxx/lib/layout/cppunit/RegistryNames.txt
@@ -0,0 +1,5 @@
+RunInfoXml
+ConfigXml
+UseBasesMask
+BarcodeCollisionDetector
+Layout
diff --git a/src/cxx/lib/layout/cppunit/testBarcodeCollisionDetector.cpp b/src/cxx/lib/layout/cppunit/testBarcodeCollisionDetector.cpp
new file mode 100644
index 0000000..32e523f
--- /dev/null
+++ b/src/cxx/lib/layout/cppunit/testBarcodeCollisionDetector.cpp
@@ -0,0 +1,177 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testBarcodeCollisionDetector.cpp
+ *
+ * \brief BarcodeCollisionDetector cppunit test declarations.
+ *
+ * \author Aaron Day
+ */
+
+
+#include "layout/BarcodeCollisionDetector.hh"
+
+#include "RegistryName.hh"
+#include "testBarcodeCollisionDetector.hh"
+
+
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(TestBarcodeCollisionDetector, registryName("BarcodeCollisionDetector"));
+
+
+void TestBarcodeCollisionDetector::setUp()
+{
+ componentMaxMismatches_.push_back(0);
+ componentMaxMismatches_.push_back(1);
+ componentMaxMismatches_.push_back(2);
+}
+
+void TestBarcodeCollisionDetector::tearDown()
+{
+}
+
+void TestBarcodeCollisionDetector::testSuccess()
+{
+ bcl2fastq::layout::BarcodesContainer barcodeContainer1;
+ std::vector<std::string> barcodeComponents1;
+ barcodeComponents1.push_back("ACGTACGT");
+ barcodeComponents1.push_back("TTTTCCCC");
+ barcodeComponents1.push_back("CGCGCGCGCGCG");
+ barcodeComponents1.push_back("AAAAAAAAAA");
+ barcodeContainer1.push_back(bcl2fastq::layout::Barcode(barcodeComponents1.begin(), barcodeComponents1.end()));
+
+ bcl2fastq::layout::BarcodesContainer barcodeContainer2;
+ std::vector<std::string> barcodeComponents2;
+ barcodeComponents2.push_back("ACGGACCT"); // 2 mismatches
+ barcodeComponents2.push_back("ATAACCCC"); // 3 mismatches
+ barcodeComponents2.push_back("CACGAGTGCCCA"); // 5 mismatches
+ barcodeComponents2.push_back("GGGGGGGGGG"); // all mismatches
+ barcodeContainer2.push_back(bcl2fastq::layout::Barcode(barcodeComponents2.begin(), barcodeComponents2.end()));
+
+ bcl2fastq::layout::BarcodeCollisionDetector collisionDetector(componentMaxMismatches_);
+
+ collisionDetector.validateBarcode(barcodeContainer1);
+
+ collisionDetector.validateBarcode(barcodeContainer2);
+}
+
+void TestBarcodeCollisionDetector::testMismatchLength()
+{
+ bcl2fastq::layout::BarcodesContainer barcodeContainer1;
+ std::vector<std::string> barcodeComponents1;
+ barcodeComponents1.push_back("ACGTACGT");
+ barcodeComponents1.push_back("TTTTCCCCG");
+ barcodeContainer1.push_back(bcl2fastq::layout::Barcode(barcodeComponents1.begin(), barcodeComponents1.end()));
+
+ bcl2fastq::layout::BarcodesContainer barcodeContainer2;
+ std::vector<std::string> barcodeComponents2;
+ barcodeComponents2.push_back("CGCGTATA");
+ barcodeComponents2.push_back("AACCGGTT");
+ barcodeContainer2.push_back(bcl2fastq::layout::Barcode(barcodeComponents2.begin(), barcodeComponents2.end()));
+
+ bcl2fastq::layout::BarcodeCollisionDetector collisionDetector(componentMaxMismatches_);
+
+ collisionDetector.validateBarcode(barcodeContainer1);
+
+ CPPUNIT_ASSERT_THROW(collisionDetector.validateBarcode(barcodeContainer2),
+ bcl2fastq::layout::BarcodeCollisionError);
+
+ // 2 barcodes of different length in the same sample
+ bcl2fastq::layout::BarcodesContainer barcodeContainer3;
+ std::vector<std::string> barcodeComponents3;
+ barcodeComponents3.push_back("AAAAAAAA");
+ barcodeComponents3.push_back("TTTTTTTT");
+ barcodeContainer3.push_back(bcl2fastq::layout::Barcode(barcodeComponents3.begin(), barcodeComponents3.end()));
+
+ barcodeComponents3.clear();
+ barcodeComponents3.push_back("AAAAAAAAAA");
+ barcodeComponents3.push_back("TTTTTTTTTT");
+ barcodeContainer3.push_back(bcl2fastq::layout::Barcode(barcodeComponents3.begin(), barcodeComponents3.end()));
+
+ CPPUNIT_ASSERT_THROW(collisionDetector.validateBarcode(barcodeContainer3),
+ bcl2fastq::layout::BarcodeCollisionError);
+}
+
+void TestBarcodeCollisionDetector::testNumComponents()
+{
+ bcl2fastq::layout::BarcodesContainer barcodeContainer1;
+ std::vector<std::string> barcodeComponents1;
+ barcodeComponents1.push_back("ACGTACGT");
+ barcodeComponents1.push_back("TTTTCCCC");
+ barcodeComponents1.push_back("CTCTGATG");
+ barcodeContainer1.push_back(bcl2fastq::layout::Barcode(barcodeComponents1.begin(), barcodeComponents1.end()));
+
+ bcl2fastq::layout::BarcodesContainer barcodeContainer2;
+ std::vector<std::string> barcodeComponents2;
+ barcodeComponents2.push_back("CGCGTATA");
+ barcodeComponents2.push_back("AACCGGTT");
+ barcodeContainer2.push_back(bcl2fastq::layout::Barcode(barcodeComponents2.begin(), barcodeComponents2.end()));
+
+ bcl2fastq::layout::BarcodeCollisionDetector collisionDetector(componentMaxMismatches_);
+
+ collisionDetector.validateBarcode(barcodeContainer1);
+
+ CPPUNIT_ASSERT_THROW(collisionDetector.validateBarcode(barcodeContainer2),
+ bcl2fastq::layout::BarcodeCollisionError);
+}
+
+void TestBarcodeCollisionDetector::testCollisionSameSample()
+{
+ // Add 2 close barcodes, which is not ok even though they're for the same sample
+ bcl2fastq::layout::BarcodesContainer barcodeContainer3;
+ std::vector<std::string> barcodeComponents3;
+ barcodeComponents3.push_back("ACGTACGT"); // 0 mismatch
+ barcodeComponents3.push_back("TTTTCCCC"); // 0 mismatches
+ barcodeComponents3.push_back("CGCGCGCGCGCG"); // 0 mismatches
+ barcodeComponents3.push_back("AAAAGGGGGG"); // 6 mismatches
+ barcodeContainer3.push_back(bcl2fastq::layout::Barcode(barcodeComponents3.begin(), barcodeComponents3.end()));
+
+ barcodeComponents3.clear();
+ barcodeComponents3.push_back("ACGTACGT"); // 0 mismatches
+ barcodeComponents3.push_back("TTTTCCCC"); // 0 mismatches
+ barcodeComponents3.push_back("CGCGCGCGCGCG"); // 0 mismatches
+ barcodeComponents3.push_back("AAAAAGGGGG"); // 5 mismatches
+ barcodeContainer3.push_back(bcl2fastq::layout::Barcode(barcodeComponents3.begin(), barcodeComponents3.end()));
+
+ bcl2fastq::layout::BarcodeCollisionDetector collisionDetector(componentMaxMismatches_);
+
+ CPPUNIT_ASSERT_THROW(collisionDetector.validateBarcode(barcodeContainer3),
+ bcl2fastq::layout::BarcodeCollisionError);
+}
+
+void TestBarcodeCollisionDetector::testCollision()
+{
+ bcl2fastq::layout::BarcodesContainer barcodeContainer1;
+ std::vector<std::string> barcodeComponents1;
+ barcodeComponents1.push_back("ACGTACGT");
+ barcodeComponents1.push_back("TTTTCCCC");
+ barcodeContainer1.push_back(bcl2fastq::layout::Barcode(barcodeComponents1.begin(), barcodeComponents1.end()));
+
+ bcl2fastq::layout::BarcodesContainer barcodeContainer2;
+ std::vector<std::string> barcodeComponents2;
+ barcodeComponents2.push_back("ACGTACGT"); // 2 mismatches
+ barcodeComponents2.push_back("TTTTTTCC"); // 2 mismatches
+ barcodeContainer2.push_back(bcl2fastq::layout::Barcode(barcodeComponents2.begin(), barcodeComponents2.end()));
+
+ bcl2fastq::layout::BarcodeCollisionDetector collisionDetector(componentMaxMismatches_);
+
+ collisionDetector.validateBarcode(barcodeContainer1);
+
+ CPPUNIT_ASSERT_THROW(collisionDetector.validateBarcode(barcodeContainer2),
+ bcl2fastq::layout::BarcodeCollisionError);
+
+ barcodeComponents1.clear();
+ barcodeComponents1.push_back("GGGGGGGGGGGG");
+ barcodeContainer1.push_back(bcl2fastq::layout::Barcode(barcodeComponents1.begin(), barcodeComponents1.end()));
+
+ barcodeComponents2.clear();
+ barcodeComponents2.push_back("AAAAGGGGGGGG"); // 4 mismatches
+ barcodeContainer2.push_back(bcl2fastq::layout::Barcode(barcodeComponents2.begin(), barcodeComponents2.end()));
+ CPPUNIT_ASSERT_THROW(collisionDetector.validateBarcode(barcodeContainer2),
+ bcl2fastq::layout::BarcodeCollisionError);
+}
+
diff --git a/src/cxx/lib/layout/cppunit/testBarcodeCollisionDetector.hh b/src/cxx/lib/layout/cppunit/testBarcodeCollisionDetector.hh
new file mode 100644
index 0000000..17c5e12
--- /dev/null
+++ b/src/cxx/lib/layout/cppunit/testBarcodeCollisionDetector.hh
@@ -0,0 +1,51 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file barcodeCollisionDetector.hh
+ *
+ * \brief BarcodeCollisionDetector cppunit test declarations.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_TEST_BARCODE_COLLISION_DETECTOR_HH
+#define BCL2FASTQ_LAYOUT_TEST_BARCODE_COLLISION_DETECTOR_HH
+
+
+#include <cppunit/extensions/HelperMacros.h>
+
+#include <vector>
+
+/// \brief Test suite for BarcodeCollisionDetector.
+class TestBarcodeCollisionDetector : public CppUnit::TestFixture
+{
+ CPPUNIT_TEST_SUITE(TestBarcodeCollisionDetector);
+ CPPUNIT_TEST(testSuccess);
+ CPPUNIT_TEST(testMismatchLength);
+ CPPUNIT_TEST(testNumComponents);
+ CPPUNIT_TEST(testCollisionSameSample);
+ CPPUNIT_TEST(testCollision);
+ CPPUNIT_TEST_SUITE_END();
+
+private:
+ std::vector<std::size_t> componentMaxMismatches_;
+
+public:
+ TestBarcodeCollisionDetector() : componentMaxMismatches_() {}
+ void setUp();
+ void tearDown();
+ void testSuccess();
+ void testMismatchLength();
+ void testNumComponents();
+ void testCollisionSameSample();
+ void testCollision();
+};
+
+
+#endif // BCL2FASTQ_LAYOUT_TEST_BARCODE_COLLISION_DETECTOR_HH
diff --git a/src/cxx/lib/layout/cppunit/testConfigXml.cpp b/src/cxx/lib/layout/cppunit/testConfigXml.cpp
new file mode 100644
index 0000000..c7bbb9b
--- /dev/null
+++ b/src/cxx/lib/layout/cppunit/testConfigXml.cpp
@@ -0,0 +1,115 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testConfigXml.cpp
+ *
+ * \brief Sample sheet helper cppunit test declarations.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <string>
+
+#include "RegistryName.hh"
+#include "testConfigXml.hh"
+
+#include "io/Xml.hh"
+#include "layout/ConfigXml.hh"
+
+
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(TestConfigXml, registryName("ConfigXml"));
+
+
+void TestConfigXml::setUp()
+{
+}
+
+void TestConfigXml::tearDown()
+{
+}
+
+void TestConfigXml::testHcs()
+{
+ const std::string configXmlData(
+ "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
+ "<BaseCallAnalysis xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n"
+ " <Run Name=\"BaseCalls\">\n"
+ " <TileSelection>\n"
+ " <Lane Index=\"6\">\n"
+ " <Sample>s</Sample>\n"
+ " <Tile>6001</Tile>\n"
+ " <Tile>6002</Tile>\n"
+ " </Lane>\n"
+ " <Lane Index=\"5\">\n"
+ " <Sample>s</Sample>\n"
+ " <Tile>5001</Tile>\n"
+ " <Tile>5002</Tile>\n"
+ " </Lane>\n"
+ " <Lane Index=\"8\">\n"
+ " <Sample>s</Sample>\n"
+ " <Tile>8001</Tile>\n"
+ " <Tile>8002</Tile>\n"
+ " </Lane>\n"
+ " <Lane Index=\"7\">\n"
+ " <Sample>s</Sample>\n"
+ " <Tile>7001</Tile>\n"
+ " <Tile>7002</Tile>\n"
+ " </Lane>\n"
+ //" <Lane Index=\"2\">\n"
+ //" <Sample>s</Sample>\n"
+ //" <Tile>2001</Tile>\n"
+ //" <Tile>2002</Tile>\n"
+ //" </Lane>\n"
+ " <Lane Index=\"1\">\n"
+ " <Sample>s</Sample>\n"
+ " <Tile>1001</Tile>\n"
+ " <Tile>1002</Tile>\n"
+ " </Lane>\n"
+ " <Lane Index=\"4\">\n"
+ " <Sample>s</Sample>\n"
+ //" <Tile>4001</Tile>\n"
+ //" <Tile>4002</Tile>\n"
+ " </Lane>\n"
+ " <Lane Index=\"3\">\n"
+ " <Sample>s</Sample>\n"
+ " <Tile>3001</Tile>\n"
+ " <Tile>3002</Tile>\n"
+ " </Lane>\n"
+ " </TileSelection>\n"
+ " </Run>\n"
+ "</BaseCallAnalysis>\n"
+ );
+
+ bcl2fastq::layout::ConfigXml configXml(bcl2fastq::io::parseXmlData(
+ configXmlData.begin(),
+ configXmlData.end()
+ ));
+
+ for (bcl2fastq::common::LaneNumber laneNumber = 1; laneNumber <= 8; ++laneNumber)
+ {
+ bcl2fastq::layout::ConfigXml::TileMetadataContainer::const_iterator tileMetadataIter = configXml.tileMetadataBegin(laneNumber);
+ const bcl2fastq::layout::ConfigXml::TileMetadataContainer::const_iterator tileMetadataEnd = configXml.tileMetadataEnd(laneNumber);
+ if ((laneNumber == 2) || (laneNumber == 4))
+ {
+ CPPUNIT_ASSERT(tileMetadataIter == tileMetadataEnd);
+ }
+ else
+ {
+ CPPUNIT_ASSERT(tileMetadataIter != tileMetadataEnd);
+ CPPUNIT_ASSERT_EQUAL(tileMetadataIter->tileNumber_, bcl2fastq::common::TileNumber((laneNumber * 1000) + 1));
+ ++tileMetadataIter;
+ CPPUNIT_ASSERT(tileMetadataIter != tileMetadataEnd);
+ CPPUNIT_ASSERT_EQUAL(tileMetadataIter->tileNumber_, bcl2fastq::common::TileNumber((laneNumber * 1000) + 2));
+ ++tileMetadataIter;
+ CPPUNIT_ASSERT(tileMetadataIter == tileMetadataEnd);
+ }
+ }
+}
+
+
diff --git a/src/cxx/lib/layout/cppunit/testConfigXml.hh b/src/cxx/lib/layout/cppunit/testConfigXml.hh
new file mode 100644
index 0000000..e1eb1d2
--- /dev/null
+++ b/src/cxx/lib/layout/cppunit/testConfigXml.hh
@@ -0,0 +1,47 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testConfigXml.hh
+ *
+ * \brief Config.xml helper cppunit test declarations.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_TEST_CONFIGXML_HH
+#define BCL2FASTQ_LAYOUT_TEST_CONFIGXML_HH
+
+
+#include <string>
+
+#include <cppunit/extensions/HelperMacros.h>
+
+#include "layout/ConfigXml.hh"
+
+
+/// \brief Test suite for ConfigXml.
+class TestConfigXml : public CppUnit::TestFixture
+{
+private:
+
+ CPPUNIT_TEST_SUITE(TestConfigXml);
+ CPPUNIT_TEST(testHcs);
+ CPPUNIT_TEST_SUITE_END();
+
+public:
+
+ void setUp();
+ void tearDown();
+ void testHcs();
+};
+
+
+#endif // #ifndef BCL2FASTQ_LAYOUT_TEST_CONFIGXML_HH
+
+
diff --git a/src/cxx/lib/layout/cppunit/testLayout.cpp b/src/cxx/lib/layout/cppunit/testLayout.cpp
new file mode 100644
index 0000000..0680870
--- /dev/null
+++ b/src/cxx/lib/layout/cppunit/testLayout.cpp
@@ -0,0 +1,640 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testLayout.cpp
+ *
+ * \brief Layout cppunit test declarations.
+ *
+ * \author Aaron Day
+ */
+
+
+#include <string>
+
+#include "RegistryName.hh"
+#include "testLayout.hh"
+
+#include "io/Xml.hh"
+#include "layout/Layout.hh"
+
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(TestLayout, registryName("Layout"));
+
+void TestLayout::setUp()
+{
+ // Turn off logging
+ bcl2fastq::common::BCL2FASTQ_MIN_LOG_LEVEL = bcl2fastq::common::LogLevel::NONE;
+
+ barcodeLengthsForLane_.push_back(std::vector<size_t>(1, 8));
+
+ const std::string runInfoXmlData(
+ "<?xml version=\"1.0\"?>\n"
+ "<RunInfo xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" Version=\"2\">\n"
+ " <Run Id=\"TestRun\" Number=\"42\">\n"
+ " <Flowcell>TestFlowcell</Flowcell>\n"
+ " <Instrument>CSSIM</Instrument>\n"
+ " <Date>8/20/2013 3:29:17 PM</Date>\n"
+ " <Reads>\n"
+ " <Read Number=\"1\" NumCycles=\"151\" IsIndexedRead=\"N\" />\n"
+ " <Read Number=\"2\" NumCycles=\"8\" IsIndexedRead=\"Y\" />\n"
+ " <Read Number=\"3\" NumCycles=\"151\" IsIndexedRead=\"N\" />\n"
+ " </Reads>\n"
+ " <FlowcellLayout LaneCount=\"4\" SurfaceCount=\"2\" SwathCount=\"3\" TileCount=\"11\" />\n"
+ " </Run>\n"
+ "</RunInfo>\n"
+ );
+
+ runInfoXml_ = std::shared_ptr<bcl2fastq::layout::RunInfoXml>(
+ new bcl2fastq::layout::RunInfoXml(bcl2fastq::io::parseXmlData(
+ runInfoXmlData.begin(),
+ runInfoXmlData.end())));
+
+ const std::string sampleSheetData(
+ "[Settings]\n"
+ "trimUmi,1\n"
+ "read1UmiLength,2\n"
+ "read2UmiLength,151\n");
+
+ sampleSheet_ = std::shared_ptr<bcl2fastq::config::SampleSheetCsv>(
+ new bcl2fastq::config::SampleSheetCsv(
+ bcl2fastq::common::parseCsvData(
+ sampleSheetData.begin(),
+ sampleSheetData.end()),
+ boost::filesystem::path()));
+}
+
+void TestLayout::tearDown()
+{
+}
+
+void TestLayout::getLaneInfos(bcl2fastq::layout::LaneInfosContainer& laneInfos)
+{
+ laneInfos.push_back(bcl2fastq::layout::LaneInfo(1));
+ laneInfos.back().addSample(bcl2fastq::layout::SampleInfo());
+
+ std::vector<std::string> barcodeComponents;
+ barcodeComponents.push_back("AAAAGGGG");
+ bcl2fastq::layout::Barcode barcode(barcodeComponents.begin(), barcodeComponents.end());
+ bcl2fastq::layout::SampleInfo sampleInfo(1, "MySampleId", "MySampleName", "MyProject");
+ sampleInfo.addBarcode(barcode);
+ laneInfos.back().addSample(sampleInfo);
+}
+
+// expectedCycles: Cycles to put in the FASTQ sequence
+// expectedUnmaskedCycles: All cycles in RunInfo.xml
+// expectedCyclesToLoad: Cycles we need to load from BCL (including UMIs, which might not be present in the FASTQ sequence)
+void TestLayout::assertExpectedCycles(const bcl2fastq::layout::LaneInfo& laneInfo,
+ const std::vector<ExpectedCycleInfosForRead>& expectedCycles,
+ const std::vector<ExpectedCycleInfosForRead>& expectedUnmaskedCycles,
+ const std::vector<ExpectedCycleInfosForRead>& expectedCyclesToLoad,
+ const std::vector<bcl2fastq::common::CycleRange>& umiCycles,
+ const std::vector<bcl2fastq::common::CycleNumber>& bclBufferOffsets)
+{
+ for (const auto& expectedCyclesForRead : expectedCycles)
+ {
+ CPPUNIT_ASSERT_EQUAL(laneInfo.getReadInfos().at(expectedCyclesForRead.readNumber_).cycleInfos().size(), expectedCyclesForRead.numCycles_);
+
+ verifyCycles(expectedCyclesForRead.firstCycle_,
+ laneInfo.getReadInfos().at(expectedCyclesForRead.readNumber_).cycleInfos());
+ }
+
+ for (const auto& expectedCyclesForRead : expectedUnmaskedCycles)
+ {
+ CPPUNIT_ASSERT_EQUAL(laneInfo.getReadInfos().at(expectedCyclesForRead.readNumber_).unmaskedCycleInfos().size(), expectedCyclesForRead.numCycles_);
+
+ verifyCycles(expectedCyclesForRead.firstCycle_,
+ laneInfo.getReadInfos().at(expectedCyclesForRead.readNumber_).unmaskedCycleInfos());
+ }
+
+ for (const auto& expectedCyclesForRead : expectedCyclesToLoad)
+ {
+ CPPUNIT_ASSERT_EQUAL(laneInfo.getReadInfos().at(expectedCyclesForRead.readNumber_).cyclesToLoad().size(), expectedCyclesForRead.numCycles_);
+
+ verifyCycles(expectedCyclesForRead.firstCycle_,
+ laneInfo.getReadInfos().at(expectedCyclesForRead.readNumber_).cyclesToLoad());
+ }
+
+ for (size_t i = 0; i < laneInfo.getReadInfos().size(); ++i)
+ {
+ CPPUNIT_ASSERT_EQUAL(laneInfo.getReadInfos().at(i).getUmiCycles().first, umiCycles[i].first);
+ CPPUNIT_ASSERT_EQUAL(laneInfo.getReadInfos().at(i).getUmiCycles().second, umiCycles[i].second);
+ CPPUNIT_ASSERT_EQUAL(laneInfo.getReadInfos().at(i).getBclBufferOffset(), bclBufferOffsets.at(i));
+ }
+}
+
+template<typename T>
+void TestLayout::verifyCycles(bcl2fastq::common::CycleNumber startCycle,
+ const T& cycleInfos)
+{
+ for (const auto& cycle : cycleInfos)
+ {
+ CPPUNIT_ASSERT_EQUAL(cycle.getNumber(), startCycle++);
+ }
+}
+
+void TestLayout::testDetectReadLayoutDefaultCycles()
+{
+ bcl2fastq::config::BarcodeMismatchCountsContainer barcodeMismatches;
+
+ bcl2fastq::layout::LaneInfosContainer laneInfos;
+ getLaneInfos(laneInfos);
+
+ bcl2fastq::layout::detectReadLayout(
+ *runInfoXml_,
+ *sampleSheet_, // used for adapters only
+ std::vector<std::string>(), // use bases masks
+ 35,
+ 0, // read1StartCycle
+ 0, // read2StartCycle
+ 0, // read1EndCycle
+ 0, // read2EndCycle
+ 0, // read1UmiLength
+ 0, // read2UmiLength
+ 1, // read1UmiStartFromCycle
+ 1, // read2UmiStartFromCycle
+ false, // trimUmi
+ false,
+ barcodeMismatches,
+ laneInfos,
+ barcodeLengthsForLane_);
+
+ CPPUNIT_ASSERT_EQUAL(laneInfos.back().getReadInfos().size(),3UL);
+ assertExpectedCycles(laneInfos.back(),
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<bcl2fastq::common::CycleRange>{bcl2fastq::common::CycleRange(0, 0),
+ bcl2fastq::common::CycleRange(0, 0),
+ bcl2fastq::common::CycleRange(0, 0)},
+ std::vector<bcl2fastq::common::CycleNumber>{0, 0, 0});
+}
+
+void TestLayout::testUmiCyclesAtBeginning()
+{
+ bcl2fastq::config::BarcodeMismatchCountsContainer barcodeMismatches;
+
+ bcl2fastq::layout::LaneInfosContainer laneInfos;
+ getLaneInfos(laneInfos);
+
+ bcl2fastq::layout::detectReadLayout(
+ *runInfoXml_,
+ *sampleSheet_, // used for adapters only
+ std::vector<std::string>(), // use bases masks
+ 35,
+ 0, // read1StartCycle
+ 0, // read2StartCycle
+ 0, // read1EndCycle
+ 0, // read2EndCycle
+ 10, // read1UmiLength
+ 12, // read2UmiLength
+ 1, // read1UmiStartFromCycle
+ 1, // read2UmiStartFromCycle
+ false, // trimUmi
+ false,
+ barcodeMismatches,
+ laneInfos,
+ barcodeLengthsForLane_);
+
+ CPPUNIT_ASSERT_EQUAL(laneInfos.back().getReadInfos().size(),3UL);
+ assertExpectedCycles(laneInfos.back(),
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<bcl2fastq::common::CycleRange>{bcl2fastq::common::CycleRange(0, 10),
+ bcl2fastq::common::CycleRange(0, 0),
+ bcl2fastq::common::CycleRange(0, 12)},
+ std::vector<bcl2fastq::common::CycleNumber>{0, 0, 0});
+}
+
+void TestLayout::testUmiCyclesAtEnd()
+{
+ bcl2fastq::config::BarcodeMismatchCountsContainer barcodeMismatches;
+
+ bcl2fastq::layout::LaneInfosContainer laneInfos;
+ getLaneInfos(laneInfos);
+
+ bcl2fastq::layout::detectReadLayout(
+ *runInfoXml_,
+ *sampleSheet_, // used for adapters only
+ std::vector<std::string>(), // use bases masks
+ 35,
+ 0, // read1StartCycle
+ 0, // read2StartCycle
+ 0, // read1EndCycle
+ 0, // read2EndCycle
+ 10, // read1UmiLength
+ 12, // read2UmiLength
+ 142, // read1UmiStartFromCycle
+ 140, // read2UmiStartFromCycle
+ false, // trimUmi
+ false,
+ barcodeMismatches,
+ laneInfos,
+ barcodeLengthsForLane_);
+
+ CPPUNIT_ASSERT_EQUAL(laneInfos.back().getReadInfos().size(),3UL);
+ assertExpectedCycles(laneInfos.back(),
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<bcl2fastq::common::CycleRange>{bcl2fastq::common::CycleRange(141, 151),
+ bcl2fastq::common::CycleRange(0, 0),
+ bcl2fastq::common::CycleRange(139, 151)},
+ std::vector<bcl2fastq::common::CycleNumber>{0, 0, 0});
+}
+
+void TestLayout::testUmiCyclesInMiddle()
+{
+ bcl2fastq::config::BarcodeMismatchCountsContainer barcodeMismatches;
+
+ bcl2fastq::layout::LaneInfosContainer laneInfos;
+ getLaneInfos(laneInfos);
+
+ CPPUNIT_ASSERT_THROW(
+ bcl2fastq::layout::detectReadLayout(
+ *runInfoXml_,
+ *sampleSheet_, // used for adapters only
+ std::vector<std::string>(), // use bases masks
+ 35,
+ 0, // read1StartCycle
+ 0, // read2StartCycle
+ 0, // read1EndCycle
+ 0, // read2EndCycle
+ 8, // read1UmiLength
+ 12, // read2UmiLength
+ 142, // read1UmiStartFromCycle
+ 140, // read2UmiStartFromCycle
+ false, // trimUmi
+ false,
+ barcodeMismatches,
+ laneInfos,
+ barcodeLengthsForLane_),
+ bcl2fastq::common::InputDataError);
+}
+
+void TestLayout::testUmiCyclesAtBeginningOfMaskedRead()
+{
+ bcl2fastq::config::BarcodeMismatchCountsContainer barcodeMismatches;
+
+ bcl2fastq::layout::LaneInfosContainer laneInfos;
+ getLaneInfos(laneInfos);
+
+ bcl2fastq::layout::detectReadLayout(
+ *runInfoXml_,
+ *sampleSheet_, // used for adapters only
+ std::vector<std::string>(), // use bases masks
+ 35,
+ 50, // read1StartCycle
+ 4, // read2StartCycle
+ 0, // read1EndCycle
+ 0, // read2EndCycle
+ 2, // read1UmiLength
+ 12, // read2UmiLength
+ 2, // read1UmiStartFromCycle
+ 3, // read2UmiStartFromCycle
+ false, // trimUmi
+ false,
+ barcodeMismatches,
+ laneInfos,
+ barcodeLengthsForLane_);
+
+ CPPUNIT_ASSERT_EQUAL(laneInfos.back().getReadInfos().size(),3UL);
+ assertExpectedCycles(laneInfos.back(),
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(102, 0, 50),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(148, 2, 163)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{},//ExpectedCycleInfosForRead(104, 0, 2),
+ //ExpectedCycleInfosForRead(8, 1, 152),
+ //ExpectedCycleInfosForRead(149, 2, 162)},
+ std::vector<bcl2fastq::common::CycleRange>{bcl2fastq::common::CycleRange(0, 2),
+ bcl2fastq::common::CycleRange(0, 0),
+ bcl2fastq::common::CycleRange(0, 12)},
+ std::vector<bcl2fastq::common::CycleNumber>{2, 0, 1});
+
+ CPPUNIT_ASSERT_EQUAL(laneInfos.back().getReadInfos().at(0).cyclesToLoad().size(), 104UL);
+}
+
+void TestLayout::testUmiCyclesAtEndOfMaskedRead()
+{
+ bcl2fastq::config::BarcodeMismatchCountsContainer barcodeMismatches;
+
+ bcl2fastq::layout::LaneInfosContainer laneInfos;
+ getLaneInfos(laneInfos);
+
+ bcl2fastq::layout::detectReadLayout(
+ *runInfoXml_,
+ *sampleSheet_, // used for adapters only
+ std::vector<std::string>(), // use bases masks
+ 35,
+ 2, // read1StartCycle
+ 3, // read2StartCycle
+ 100, // read1EndCycle
+ 99, // read2EndCycle
+ 10, // read1UmiLength
+ 12, // read2UmiLength
+ 142, // read1UmiStartFromCycle
+ 140, // read2UmiStartFromCycle
+ false, // trimUmi
+ false,
+ barcodeMismatches,
+ laneInfos,
+ barcodeLengthsForLane_);
+
+ CPPUNIT_ASSERT_EQUAL(laneInfos.back().getReadInfos().size(),3UL);
+ assertExpectedCycles(laneInfos.back(),
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(99, 0, 2),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(97, 2, 162)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{},//ExpectedCycleInfosForRead(109, 0, 2),
+ //ExpectedCycleInfosForRead(8, 1, 152),
+ //ExpectedCycleInfosForRead(149, 2, 162)},
+ std::vector<bcl2fastq::common::CycleRange>{bcl2fastq::common::CycleRange(99, 109),
+ bcl2fastq::common::CycleRange(0, 0),
+ bcl2fastq::common::CycleRange(97, 109)},
+ std::vector<bcl2fastq::common::CycleNumber>{0, 0, 0});
+}
+
+
+
+//
+// Repeate above but this time trim the UMI
+//
+
+
+void TestLayout::testDetectReadLayoutDefaultCyclesTrim()
+{
+ bcl2fastq::config::BarcodeMismatchCountsContainer barcodeMismatches;
+
+ bcl2fastq::layout::LaneInfosContainer laneInfos;
+ getLaneInfos(laneInfos);
+
+ bcl2fastq::layout::detectReadLayout(
+ *runInfoXml_,
+ *sampleSheet_, // used for adapters only
+ std::vector<std::string>(), // use bases masks
+ 35,
+ 0, // read1StartCycle
+ 0, // read2StartCycle
+ 0, // read1EndCycle
+ 0, // read2EndCycle
+ 0, // read1UmiLength
+ 0, // read2UmiLength
+ 1, // read1UmiStartFromCycle
+ 1, // read2UmiStartFromCycle
+ true, // trimUmi
+ false,
+ barcodeMismatches,
+ laneInfos,
+ barcodeLengthsForLane_);
+
+ CPPUNIT_ASSERT_EQUAL(laneInfos.back().getReadInfos().size(),3UL);
+ assertExpectedCycles(laneInfos.back(),
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<bcl2fastq::common::CycleRange>{bcl2fastq::common::CycleRange(0, 0),
+ bcl2fastq::common::CycleRange(0, 0),
+ bcl2fastq::common::CycleRange(0, 0)},
+ std::vector<bcl2fastq::common::CycleNumber>{0, 0, 0});
+}
+
+void TestLayout::testUmiCyclesAtBeginningTrim()
+{
+ bcl2fastq::config::BarcodeMismatchCountsContainer barcodeMismatches;
+
+ bcl2fastq::layout::LaneInfosContainer laneInfos;
+ getLaneInfos(laneInfos);
+
+ bcl2fastq::layout::detectReadLayout(
+ *runInfoXml_,
+ *sampleSheet_, // used for adapters only
+ std::vector<std::string>(), // use bases masks
+ 35,
+ 0, // read1StartCycle
+ 0, // read2StartCycle
+ 0, // read1EndCycle
+ 0, // read2EndCycle
+ 10, // read1UmiLength
+ 12, // read2UmiLength
+ 1, // read1UmiStartFromCycle
+ 1, // read2UmiStartFromCycle
+ true, // trimUmi
+ false,
+ barcodeMismatches,
+ laneInfos,
+ barcodeLengthsForLane_);
+
+ CPPUNIT_ASSERT_EQUAL(laneInfos.back().getReadInfos().size(),3UL);
+ assertExpectedCycles(laneInfos.back(),
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(141, 0, 11),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(139, 2, 172)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<bcl2fastq::common::CycleRange>{bcl2fastq::common::CycleRange(0, 10),
+ bcl2fastq::common::CycleRange(0, 0),
+ bcl2fastq::common::CycleRange(0, 12)},
+ std::vector<bcl2fastq::common::CycleNumber>{10, 0, 12});
+}
+
+void TestLayout::testUmiCyclesAtEndTrim()
+{
+ bcl2fastq::config::BarcodeMismatchCountsContainer barcodeMismatches;
+
+ bcl2fastq::layout::LaneInfosContainer laneInfos;
+ getLaneInfos(laneInfos);
+
+ bcl2fastq::layout::detectReadLayout(
+ *runInfoXml_,
+ *sampleSheet_, // used for adapters only
+ std::vector<std::string>(), // use bases masks
+ 35,
+ 0, // read1StartCycle
+ 0, // read2StartCycle
+ 0, // read1EndCycle
+ 0, // read2EndCycle
+ 10, // read1UmiLength
+ 12, // read2UmiLength
+ 142, // read1UmiStartFromCycle
+ 140, // read2UmiStartFromCycle
+ true, // trimUmi
+ false,
+ barcodeMismatches,
+ laneInfos,
+ barcodeLengthsForLane_);
+
+ CPPUNIT_ASSERT_EQUAL(laneInfos.back().getReadInfos().size(),3UL);
+ assertExpectedCycles(laneInfos.back(),
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(141, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(139, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<bcl2fastq::common::CycleRange>{bcl2fastq::common::CycleRange(141, 151),
+ bcl2fastq::common::CycleRange(0, 0),
+ bcl2fastq::common::CycleRange(139, 151)},
+ std::vector<bcl2fastq::common::CycleNumber>{0, 0, 0});
+}
+
+void TestLayout::testUmiCyclesInMiddleTrim()
+{
+ bcl2fastq::config::BarcodeMismatchCountsContainer barcodeMismatches;
+
+ bcl2fastq::layout::LaneInfosContainer laneInfos;
+ getLaneInfos(laneInfos);
+
+ CPPUNIT_ASSERT_THROW(
+ bcl2fastq::layout::detectReadLayout(
+ *runInfoXml_,
+ *sampleSheet_, // used for adapters only
+ std::vector<std::string>(), // use bases masks
+ 35,
+ 0, // read1StartCycle
+ 0, // read2StartCycle
+ 0, // read1EndCycle
+ 0, // read2EndCycle
+ 8, // read1UmiLength
+ 12, // read2UmiLength
+ 142, // read1UmiStartFromCycle
+ 140, // read2UmiStartFromCycle
+ true, // trimUmi
+ false,
+ barcodeMismatches,
+ laneInfos,
+ barcodeLengthsForLane_),
+ bcl2fastq::common::InputDataError);
+}
+
+
+
+
+void TestLayout::testUmiCyclesAtBeginningOfMaskedReadTrim()
+{
+ bcl2fastq::config::BarcodeMismatchCountsContainer barcodeMismatches;
+
+ bcl2fastq::layout::LaneInfosContainer laneInfos;
+ getLaneInfos(laneInfos);
+
+ bcl2fastq::layout::detectReadLayout(
+ *runInfoXml_,
+ *sampleSheet_, // used for adapters only
+ std::vector<std::string>(), // use bases masks
+ 35,
+ 50, // read1StartCycle
+ 4, // read2StartCycle
+ 0, // read1EndCycle
+ 0, // read2EndCycle
+ 2, // read1UmiLength
+ 12, // read2UmiLength
+ 2, // read1UmiStartFromCycle
+ 3, // read2UmiStartFromCycle
+ true, // trimUmi
+ false,
+ barcodeMismatches,
+ laneInfos,
+ barcodeLengthsForLane_);
+
+ CPPUNIT_ASSERT_EQUAL(laneInfos.back().getReadInfos().size(),3UL);
+ assertExpectedCycles(laneInfos.back(),
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(102, 0, 50),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(137, 2, 174)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{},//ExpectedCycleInfosForRead(104, 0, 2),
+ //ExpectedCycleInfosForRead(8, 1, 152),
+ //ExpectedCycleInfosForRead(149, 2, 162)},
+ std::vector<bcl2fastq::common::CycleRange>{bcl2fastq::common::CycleRange(0, 2),
+ bcl2fastq::common::CycleRange(0, 0),
+ bcl2fastq::common::CycleRange(0, 12)},
+ std::vector<bcl2fastq::common::CycleNumber>{2, 0, 12});
+
+ CPPUNIT_ASSERT_EQUAL(laneInfos.back().getReadInfos().at(0).cyclesToLoad().size(), 104UL);
+}
+
+void TestLayout::testUmiCyclesAtEndOfMaskedReadTrim()
+{
+ bcl2fastq::config::BarcodeMismatchCountsContainer barcodeMismatches;
+
+ bcl2fastq::layout::LaneInfosContainer laneInfos;
+ getLaneInfos(laneInfos);
+
+ bcl2fastq::layout::detectReadLayout(
+ *runInfoXml_,
+ *sampleSheet_, // used for adapters only
+ std::vector<std::string>(), // use bases masks
+ 35,
+ 2, // read1StartCycle
+ 3, // read2StartCycle
+ 100, // read1EndCycle
+ 99, // read2EndCycle
+ 10, // read1UmiLength
+ 12, // read2UmiLength
+ 140, // read1UmiStartFromCycle
+ 138, // read2UmiStartFromCycle
+ true, // trimUmi
+ false,
+ barcodeMismatches,
+ laneInfos,
+ barcodeLengthsForLane_);
+
+ CPPUNIT_ASSERT_EQUAL(laneInfos.back().getReadInfos().size(),3UL);
+ assertExpectedCycles(laneInfos.back(),
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(99, 0, 2),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(97, 2, 162)},
+ std::vector<ExpectedCycleInfosForRead>{ExpectedCycleInfosForRead(151, 0, 1),
+ ExpectedCycleInfosForRead(8, 1, 152),
+ ExpectedCycleInfosForRead(151, 2, 160)},
+ std::vector<ExpectedCycleInfosForRead>{},//ExpectedCycleInfosForRead(109, 0, 2),
+ //ExpectedCycleInfosForRead(8, 1, 152),
+ //ExpectedCycleInfosForRead(149, 2, 162)},
+ std::vector<bcl2fastq::common::CycleRange>{bcl2fastq::common::CycleRange(99, 109),
+ bcl2fastq::common::CycleRange(0, 0),
+ bcl2fastq::common::CycleRange(97, 109)},
+ std::vector<bcl2fastq::common::CycleNumber>{0, 0, 0});
+}
+
diff --git a/src/cxx/lib/layout/cppunit/testLayout.hh b/src/cxx/lib/layout/cppunit/testLayout.hh
new file mode 100644
index 0000000..47220ac
--- /dev/null
+++ b/src/cxx/lib/layout/cppunit/testLayout.hh
@@ -0,0 +1,104 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testLayout.hh
+ *
+ * \brief Layout cppunit test declarations.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_TEST_LAYOUT_HH
+#define BCL2FASTQ_LAYOUT_TEST_LAYOUT_HH
+
+
+#include <string>
+#include <memory>
+
+#include "config/SampleSheetCsv.hh"
+#include "layout/RunInfoXml.hh"
+
+#include <cppunit/extensions/HelperMacros.h>
+
+
+/// \brief Test suite for Layout.
+class TestLayout : public CppUnit::TestFixture
+{
+ CPPUNIT_TEST_SUITE(TestLayout);
+ CPPUNIT_TEST(testDetectReadLayoutDefaultCycles);
+ CPPUNIT_TEST(testUmiCyclesAtBeginning);
+ CPPUNIT_TEST(testUmiCyclesAtEnd);
+ CPPUNIT_TEST(testUmiCyclesInMiddle);
+ CPPUNIT_TEST(testUmiCyclesAtBeginningOfMaskedRead);
+ CPPUNIT_TEST(testUmiCyclesAtEndOfMaskedRead);
+
+ CPPUNIT_TEST(testDetectReadLayoutDefaultCyclesTrim);
+ CPPUNIT_TEST(testUmiCyclesAtBeginningTrim);
+ CPPUNIT_TEST(testUmiCyclesAtEndTrim);
+ CPPUNIT_TEST(testUmiCyclesInMiddleTrim);
+ CPPUNIT_TEST(testUmiCyclesAtBeginningOfMaskedReadTrim);
+ CPPUNIT_TEST(testUmiCyclesAtEndOfMaskedReadTrim);
+ CPPUNIT_TEST_SUITE_END();
+
+private:
+ struct ExpectedCycleInfosForRead
+ {
+ ExpectedCycleInfosForRead(size_t numCycles,
+ bcl2fastq::common::ReadNumber readNumber,
+ bcl2fastq::common::CycleNumber firstCycle)
+ : numCycles_(numCycles)
+ , readNumber_(readNumber)
+ , firstCycle_(firstCycle)
+ {
+ }
+
+ size_t numCycles_;
+ bcl2fastq::common::ReadNumber readNumber_;
+ bcl2fastq::common::CycleNumber firstCycle_;
+ };
+
+ void getLaneInfos(bcl2fastq::layout::LaneInfosContainer& laneInfos);
+ void assertExpectedCycles(const bcl2fastq::layout::LaneInfo& laneInfo,
+ const std::vector<ExpectedCycleInfosForRead>& expectedCycles,
+ const std::vector<ExpectedCycleInfosForRead>& expectedUnmaskedCycles,
+ const std::vector<ExpectedCycleInfosForRead>& expectedCyclesToLoad,
+ const std::vector<bcl2fastq::common::CycleRange>& umiCycles,
+ const std::vector<bcl2fastq::common::CycleNumber>& bclBufferOffsets);
+
+ template<typename T>
+ void verifyCycles(bcl2fastq::common::CycleNumber startCycle,
+ const T& cycleInfos);
+
+ std::vector<std::vector<size_t>> barcodeLengthsForLane_;
+ std::shared_ptr<bcl2fastq::layout::RunInfoXml> runInfoXml_;
+ std::shared_ptr<bcl2fastq::config::SampleSheetCsv> sampleSheet_;
+
+public:
+ TestLayout() : barcodeLengthsForLane_(), runInfoXml_(), sampleSheet_() { }
+ void setUp();
+ void tearDown();
+
+ void testDetectReadLayoutDefaultCycles();
+ void testUmiCyclesAtBeginning();
+ void testUmiCyclesAtEnd();
+ void testUmiCyclesInMiddle();
+ void testUmiCyclesAtBeginningOfMaskedRead();
+ void testUmiCyclesAtEndOfMaskedRead();
+
+ void testDetectReadLayoutDefaultCyclesTrim();
+ void testUmiCyclesAtBeginningTrim();
+ void testUmiCyclesAtEndTrim();
+ void testUmiCyclesInMiddleTrim();
+ void testUmiCyclesAtBeginningOfMaskedReadTrim();
+ void testUmiCyclesAtEndOfMaskedReadTrim();
+};
+
+#endif // BCL2FASTQ_LAYOUT_TEST_LAYOUT_HH
+
+
diff --git a/src/cxx/lib/layout/cppunit/testRunInfoXml.cpp b/src/cxx/lib/layout/cppunit/testRunInfoXml.cpp
new file mode 100644
index 0000000..d4807b4
--- /dev/null
+++ b/src/cxx/lib/layout/cppunit/testRunInfoXml.cpp
@@ -0,0 +1,98 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testRunInfoXml.cpp
+ *
+ * \brief Sample sheet helper cppunit test declarations.
+ *
+ * \author Marek Balint
+ */
+
+
+#include <string>
+
+#include "RegistryName.hh"
+#include "testRunInfoXml.hh"
+
+#include "io/Xml.hh"
+#include "layout/RunInfoXml.hh"
+
+
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(TestRunInfoXml, registryName("RunInfoXml"));
+
+
+void TestRunInfoXml::setUp()
+{
+}
+
+void TestRunInfoXml::tearDown()
+{
+}
+
+void TestRunInfoXml::testNova()
+{
+ const std::string runInfoXmlData(
+ "<?xml version=\"1.0\"?>\n"
+ "<RunInfo xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" Version=\"2\">\n"
+ " <Run Id=\"TestRun\" Number=\"42\">\n"
+ " <Flowcell>TestFlowcell</Flowcell>\n"
+ " <Instrument>CSSIM</Instrument>\n"
+ " <Date>8/20/2013 3:29:17 PM</Date>\n"
+ " <Reads>\n"
+ " <Read FirstCycle=\"1\" LastCycle=\"91\" />\n"
+ " <Read NumCycles=\"8\" IsIndexedRead=\"Y\" />\n"
+ " <Read FirstCycle=\"100\" LastCycle=\"107\" >\n"
+ " <Index />\n"
+ " </Read>\n"
+ " <Read FirstCycle=\"108\" LastCycle=\"200\" IsIndexedRead=\"N\" />\n"
+ " </Reads>\n"
+ " <FlowcellLayout LaneCount=\"4\" SurfaceCount=\"2\" SwathCount=\"3\" TileCount=\"11\" SectionPerLane=\"3\" LanePerSection=\"2\" />\n"
+ " </Run>\n"
+ "</RunInfo>\n"
+ );
+
+ bcl2fastq::layout::RunInfoXml runInfoXml(bcl2fastq::io::parseXmlData(
+ runInfoXmlData.begin(),
+ runInfoXmlData.end()
+ ));
+
+ CPPUNIT_ASSERT_EQUAL(runInfoXml.getInstrument(), std::string("CSSIM"));
+ CPPUNIT_ASSERT_EQUAL(runInfoXml.getRunNumber(), std::string("42"));
+ CPPUNIT_ASSERT_EQUAL(runInfoXml.getFlowcellId(), std::string("TestFlowcell"));
+ CPPUNIT_ASSERT_EQUAL(runInfoXml.getLanesCount(), bcl2fastq::common::LaneNumber(4));
+
+ bcl2fastq::layout::ReadMetadataContainer::const_iterator readMetadataIter = runInfoXml.readMetadataBegin();
+ const bcl2fastq::layout::ReadMetadataContainer::const_iterator readMetadataEnd = runInfoXml.readMetadataEnd();
+ CPPUNIT_ASSERT(readMetadataIter != readMetadataEnd);
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->readNumber_, bcl2fastq::common::ReadNumber(1));
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->firstCycle_, bcl2fastq::common::CycleNumber(1));
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->lastCycle_, bcl2fastq::common::CycleNumber(91));
+ ++readMetadataIter;
+ CPPUNIT_ASSERT(readMetadataIter != readMetadataEnd);
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->readNumber_, bcl2fastq::common::ReadNumber(1));
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->firstCycle_, bcl2fastq::common::CycleNumber(92));
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->lastCycle_, bcl2fastq::common::CycleNumber(99));
+ ++readMetadataIter;
+ CPPUNIT_ASSERT(readMetadataIter != readMetadataEnd);
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->readNumber_, bcl2fastq::common::ReadNumber(2));
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->firstCycle_, bcl2fastq::common::CycleNumber(100));
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->lastCycle_, bcl2fastq::common::CycleNumber(107));
+ ++readMetadataIter;
+ CPPUNIT_ASSERT(readMetadataIter != readMetadataEnd);
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->readNumber_, bcl2fastq::common::ReadNumber(2));
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->firstCycle_, bcl2fastq::common::CycleNumber(108));
+ CPPUNIT_ASSERT_EQUAL(readMetadataIter->lastCycle_, bcl2fastq::common::CycleNumber(200));
+ ++readMetadataIter;
+ CPPUNIT_ASSERT(readMetadataIter == readMetadataEnd);
+}
+
+
diff --git a/src/cxx/lib/layout/cppunit/testRunInfoXml.hh b/src/cxx/lib/layout/cppunit/testRunInfoXml.hh
new file mode 100644
index 0000000..762bcfe
--- /dev/null
+++ b/src/cxx/lib/layout/cppunit/testRunInfoXml.hh
@@ -0,0 +1,47 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testRunInfoXml.hh
+ *
+ * \brief RunInfo.xml helper cppunit test declarations.
+ *
+ * \author Marek Balint
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_TEST_RUNINFOXML_HH
+#define BCL2FASTQ_LAYOUT_TEST_RUNINFOXML_HH
+
+
+#include <string>
+
+#include <cppunit/extensions/HelperMacros.h>
+
+#include "layout/RunInfoXml.hh"
+
+
+/// \brief Test suite for RunInfoXml.
+class TestRunInfoXml : public CppUnit::TestFixture
+{
+private:
+
+ CPPUNIT_TEST_SUITE(TestRunInfoXml);
+ CPPUNIT_TEST(testNova);
+ CPPUNIT_TEST_SUITE_END();
+
+public:
+
+ void setUp();
+ void tearDown();
+ void testNova();
+};
+
+
+#endif // #ifndef BCL2FASTQ_LAYOUT_TEST_RUNINFOXML_HH
+
+
diff --git a/src/cxx/lib/layout/cppunit/testUseBasesMask.cpp b/src/cxx/lib/layout/cppunit/testUseBasesMask.cpp
new file mode 100644
index 0000000..4dab6de
--- /dev/null
+++ b/src/cxx/lib/layout/cppunit/testUseBasesMask.cpp
@@ -0,0 +1,399 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testUseBasesMask.cpp
+ *
+ * \brief UseBasesMask cppunit test declarations.
+ *
+ * \author Aaron Day
+ */
+
+
+#include <string>
+
+#include <boost/foreach.hpp>
+#include <boost/assign/list_of.hpp>
+
+#include "RegistryName.hh"
+#include "testUseBasesMask.hh"
+
+#include "layout/UseBasesMask.hh"
+
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(TestUseBasesMask, registryName("UseBasesMask"));
+
+
+void TestUseBasesMask::setUp()
+{
+ using boost::assign::list_of;
+
+ readMetadata_.push_back( list_of(std::make_pair(1,51))
+ (std::make_pair(52,58))
+ (std::make_pair(59,65))
+ (std::make_pair(66,116)) );
+ readMetadata_.back()[2].readType_ = bcl2fastq::common::ReadType::INDEX;
+ readMetadata_.back()[3].readType_ = bcl2fastq::common::ReadType::INDEX;
+
+ readMetadata_.push_back( list_of(std::make_pair(1,52))
+ (std::make_pair(53,63)) );
+
+ readMetadata_.push_back( list_of(std::make_pair(1,12))
+ (std::make_pair(13,19)) );
+
+ readMetadata_.push_back( list_of(std::make_pair(1,21))
+ (std::make_pair(22,42)) );
+
+ readMetadata_.push_back( list_of(std::make_pair(1,100)) );
+
+ readMetadata_.push_back( list_of(std::make_pair(5,5)) );
+}
+
+void TestUseBasesMask::tearDown()
+{
+}
+
+void TestUseBasesMask::testAllButOne()
+{
+ bcl2fastq::layout::UseBasesMask useBases("y50n,I6n,I6n,y50n",
+ readMetadata_[0].begin(),
+ readMetadata_[0].end());
+
+ bcl2fastq::layout::ReadMetadataContainer::const_iterator iter = useBases.readMetadataBegin();
+
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 50UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 52UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 57UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 59UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 64UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 66UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 115UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter == useBases.readMetadataEnd());
+}
+
+void TestUseBasesMask::testMaskingR1()
+{
+ bcl2fastq::layout::UseBasesMask useBases("y50nn,y10n",
+ readMetadata_[1].begin(),
+ readMetadata_[1].end());
+
+ bcl2fastq::layout::ReadMetadataContainer::const_iterator iter = useBases.readMetadataBegin();
+
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 50UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 53UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 62UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter == useBases.readMetadataEnd());
+}
+
+void TestUseBasesMask::testMaskingR2()
+{
+ bcl2fastq::layout::UseBasesMask useBases("y12,y5nn",
+ readMetadata_[2].begin(),
+ readMetadata_[2].end());
+
+ bcl2fastq::layout::ReadMetadataContainer::const_iterator iter = useBases.readMetadataBegin();
+
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 12UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 13UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 17UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter == useBases.readMetadataEnd());
+}
+
+void TestUseBasesMask::testMixedDataIndex()
+{
+ bcl2fastq::layout::UseBasesMask useBases("y10i10n,i10y11",
+ readMetadata_[3].begin(),
+ readMetadata_[3].end());
+
+ bcl2fastq::layout::ReadMetadataContainer::const_iterator iter = useBases.readMetadataBegin();
+
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 10UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 11UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 20UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 22UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 31UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 32UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 42UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter == useBases.readMetadataEnd());
+}
+
+void TestUseBasesMask::testExpansion()
+{
+ bcl2fastq::layout::UseBasesMask useBases("y10i*n,i10y*",
+ readMetadata_[3].begin(),
+ readMetadata_[3].end());
+
+ bcl2fastq::layout::ReadMetadataContainer::const_iterator iter = useBases.readMetadataBegin();
+
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 10UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 11UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 20UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 22UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 31UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 32UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 42UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter == useBases.readMetadataEnd());
+
+ // Test * for 1 base
+ bcl2fastq::layout::UseBasesMask useBases2("i20n*,y*",
+ readMetadata_[3].begin(),
+ readMetadata_[3].end());
+
+ bcl2fastq::layout::ReadMetadataContainer::const_iterator iter2 = useBases2.readMetadataBegin();
+
+ CPPUNIT_ASSERT(iter2 != useBases2.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter2->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter2->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(iter2->firstCycle_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter2->lastCycle_, 20UL);
+
+ ++iter2;
+ CPPUNIT_ASSERT(iter2 != useBases2.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter2->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter2->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter2->firstCycle_, 22UL);
+ CPPUNIT_ASSERT_EQUAL(iter2->lastCycle_, 42UL);
+
+ ++iter2;
+ CPPUNIT_ASSERT(iter2 == useBases2.readMetadataEnd());
+}
+
+void TestUseBasesMask::testMixedDataIndexSingleRead()
+{
+ bcl2fastq::layout::UseBasesMask useBases("y40i10y40i10",
+ readMetadata_[4].begin(),
+ readMetadata_[4].end());
+
+ bcl2fastq::layout::ReadMetadataContainer::const_iterator iter = useBases.readMetadataBegin();
+
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 40UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 41UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 50UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 51UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 90UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 91UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 100UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter == useBases.readMetadataEnd());
+}
+
+void TestUseBasesMask::testIsomorphic()
+{
+ CPPUNIT_ASSERT_THROW(
+ bcl2fastq::layout::UseBasesMask useBases("y50n,y10", readMetadata_[1].begin(), readMetadata_[1].end()),
+ bcl2fastq::layout::UseBasesMaskFormatError
+ );
+}
+
+void TestUseBasesMask::testMismatchingCyclesInRead()
+{
+ // The number of cycles in each read do not match, but the total cycles match so it is ok.
+ bcl2fastq::layout::UseBasesMask useBases("y50n,y12", readMetadata_[1].begin(), readMetadata_[1].end());
+
+ bcl2fastq::layout::ReadMetadataContainer::const_iterator iter = useBases.readMetadataBegin();
+
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 50UL);
+
+ ++iter;
+
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 52UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 63UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter == useBases.readMetadataEnd());
+}
+
+void TestUseBasesMask::testZero()
+{
+ CPPUNIT_ASSERT_THROW(
+ bcl2fastq::layout::UseBasesMask useBases("y0n,y10", readMetadata_[5].begin(), readMetadata_[5].end()),
+ bcl2fastq::layout::UseBasesMaskFormatError
+ );
+}
+
+void TestUseBasesMask::testMissingRead()
+{
+ bcl2fastq::layout::UseBasesMask useBases("Y*,I5nn,n*,Y*",
+ readMetadata_[0].begin(),
+ readMetadata_[0].end());
+
+ bcl2fastq::layout::ReadMetadataContainer::const_iterator iter = useBases.readMetadataBegin();
+
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 51UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 52UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 56UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 66UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 116UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter == useBases.readMetadataEnd());
+}
+
+void TestUseBasesMask::testMissingAsteriskNone()
+{
+ // The "n*" matches 0 bases
+ bcl2fastq::layout::UseBasesMask useBases("Y*,I5nn,i7n*,Y*",
+ readMetadata_[0].begin(),
+ readMetadata_[0].end());
+
+ bcl2fastq::layout::ReadMetadataContainer::const_iterator iter = useBases.readMetadataBegin();
+
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 51UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 1UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 52UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 56UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::INDEX);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 59UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 65UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter != useBases.readMetadataEnd());
+ CPPUNIT_ASSERT_EQUAL(iter->readNumber_, 2UL);
+ CPPUNIT_ASSERT_EQUAL(iter->readType_, bcl2fastq::common::ReadType::DATA);
+ CPPUNIT_ASSERT_EQUAL(iter->firstCycle_, 66UL);
+ CPPUNIT_ASSERT_EQUAL(iter->lastCycle_, 116UL);
+
+ ++iter;
+ CPPUNIT_ASSERT(iter == useBases.readMetadataEnd());
+}
diff --git a/src/cxx/lib/layout/cppunit/testUseBasesMask.hh b/src/cxx/lib/layout/cppunit/testUseBasesMask.hh
new file mode 100644
index 0000000..7fc5a54
--- /dev/null
+++ b/src/cxx/lib/layout/cppunit/testUseBasesMask.hh
@@ -0,0 +1,68 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file testUseBasesMask.hh
+ *
+ * \brief UseBasesMask cppunit test declarations.
+ *
+ * \author Aaron Day
+ */
+
+
+#ifndef BCL2FASTQ_LAYOUT_TEST_USE_BASES_MASK_HH
+#define BCL2FASTQ_LAYOUT_TEST_USE_BASES_MASK_HH
+
+
+#include <string>
+
+#include "layout/ReadMetadata.hh"
+
+#include <cppunit/extensions/HelperMacros.h>
+
+
+/// \brief Test suite for UseBasesMask.
+class TestUseBasesMask : public CppUnit::TestFixture
+{
+ CPPUNIT_TEST_SUITE(TestUseBasesMask);
+ CPPUNIT_TEST(testAllButOne);
+ CPPUNIT_TEST(testMaskingR1);
+ CPPUNIT_TEST(testMaskingR2);
+ CPPUNIT_TEST(testMixedDataIndex);
+ CPPUNIT_TEST(testExpansion);
+ CPPUNIT_TEST(testMixedDataIndexSingleRead);
+ CPPUNIT_TEST(testIsomorphic);
+ CPPUNIT_TEST(testMismatchingCyclesInRead);
+ CPPUNIT_TEST(testZero);
+ CPPUNIT_TEST(testMissingRead);
+ CPPUNIT_TEST(testMissingAsteriskNone);
+ CPPUNIT_TEST_SUITE_END();
+
+private:
+ std::vector< bcl2fastq::layout::ReadMetadataContainer > readMetadata_;
+
+public:
+ TestUseBasesMask() : readMetadata_(0) {}
+ void setUp();
+ void tearDown();
+ void testAllButOne();
+ void testMaskingR1();
+ void testMaskingR2();
+ void testMixedDataIndex();
+ void testExpansion();
+ void testMixedDataIndexSingleRead();
+ void testIsomorphic();
+ void testMismatchingCyclesInRead();
+ void testZero();
+ void testMissingRead();
+ void testMissingAsteriskNone();
+};
+
+
+#endif // BCL2FASTQ_LAYOUT_TEST_USE_BASES_MASK_HH
+
+
diff --git a/src/cxx/lib/stats/CMakeLists.txt b/src/cxx/lib/stats/CMakeLists.txt
new file mode 100644
index 0000000..6f829c4
--- /dev/null
+++ b/src/cxx/lib/stats/CMakeLists.txt
@@ -0,0 +1,20 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the cxx/lib/reports subfolder
+##
+## author Mauricio Varea
+##
+################################################################################
+
+include(${BCL2FASTQ_CXX_LIBRARY_CMAKE})
diff --git a/src/cxx/lib/stats/ConversionStatsXml.cpp b/src/cxx/lib/stats/ConversionStatsXml.cpp
new file mode 100644
index 0000000..e959e4b
--- /dev/null
+++ b/src/cxx/lib/stats/ConversionStatsXml.cpp
@@ -0,0 +1,110 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file ConversionStatsXml.cpp
+ *
+ * \brief Xml Serialization of Conversion statistics.
+ *
+ * \author Mauricio Varea
+ */
+
+#include <boost/lexical_cast.hpp>
+
+#include "stats/ConversionStatsXml.hh"
+
+namespace bcl2fastq
+{
+namespace stats
+{
+
+ConversionStatsXml::ConversionStatsXml()
+{
+}
+
+boost::property_tree::path ConversionStatsXml::getPrefix(
+ const std::string &flowcellId,
+ const std::string &projectName,
+ const std::string &sampleName,
+ const std::string &barcodeName,
+ common::TileNumber tileNumber,
+ const unsigned lane )
+{
+ return boost::property_tree::path( "Stats"
+ "/<indexed>Flowcell/<flowcell-id>" + flowcellId
+ +"/<indexed>Project/<name>" + projectName
+ +"/<indexed>Sample/<name>" + sampleName
+ +"/<indexed>Barcode/<name>" + barcodeName
+ +"/<indexed>Lane/<number>" + boost::lexical_cast<std::string>(lane)
+ +"/<indexed>Tile/<number>" + boost::lexical_cast<std::string>(tileNumber)
+ , '/');
+}
+
+void ConversionStatsXml::addLaneTile(
+ const std::string &flowcellId,
+ const std::string &projectName,
+ const std::string &sampleName,
+ const std::string &barcodeName,
+ common::TileNumber tileNumber,
+ const unsigned lane,
+ const TileBarcodeStats& tileStats)
+{
+ for(TileBarcodeStats::size_type i=0; i < tileStats.size(); i++)
+ {
+ add( getPrefix( flowcellId, projectName, sampleName, barcodeName, tileNumber, lane )
+ / TileBarcodeStats::str_type[i] / "ClusterCount", tileStats[i].clusterCount );
+ }
+}
+
+void ConversionStatsXml::addLaneTileRead(
+ const std::string &flowcellId,
+ const std::string &projectName,
+ const std::string &sampleName,
+ const std::string &barcodeName,
+ common::TileNumber tileNumber,
+ common::ReadNumber readNumber,
+ const unsigned lane,
+ const ReadBarcodeStats& tileStats)
+{
+ const boost::property_tree::path readNode( "<indexed>Read/<number>"
+ + boost::lexical_cast<std::string>(readNumber), '/');
+
+ for(TileBarcodeStats::size_type i=0; i < tileStats.size(); i++)
+ {
+ add( getPrefix( flowcellId, projectName, sampleName, barcodeName, tileNumber, lane )
+ / TileBarcodeStats::str_type[i] / readNode / "Yield", tileStats[i].yield );
+ add( getPrefix( flowcellId, projectName, sampleName, barcodeName, tileNumber, lane )
+ / TileBarcodeStats::str_type[i] / readNode / "YieldQ30", tileStats[i].yieldQ30 );
+ add( getPrefix( flowcellId, projectName, sampleName, barcodeName, tileNumber, lane )
+ / TileBarcodeStats::str_type[i] / readNode / "QualityScoreSum", tileStats[i].qualityScoreSum );
+ }
+}
+
+
+void ConversionStatsXml::addFlowcellLane(
+ const std::string &flowcellId,
+ const unsigned lane,
+ const stats::BarcodeHits::Popular &topUnknownBarcodes)
+{
+ const boost::property_tree::path laneValuePrefix("Stats"
+ "/<indexed>Flowcell/<flowcell-id>" + flowcellId
+ +"/<indexed>Lane/<number>" + boost::lexical_cast<std::string>(lane)
+ +"/TopUnknownBarcodes", '/');
+
+ BOOST_FOREACH(const stats::BarcodeHits::Popular::value_type &unknownBarcode, topUnknownBarcodes)
+ {
+ add( laneValuePrefix / ("<indexed>Barcode/<sequence>" + unknownBarcode.first.toString()).c_str()
+ / "<xmlattr>"
+ / "count",
+ unknownBarcode.second );
+ }
+}
+
+
+} //namespace stats
+} //namespace bcl2fastq
+
diff --git a/src/cxx/lib/stats/DemultiplexingStatsXml.cpp b/src/cxx/lib/stats/DemultiplexingStatsXml.cpp
new file mode 100644
index 0000000..55989cb
--- /dev/null
+++ b/src/cxx/lib/stats/DemultiplexingStatsXml.cpp
@@ -0,0 +1,70 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file \file DemultiplexingStatsXml.cpp
+ *
+ * \brief Xml Serialization of Demultiplexing statistics.
+ *
+ * \author Mauricio Varea
+ */
+
+#include <boost/lexical_cast.hpp>
+
+#include "stats/DemultiplexingStatsXml.hh"
+
+namespace bcl2fastq
+{
+namespace stats
+{
+
+DemultiplexingStatsXml::DemultiplexingStatsXml()
+{
+}
+
+
+void DemultiplexingStatsXml::addLaneBarcode(
+ const std::string &flowcellId,
+ const std::string &projectName,
+ const std::string &sampleName,
+ const std::string &barcodeName,
+ const unsigned lane,
+ const stats::BarcodeStats& barcodeStats)
+{
+ std::string matchCountStrs[stats::BarcodeStats::MAX_MISMATCHES+1] =
+ {"PerfectBarcodeCount",
+ "OneMismatchBarcodeCount",
+ "TwoMismatchesBarcodeCount",
+ "ThreeMismatchesBarcodeCount",
+ "FourMismatchesBarcodeCount",
+ "FiveMismatchesBarcodeCount"};
+
+ const boost::property_tree::path tileValuePrefix("Stats"
+ "/<indexed>Flowcell/<flowcell-id>" + flowcellId
+ +"/<indexed>Project/<name>" + projectName
+ +"/<indexed>Sample/<name>" + sampleName
+ +"/<indexed>Barcode/<name>" + barcodeName
+ +"/<indexed>Lane/<number>" + boost::lexical_cast<std::string>(lane)
+ , '/');
+
+ add(tileValuePrefix / "BarcodeCount", barcodeStats.getBarcodeCount());
+
+ auto pos = std::find_if(barcodeStats.getBarcodeMismatchCounts().rbegin(),
+ barcodeStats.getBarcodeMismatchCounts().rend(),
+ [](common::ClustersCount count) {return count != 0; });
+
+ size_t numMismatches = 0;
+ for (auto iter = barcodeStats.getBarcodeMismatchCounts().begin(); iter != pos.base(); ++iter)
+ {
+ add(tileValuePrefix / matchCountStrs[numMismatches++].c_str(), *iter);
+ }
+}
+
+
+} //namespace stats
+} //namespace bcl2fastq
+
diff --git a/src/cxx/lib/stats/DemuxReportGenerator.cpp b/src/cxx/lib/stats/DemuxReportGenerator.cpp
new file mode 100644
index 0000000..1561677
--- /dev/null
+++ b/src/cxx/lib/stats/DemuxReportGenerator.cpp
@@ -0,0 +1,184 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file DemuxReportGenerator.cpp
+ *
+ * \brief Stats
+ *
+ * \author Mauricio Varea
+ */
+
+#include <boost/filesystem.hpp>
+#include <boost/foreach.hpp>
+#include <boost/format.hpp>
+#include <boost/assign/list_of.hpp>
+
+#include "config.h"
+
+#include <libxslt/xslt.h>
+#include <libxslt/xsltInternals.h>
+#include <libxslt/transform.h>
+#include <libxslt/xsltutils.h>
+#include <libexslt/exslt.h>
+
+#include "common/Debug.hh"
+#include "common/FileSystem.hh"
+#include "common/InstallationPath.hh"
+#include "layout/Layout.hh"
+
+#include "stats/DemuxReportGenerator.hh"
+
+#include <set>
+
+#ifndef WIN32
+extern int xmlLoadExtDtdDefaultValue;
+#endif
+
+namespace bcl2fastq
+{
+namespace stats
+{
+
+
+DemuxReportGenerator::DemuxReportGenerator( const layout::Layout &flowcellLayout,
+ const boost::filesystem::path &outputDirectory )
+: outputDirectoryHtml_(outputDirectory/"html")
+{
+ std::vector<boost::filesystem::path> createList =
+ boost::assign::list_of(outputDirectory)(outputDirectoryHtml_);
+ const boost::filesystem::path flowcellHtmlPath = outputDirectoryHtml_ / flowcellLayout.getFlowcellInfo().getFlowcellId();
+ createList.push_back(flowcellHtmlPath);
+ createList.push_back(flowcellHtmlPath/"all");
+ createList.push_back(flowcellHtmlPath/"all"/"all");
+ createList.push_back(flowcellHtmlPath/"all"/"all"/"all");
+ createList.push_back(flowcellHtmlPath/"all"/"all"/std::string(layout::Barcode::DEFAULT_BARCODE));
+
+ BOOST_FOREACH (const layout::LaneInfo &laneInfo, std::make_pair(flowcellLayout.laneInfosBegin(), flowcellLayout.laneInfosEnd()))
+ {
+ std::map<std::string, std::string> uniqueSampleNameMap;
+ makeUniqueSampleNameMap(laneInfo,
+ uniqueSampleNameMap);
+
+ BOOST_FOREACH (const layout::SampleInfo &sampleInfo, std::make_pair(laneInfo.sampleInfosBegin(), laneInfo.sampleInfosEnd()))
+ {
+ const std::string sampleName = uniqueSampleNameMap[sampleInfo.getId()];
+
+ /* No need to worry about repeated directories, due to more than one sample belonging to
+ the same project, as createDirectories() should take care of this */
+ layout::SampleInfo::Project projectName = sampleInfo.getProject();
+ createList.push_back(flowcellHtmlPath/projectName);
+ createList.push_back(flowcellHtmlPath/projectName/"all");
+ createList.push_back(flowcellHtmlPath/projectName/"all"/"all");
+ createList.push_back(flowcellHtmlPath/projectName/"all"/std::string(layout::Barcode::DEFAULT_BARCODE));
+ createList.push_back(flowcellHtmlPath/projectName/sampleName);
+ createList.push_back(flowcellHtmlPath/projectName/sampleName/"all");
+ if ( sampleInfo.hasBarcodes() )
+ {
+ for (const auto& barcode : sampleInfo.getBarcodes())
+ {
+ createList.push_back(flowcellHtmlPath/projectName/sampleName/barcode.toString());
+ }
+ } //else {
+ createList.push_back(flowcellHtmlPath/projectName/sampleName/layout::Barcode::DEFAULT_BARCODE);
+ //}
+ }
+ }
+ common::createDirectories(createList);
+}
+
+
+void DemuxReportGenerator::run( const boost::filesystem::path &demultiplexingStatsXmlPath,
+ const boost::filesystem::path &conversionStatsXmlPath )
+{
+ const char *params[] = {"OUTPUT_DIRECTORY_HTML_PARAM", "''",
+ "DEMULTIPLEXING_STATS_XML_PARAM", "''",
+ "BCL2FASTQ_FULL_DATADIR_PARAM", "''",
+ 0};
+ std::string quotedOutputHtmlDirectory = "'" + outputDirectoryHtml_.string() + "'";
+
+ std::string quotedDemultiplexingStatsXml= "'" + demultiplexingStatsXmlPath.string() + "'";
+
+ const boost::filesystem::path fullDataDir(common::InstallationPath::getSingleton().expandPath(BCL2FASTQ_DATADIR));
+ std::string quotedFullDataDirPath = "'" + fullDataDir.string() + "'";
+
+ boost::filesystem::path reportGenerator(fullDataDir / "xsl" / "demux" / "GenerateReport.xsl");
+
+ // When executing under cygwin, generic_string must be called to get the forward slashes.
+ // Otherwise, xsltParseStylesheetFile will fail with cryptic error messages.
+ std::string reportGeneratorStr = reportGenerator.string();
+
+#ifdef WIN32
+ std::replace(quotedOutputHtmlDirectory.begin(), quotedOutputHtmlDirectory.end(), '\\', '/');
+ std::replace(quotedDemultiplexingStatsXml.begin(), quotedDemultiplexingStatsXml.end(), '\\', '/');
+ std::replace(quotedFullDataDirPath.begin(), quotedFullDataDirPath.end(), '\\', '/');
+ std::replace(reportGeneratorStr.begin(), reportGeneratorStr.end(), '\\', '/');
+#endif
+
+ params[1] = quotedOutputHtmlDirectory.c_str();
+ params[3] = quotedDemultiplexingStatsXml.c_str();
+ params[5] = quotedFullDataDirPath.c_str();
+
+ xmlSubstituteEntitiesDefault(1);
+ xmlLoadExtDtdDefaultValue = 1;
+
+ exsltRegisterAll();
+
+ xsltStylesheetPtr cur = xsltParseStylesheetFile((const xmlChar *) reportGeneratorStr.c_str() );
+ xmlDocPtr doc = xmlParseFile(conversionStatsXmlPath.string().c_str());
+ xmlDocPtr res = xsltApplyStylesheet(cur, doc, params);
+
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "ReportGenerator: Applying stylesheet " << reportGenerator << std::endl;
+ for(unsigned int i=0; i<( (sizeof(params)/sizeof(const char *))-2 ); i+=2)
+ {
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "ReportGenerator: --stringparam " << std::string(params[i]) << " " << std::string(params[i+1]) << std::endl;
+ }
+ BCL2FASTQ_LOG(common::LogLevel::DEBUG) << "ReportGenerator: Using data from " << conversionStatsXmlPath << std::endl;
+
+ xsltFreeStylesheet(cur);
+ xmlFreeDoc(res);
+ xmlFreeDoc(doc);
+
+ xsltCleanupGlobals();
+ xmlCleanupParser();
+ if (!res)
+ {
+ BOOST_THROW_EXCEPTION(LibXsltError());
+ }
+}
+
+void DemuxReportGenerator::makeUniqueSampleNameMap(const layout::LaneInfo& laneInfo,
+ std::map<std::string, std::string>& uniqueSampleNameMap)
+{
+ std::set<std::string> sampleNameSet;
+ std::set<std::string> duplicateNames;
+ BOOST_FOREACH(const layout::SampleInfo& sampleInfo, std::make_pair(laneInfo.sampleInfosBegin(), laneInfo.sampleInfosEnd()))
+ {
+ if (!sampleNameSet.insert(sampleInfo.getName()).second)
+ {
+ duplicateNames.insert(sampleInfo.getName());
+ }
+ }
+
+ BOOST_FOREACH(const layout::SampleInfo& sampleInfo, std::make_pair(laneInfo.sampleInfosBegin(), laneInfo.sampleInfosEnd()))
+ {
+ if (duplicateNames.count(sampleInfo.getName()) != 0)
+ {
+ uniqueSampleNameMap.insert(
+ std::make_pair(sampleInfo.getId(),
+ sampleInfo.getName() + "_S" + boost::lexical_cast<std::string>(sampleInfo.getNumber())));
+ }
+ else
+ {
+ uniqueSampleNameMap.insert(std::make_pair(sampleInfo.getId(), sampleInfo.getName()));
+ }
+ }
+}
+
+
+} // namespace stats
+} // namespace bcl2fastq
diff --git a/src/cxx/lib/stats/Json.cpp b/src/cxx/lib/stats/Json.cpp
new file mode 100644
index 0000000..fbf9cea
--- /dev/null
+++ b/src/cxx/lib/stats/Json.cpp
@@ -0,0 +1,107 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file JsonStats.cpp
+ *
+ * \brief Json creation
+ *
+ * \author Aaron Day
+ */
+
+#include "stats/Json.hh"
+
+namespace bcl2fastq
+{
+namespace stats
+{
+
+void JsonType::streamIndent(std::ostream& os, int depth) const
+{
+ for (int i = 0; i < depth; ++i)
+ {
+ os << " ";
+ }
+}
+
+void JsonString::stream(std::ostream& os, int depth /*= 0*/) const
+{
+ os << "\"" << value_ << "\"";
+}
+
+void JsonNumber::stream(std::ostream& os, int depth /*= 0*/) const
+{
+ if (value_ == std::numeric_limits<int64_t>::max())
+ {
+ os << "null";
+ }
+ else
+ {
+ os << value_;
+ }
+}
+
+void JsonBool::stream(std::ostream& os, int depth /*= 0*/) const
+{
+ os << (value_ ? "true" : "false");
+}
+
+
+void JsonArray::stream(std::ostream& os, int depth /*= 0*/) const
+{
+ os << "[" << std::endl;
+
+ bool first = true;
+ for (auto& jsonValue : values_)
+ {
+ if (first)
+ {
+ first = false;
+ }
+ else
+ {
+ os << "," << std::endl;
+ }
+
+ streamIndent(os, depth+1);
+ jsonValue->stream(os, depth+1);
+ }
+ os << std::endl;
+ streamIndent(os, depth);
+ os << "]";
+}
+
+void JsonObject::stream(std::ostream& os, int depth /*= 0*/) const
+{
+ os << "{" << std::endl;
+ int nextDepth = depth + 1;
+
+ bool first = true;
+ for (auto& jsonValue : attributes_)
+ {
+ if (first)
+ {
+ first = false;
+ }
+ else
+ {
+ os << "," << std::endl;
+ }
+
+ streamIndent(os, nextDepth);
+ os << "\"" << jsonValue.first << "\" : ";
+ jsonValue.second->stream(os, nextDepth);
+ }
+
+ os << std::endl;
+ streamIndent(os, depth);
+ os << "}";
+}
+
+} //namespace stats
+} //namespace bcl2fastq
+
diff --git a/src/cxx/lib/stats/TileStats.cpp b/src/cxx/lib/stats/TileStats.cpp
new file mode 100644
index 0000000..60b6ca2
--- /dev/null
+++ b/src/cxx/lib/stats/TileStats.cpp
@@ -0,0 +1,32 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file TileStats.cpp
+ *
+ * \brief Tile and Tile-Barcode Stats.
+ *
+ * \author Mauricio Varea
+ */
+
+#include "stats/TileStats.hpp"
+
+namespace bcl2fastq
+{
+namespace stats
+{
+
+// Do not be tempted to use [detail::TileStatsCount]
+// keeping the size hardcoded forces us to come back to this part of the code every time we do a change
+// in the header and forget to change it here as well
+template<> const char *TileStats<AllReadsStats>::str_type[detail::TileStatsCount] = {"Raw","Pf"};
+template<> const char *TileStats<ReadStats>::str_type[detail::TileStatsCount] = {"Raw","Pf"};
+
+
+} //namespace stats
+} //namespace bcl2fastq
+
diff --git a/src/cxx/libexec/CMakeLists.txt b/src/cxx/libexec/CMakeLists.txt
new file mode 100644
index 0000000..f053704
--- /dev/null
+++ b/src/cxx/libexec/CMakeLists.txt
@@ -0,0 +1,37 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the cxx/libexec subdirectory.
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include(${BCL2FASTQ_CXX_EXECUTABLE_CMAKE})
+
+file (GLOB BCL2FASTQ_PROGRAM_SOURCE_LIST [a-zA-Z0-9]*.cpp)
+
+##
+## Generic rule for all the other programs
+##
+foreach(BCL2FASTQ_PROGRAM_SOURCE ${BCL2FASTQ_PROGRAM_SOURCE_LIST})
+ get_filename_component(BCL2FASTQ_PROGRAM ${BCL2FASTBCL2FASTQE} NAME_WE)
+ add_executable (${BCL2FASTQ_PROGRAM} ${BCL2FASTQ_PROGRAM_SOURCE})
+ target_link_libraries (${BCL2FASTQ_PROGRAM} ${BCL2FASTQ_AVAILABLE_LIBRARIES}
+ ${Boost_LIBRARIES} ${BCL2FASTQ_DEP_LIB}
+ ${BCL2FASTQ_ADDITIONAL_LIB} )
+ install(TARGETS ${BCL2FASTQ_PROGRAM} RUNTIME DESTINATION ${BCL2FASTQ_LIBEXECDIR})
+endforeach(BCL2FASTQ_PROGRAM_SOURCE)
+
+
diff --git a/src/cxx/unittest/CMakeLists.txt b/src/cxx/unittest/CMakeLists.txt
new file mode 100644
index 0000000..322b471
--- /dev/null
+++ b/src/cxx/unittest/CMakeLists.txt
@@ -0,0 +1,24 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the cxx/unittest subfolder
+##
+## author Come Raczy
+##
+################################################################################
+
+
+include_directories(${CPPUNIT_INCLUDE_DIR})
+add_library(bcl2fastq_cppunit cppunitTest.cpp RegistryName.cpp)
+
+
diff --git a/src/cxx/unittest/RegistryName.cpp b/src/cxx/unittest/RegistryName.cpp
new file mode 100644
index 0000000..34eca47
--- /dev/null
+++ b/src/cxx/unittest/RegistryName.cpp
@@ -0,0 +1,63 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file RegistryName.cpp
+ *
+ * \brief Management of the registry names for the cppunit tests..
+ *
+ * \author Come Raczy
+ */
+
+
+#include "RegistryName.hh"
+
+#include <boost/filesystem.hpp>
+#include <vector>
+#include <string>
+#include <fstream>
+
+boost::filesystem::path getFilePath() {return "RegistryNames.txt";}
+
+std::vector<std::string> initializeNameList()
+{
+ std::vector<std::string> nameList;
+ if(boost::filesystem::exists(getFilePath()))
+ {
+ std::ifstream is(getFilePath().string().c_str());
+ std::string name;
+ while (getline(is, name))
+ {
+ if (!name.empty() && nameList.end() == std::find(nameList.begin(), nameList.end(), name))
+ {
+ nameList.push_back(name);
+ }
+ }
+ }
+ return nameList;
+}
+
+const std::vector<std::string> &getRegistryNameList()
+{
+ static const std::vector<std::string> nameList = initializeNameList();
+ return nameList;
+}
+
+std::string registryName(const std::string &name) throw (std::invalid_argument)
+{
+ const std::vector<std::string> nameList = getRegistryNameList();
+ const std::vector<std::string>::const_iterator found = std::find(nameList.begin(), nameList.end(), name);
+ if (found != nameList.end())
+ return name;
+ else
+ throw std::invalid_argument(std::string("Not a registryName: ") + name +
+ std::string(" [check that ") + getFilePath().string() +
+ std::string(" constains '") + name +
+ std::string("']"));
+}
+
+
diff --git a/src/cxx/unittest/RegistryName.hh b/src/cxx/unittest/RegistryName.hh
new file mode 100644
index 0000000..efd49c8
--- /dev/null
+++ b/src/cxx/unittest/RegistryName.hh
@@ -0,0 +1,29 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file RegistryName.hh
+ *
+ * \brief Management of the registry names for the cppunit tests.
+ *
+ * \author Come Raczy
+ */
+
+
+#ifndef BCL2FASTQ_UNIT_TEST_REGISTRY_NAME
+#define BCL2FASTQ_UNIT_TEST_REGISTRY_NAME
+
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+const std::vector<std::string> &getRegistryNameList();
+std::string registryName(const std::string &name) throw (std::invalid_argument);
+
+#endif // #ifndef BCL2FASTQ_UNIT_TEST_REGISTRY_NAME
+
+
diff --git a/src/cxx/unittest/check-source.sh b/src/cxx/unittest/check-source.sh
new file mode 100755
index 0000000..f9aecfa
--- /dev/null
+++ b/src/cxx/unittest/check-source.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2017 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file check-source.sh
+##
+## Basic sanity checks on the source code before running the cppuint tests
+##
+## author Come Raczy
+##
+################################################################################
+
+target=$1
+shift
+good=yes
+for file in $* ; do
+ check=`grep -nH CPPUNIT_TEST_SUITE_NAMED_REGISTRATION $file | grep -v registryName`
+ if [[ $check ]] ; then
+ if [[ $good ]] ; then
+ good=
+ echo >&2
+ echo use of unchecked registry names: >&2
+ fi
+ echo " "$check >&2
+ echo >&2
+ fi
+done
+if [[ $good ]] ; then
+ echo checked > $target
+else
+ exit 1
+fi
+
+
diff --git a/src/cxx/unittest/cppunitTest.cpp b/src/cxx/unittest/cppunitTest.cpp
new file mode 100644
index 0000000..d22cdd2
--- /dev/null
+++ b/src/cxx/unittest/cppunitTest.cpp
@@ -0,0 +1,52 @@
+/**
+ * BCL to FASTQ file converter
+ * Copyright (c) 2007-2017 Illumina, Inc.
+ *
+ * This software is covered by the accompanying EULA
+ * and certain third party copyright/licenses, and any user of this
+ * source file is bound by the terms therein.
+ *
+ * \file cppunitTest.cpp
+ *
+ * \brief Main program used for all the cppunit tests.
+ *
+ * \author Come Raczy
+ */
+
+
+#include <sstream>
+#include <cppunit/CompilerOutputter.h>
+#include <cppunit/extensions/TestFactoryRegistry.h>
+#include <cppunit/ui/text/TestRunner.h>
+
+#include "RegistryName.hh"
+
+int main()
+{
+
+ CppUnit::TextUi::TestRunner runner;
+ // First add the tests from the named registries in the right order
+ // To add/remove/modify a registry name, or to change its sequence
+ // number, edit RegistryName.cpp
+ for (std::vector<std::string>::const_iterator name = getRegistryNameList().begin();
+ getRegistryNameList().end() != name; ++name)
+ {
+ CppUnit::Test *namedSuite = CppUnit::TestFactoryRegistry::getRegistry(*name).makeTest();
+ runner.addTest(namedSuite);
+ }
+
+ // Add the top level (unnamed) suite from the list of tests to run
+ CppUnit::Test *suite = CppUnit::TestFactoryRegistry::getRegistry().makeTest();
+ runner.addTest( suite );
+
+ // Change the default outputter to a compiler error format outputter
+ runner.setOutputter( new CppUnit::CompilerOutputter( &runner.result(),
+ std::cerr ) );
+ // Run the tests.
+ bool wasSucessful = runner.run();
+
+ // Return error code 1 if the one of test failed.
+ return wasSucessful ? 0 : 1;
+}
+
+
diff --git a/src/xsl/CMakeLists.txt b/src/xsl/CMakeLists.txt
new file mode 100644
index 0000000..7073e86
--- /dev/null
+++ b/src/xsl/CMakeLists.txt
@@ -0,0 +1,26 @@
+################################################################################
+##
+## BCL to FASTQ file converter
+## Copyright (c) 2007-2015 Illumina, Inc.
+##
+## This software is covered by the accompanying EULA
+## and certain third party copyright/licenses, and any user of this
+## source file is bound by the terms therein.
+##
+################################################################################
+##
+## file CMakeLists.txt
+##
+## Configuration file for the xsl subfolder
+##
+## author Mauricio Varea
+##
+################################################################################
+include ("${BCL2FASTQ_GLOBALS_CMAKE}")
+
+install(CODE "
+ include (\"${BCL2FASTQ_MACROS_CMAKE}\")
+ configure_files_recursively (\"${CMAKE_CURRENT_SOURCE_DIR}\" \"${CMAKE_CURRENT_BINARY_DIR}\" \"*.xsl\")
+ install_files_recursively (\"${CMAKE_CURRENT_BINARY_DIR}\" \"${BCL2FASTQ_ORIG_DATADIR}/xsl\" \"*.xsl\" \"\${BCL2FASTQ_LIBRARY_PERMISSIONS}\")
+ ")
+
diff --git a/src/xsl/common/Utils.xsl b/src/xsl/common/Utils.xsl
new file mode 100644
index 0000000..40eb8ad
--- /dev/null
+++ b/src/xsl/common/Utils.xsl
@@ -0,0 +1,163 @@
+<?xml version="1.0"?>
+<!--
+/**
+ ** Isaac Genome Alignment Software
+ ** Copyright (c) 2010-2012 Illumina, Inc.
+ **
+ ** This software is provided under the terms and conditions of the
+ ** Illumina Open Source Software License 1.
+ **
+ ** You should have received a copy of the Illumina Open Source
+ ** Software License 1 along with this program. If not, see
+ ** <https://github.com/sequencing/licenses/>.
+ **
+ ** The distribution includes the code libraries listed below in the
+ ** 'redist' sub-directory. These are distributed according to the
+ ** licensing terms governing each library.
+ **
+ ** \file Utils.xsl
+ **
+ ** \author Roman Petrovski
+ **/
+-->
+<xsl:stylesheet version="1.0"
+xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+xmlns:math="http://exslt.org/math"
+xmlns:ckbk="http://www.oreily.com/xsltckbk"
+>
+
+<xsl:template name="generateSequence">
+ <xsl:param name="var"/>
+ <xsl:param name="stop"/>
+ <xsl:param name="step"/>
+
+ <!--xsl:message><xsl:value-of select="concat($var, '!=', $stop, ':', $step)"/></xsl:message-->
+
+ <xsl:choose>
+ <xsl:when test="$var != $stop">
+ <xsl:value-of select="concat($var, ' ')"/>
+ <xsl:call-template name="generateSequence">
+ <xsl:with-param name="var" select="number($var) + number($step)"/>
+ <xsl:with-param name="stop" select="$stop"/>
+ <xsl:with-param name="step" select="$step"/>
+ </xsl:call-template>
+ </xsl:when>
+ <xsl:otherwise></xsl:otherwise>
+ </xsl:choose>
+
+</xsl:template>
+
+<xsl:template name="ckbk:variance">
+ <xsl:param name="nodes" select="/.."/>
+ <xsl:param name="sum" select="0"/>
+ <xsl:param name="sum-sq" select="0"/>
+ <xsl:param name="count" select="0"/>
+ <xsl:choose>
+ <xsl:when test="not($nodes)">
+ <xsl:value-of select="($sum-sq - ($sum * $sum) div $count) div ($count - 1)"/>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:variable name="value" select="$nodes[1]"/>
+ <xsl:call-template name="ckbk:variance">
+ <xsl:with-param name="nodes" select="$nodes[position( ) != 1]"/>
+ <xsl:with-param name="sum" select="$sum + $value"/>
+ <xsl:with-param name="sum-sq" select="$sum-sq + ($value * $value)"/>
+ <xsl:with-param name="count" select="$count + 1"/>
+ </xsl:call-template>
+ </xsl:otherwise>
+ </xsl:choose>
+</xsl:template>
+
+
+<xsl:template name="mean">
+ <xsl:param name="nodes" select="/.."/>
+ <xsl:param name="sum" select="0"/>
+ <xsl:param name="count" select="0"/>
+ <xsl:choose>
+ <xsl:when test="not($nodes)">
+ <xsl:value-of select="$sum div $count"/>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:variable name="value" select="$nodes[1]"/>
+ <xsl:call-template name="mean">
+ <xsl:with-param name="nodes" select="$nodes[position( ) != 1]"/>
+ <xsl:with-param name="sum" select="$sum + $value"/>
+ <xsl:with-param name="count" select="$count + 1"/>
+ </xsl:call-template>
+ </xsl:otherwise>
+ </xsl:choose>
+</xsl:template>
+
+
+<xsl:template name="stdev">
+ <xsl:param name="nodes"/>
+ <xsl:param name="mean" select="0"/>
+ <xsl:param name="sum" select="0"/>
+ <xsl:param name="sum-sq" select="0"/>
+ <xsl:param name="count" select="0"/>
+ <!--xsl:message>tada <xsl:for-each select="$nodes"><xsl:value-of select="."/></xsl:for-each></xsl:message-->
+ <xsl:choose>
+ <xsl:when test="not($nodes)">
+ <xsl:choose>
+ <xsl:when test="3 > $count">
+ <xsl:value-of select="''"/> <!-- don't report standard deviation for sets where it makes no sense-->
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:value-of select="math:sqrt($sum-sq div $count)"/>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:variable name="value" select="$nodes[1]"/>
+ <xsl:call-template name="stdev">
+ <xsl:with-param name="nodes" select="$nodes[position( ) != 1]"/>
+ <xsl:with-param name="mean" select="$mean"/>
+ <xsl:with-param name="sum" select="$sum + $value"/>
+ <xsl:with-param name="sum-sq" select="$sum-sq + (($mean - $value) * ($mean - $value))"/>
+ <xsl:with-param name="count" select="$count + 1"/>
+ </xsl:call-template>
+ </xsl:otherwise>
+ </xsl:choose>
+</xsl:template>
+
+
+<xsl:template name="meanAndStdev">
+ <xsl:param name="nodes" select="/.."/>
+ <xsl:param name="meanFmt" select="'0.00'"/>
+ <xsl:param name="devFmt" select="'0.00'"/>
+ <xsl:choose>
+ <xsl:when test="not($nodes)">
+ <xsl:value-of select="0"/>
+ </xsl:when>
+ <xsl:otherwise>
+
+ <xsl:variable name="m">
+ <xsl:call-template name="mean">
+ <xsl:with-param name="nodes" select="$nodes"/>
+ </xsl:call-template>
+ </xsl:variable>
+
+ <xsl:variable name="std">
+ <xsl:call-template name="stdev">
+ <xsl:with-param name="nodes" select="$nodes"/>
+ <xsl:with-param name="mean" select="$m"/>
+ </xsl:call-template>
+ </xsl:variable>
+
+ <xsl:variable name="roundedMean" select="format-number($m, $meanFmt)"/>
+ <xsl:variable name="roundedStdev" select="format-number($std, $devFmt)"/>
+ <xsl:choose>
+ <xsl:when test="string($std) = ''">
+ <xsl:value-of select="$roundedMean"/>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:value-of select="concat($roundedMean, ' +/-', $roundedStdev)"/>
+ </xsl:otherwise>
+ </xsl:choose>
+
+ </xsl:otherwise>
+ </xsl:choose>
+</xsl:template>
+
+
+</xsl:stylesheet>
diff --git a/src/xsl/demux/FlowcellSummaryPage.xsl b/src/xsl/demux/FlowcellSummaryPage.xsl
new file mode 100644
index 0000000..9507615
--- /dev/null
+++ b/src/xsl/demux/FlowcellSummaryPage.xsl
@@ -0,0 +1,115 @@
+<?xml version="1.0"?>
+<!--
+/**
+ ** BCL to FASTQ file converter
+ ** Copyright (c) 2007-2015 Illumina, Inc.
+ **
+ ** This software is covered by the accompanying EULA
+ ** and certain third party copyright/licenses, and any user of this
+ ** source file is bound by the terms therein.
+ **
+ ** \file FlowcellSummaryPage.xsl
+ **
+ ** \author Roman Petrovski
+ ** \author Mauricio Varea
+ **/
+-->
+<xsl:stylesheet version="1.0"
+xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+xmlns:bcl2fastq="http://www.illumina.com/bcl2fastq"
+xmlns:str="http://exslt.org/strings"
+xmlns:math="http://exslt.org/math"
+xmlns:exsl="http://exslt.org/common"
+extension-element-prefixes="exsl"
+exclude-result-prefixes="str math"
+>
+
+
+<xsl:template name="generateFlowcellYieldSummaryTable">
+ <xsl:param name="projectNode" select="../.."/>
+ <xsl:param name="sampleId" select="../@name"/>
+ <xsl:param name="barcodeId" select="@name"/>
+
+
+ <xsl:variable name="clustersPF" select="sum($projectNode/Sample[@name=$sampleId]/Barcode[@name=$barcodeId]/Lane/Tile/Pf/ClusterCount)"/>
+ <xsl:variable name="clustersRaw" select="sum($projectNode/Sample[@name=$sampleId]/Barcode[@name=$barcodeId]/Lane/Tile/Raw/ClusterCount)"/>
+
+ <xsl:variable name="flowcellYieldPf" select="sum($projectNode/Sample[@name=$sampleId]/Barcode[@name=$barcodeId]/Lane/Tile/Pf/Read/Yield)"/>
+
+ <table border="1" ID="ReportTable">
+ <tr><th>Clusters (Raw)</th><th>Clusters(PF)</th><th>Yield (MBases)</th></tr>
+ <tr>
+ <td><xsl:value-of select="format-number($clustersRaw, '###,###,###,###,###')"/></td>
+ <td><xsl:value-of select="format-number($clustersPF, '###,###,###,###,###')"/></td>
+ <td><xsl:value-of select="format-number(round($flowcellYieldPf) div 1000000, '###,###,###,###,###')"/></td>
+ </tr>
+ </table>
+</xsl:template>
+
+<xsl:template name="generateFlowcellSummaryTables">
+ <xsl:param name="flowcellNode" select="../../.."/>
+ <xsl:param name="projectId" select="../../@name"/>
+ <xsl:param name="sampleId" select="../@name"/>
+ <xsl:param name="barcodeId" select="@name"/>
+
+ <xsl:variable name="conversionStatsFlowcellNode"
+ select="/Stats/Flowcell[@flowcell-id=$flowcellNode/@flowcell-id]"/>
+
+ <xsl:call-template name="generateFlowcellYieldSummaryTable">
+ <xsl:with-param name="projectNode" select="$conversionStatsFlowcellNode/Project[@name=$projectId]"/>
+ </xsl:call-template>
+
+</xsl:template>
+
+<xsl:template name="generateFlowcellSummaryPage">
+ <xsl:param name="flowcellNode" select="../../.."/>
+ <xsl:param name="projectId" select="../../@name"/>
+ <xsl:param name="sampleId" select="../@name"/>
+ <xsl:param name="barcodeId" select="@name"/>
+ <xsl:param name="showBarcodes"/>
+ <xsl:param name="alternativeViewRef"/>
+ <xsl:param name="alternativeViewText"/>
+<html>
+<link rel="stylesheet" href="../../../../{$CSS_FILE_NAME}" type="text/css"/>
+<body>
+
+ <table width="100%">
+ <tr>
+ <td>
+ <p><xsl:call-template name="getBarcodeDisplayPath"/></p>
+ </td>
+ <td>
+ <xsl:if test="../../../@flowcell-id!='all' and $barcodeId='all'">
+ <p align="right">
+ <xsl:element name="a">
+ <xsl:attribute name="href"><xsl:value-of select="$alternativeViewRef"/></xsl:attribute>
+ <xsl:value-of select="$alternativeViewText"/>
+ </xsl:element>
+ </p>
+ </xsl:if>
+ </td>
+ </tr>
+ </table>
+
+<h2>Flowcell Summary</h2>
+ <xsl:call-template name="generateFlowcellSummaryTables"/>
+
+<h2>Lane Summary</h2>
+ <xsl:call-template name="generateExpandedLaneResultsSummaryTable">
+ <xsl:with-param name="showBarcodes" select="$showBarcodes"/>
+ </xsl:call-template>
+
+ <xsl:if test="'unknown'=$barcodeId or 'all'=$projectId and 'all'=$sampleId and 'all'=$barcodeId and $showBarcodes">
+<h2>Top Unknown Barcodes</h2>
+ <xsl:call-template name="generateUnknownBarcodesTable"/>
+ </xsl:if>
+
+<p>@BCL2FASTQ_VERSION_FULL@</p>
+
+
+</body>
+</html>
+
+</xsl:template>
+
+</xsl:stylesheet>
diff --git a/src/xsl/demux/GenerateReport.xsl b/src/xsl/demux/GenerateReport.xsl
new file mode 100644
index 0000000..23617c9
--- /dev/null
+++ b/src/xsl/demux/GenerateReport.xsl
@@ -0,0 +1,200 @@
+<?xml version="1.0"?>
+<!--
+/**
+ ** BCL to FASTQ file converter
+ ** Copyright (c) 2007-2015 Illumina, Inc.
+ **
+ ** This software is covered by the accompanying EULA
+ ** and certain third party copyright/licenses, and any user of this
+ ** source file is bound by the terms therein.
+ **
+ ** \file GenerateReport.xsl
+ **
+ ** \author Roman Petrovski
+ ** \author Mauricio Varea
+ **/
+-->
+<xsl:stylesheet version="1.0"
+xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+xmlns:bcl2fastq="http://www.illumina.com/bcl2fastq"
+xmlns:str="http://exslt.org/strings"
+xmlns:math="http://exslt.org/math"
+xmlns:exsl="http://exslt.org/common"
+extension-element-prefixes="exsl"
+exclude-result-prefixes="str math"
+>
+
+<xsl:variable name="DEMULTIPLEXING_STATS_XML" select="document($DEMULTIPLEXING_STATS_XML_PARAM)"/>
+
+<xsl:include href="../common/Utils.xsl"/>
+<xsl:include href="NameUtils.xsl"/>
+<xsl:include href="PathUtils.xsl"/>
+<xsl:include href="LaneSummary.xsl"/>
+<xsl:include href="FlowcellSummaryPage.xsl"/>
+
+<xsl:variable name="homeFilePath" select="concat($OUTPUT_DIRECTORY_HTML_PARAM, '/index.html')"/>
+
+<xsl:variable name="CSS_FILE_NAME" select="'Report.css'"/>
+<xsl:variable name="outputCssFilePath" select="concat($OUTPUT_DIRECTORY_HTML_PARAM, '/', $CSS_FILE_NAME)"/>
+<xsl:variable name="inputCssFilePath" select="concat($BCL2FASTQ_FULL_DATADIR_PARAM, '/css/Report.css.xml')"/>
+
+
+<xsl:template match="/">
+
+ <exsl:document href="{$outputCssFilePath}" method="text">
+ <xsl:value-of select="document($inputCssFilePath)/css"/>
+ </exsl:document>
+
+ <exsl:document href="{$homeFilePath}" method="html" version="4.0" indent="yes">
+<html>
+ <frameset cols="15%, 85%">
+ <frame>
+ <xsl:attribute name="src"><xsl:call-template name="getFlowcellRefsLocalPath"/></xsl:attribute>
+ </frame>
+ <frame name="flowcellsummaryframe">
+ <xsl:attribute name="src">
+ <xsl:call-template name="getFlowcellSampleBarcodeSimpleLocalPath">
+ <xsl:with-param name="flowcellId">
+ <xsl:value-of select="/Stats/Flowcell/@flowcell-id"/>
+ </xsl:with-param>
+ <xsl:with-param name="projectId" select="'all'"/>
+ <xsl:with-param name="sampleId" select="'all'"/>
+ <xsl:with-param name="barcodeId" select="'all'"/>
+ </xsl:call-template>
+ </xsl:attribute>
+ </frame>
+ </frameset>
+<body>
+<p>@BCL2FASTQ_VERSION_FULL@</p>
+</body>
+</html>
+ </exsl:document>
+
+ <xsl:apply-templates/>
+</xsl:template>
+
+<xsl:template match="/Stats">
+ <xsl:variable name="flowcellsPageFilePath"><xsl:call-template name="getFlowcellRefsGlobalPath"/></xsl:variable>
+ <exsl:document href="{$flowcellsPageFilePath}" method="html" version="4.0" indent="yes">
+
+<link rel="stylesheet" href="{$CSS_FILE_NAME}" type="text/css"/>
+
+<table>
+ <xsl:for-each select="Flowcell">
+ <xsl:variable name="flowcellDisplayName"><xsl:call-template name="getNodeDisplayName"/></xsl:variable>
+
+ <tr><td colspan="4">
+ <xsl:for-each select="Project[@name='all']/Sample[@name='all']/Barcode[@name='all']">
+ <xsl:element name="a">
+ <xsl:attribute name="href"><xsl:call-template name="getFlowcellSampleBarcodeSimpleLocalPath"/></xsl:attribute>
+ <xsl:attribute name="target">flowcellsummaryframe</xsl:attribute>
+ <xsl:value-of select="$flowcellDisplayName"/>
+ </xsl:element>
+ </xsl:for-each>
+ </td></tr>
+ <xsl:for-each select="Project[@name != 'all']">
+ <xsl:variable name="projectDisplayName"><xsl:call-template name="getNodeDisplayName"/></xsl:variable>
+
+ <tr><td/><td colspan="3">
+ <xsl:for-each select="Sample[@name = 'all']/Barcode[@name='all']">
+ <xsl:element name="a">
+ <xsl:attribute name="href"><xsl:call-template name="getFlowcellSampleBarcodeSimpleLocalPath"/></xsl:attribute>
+ <xsl:attribute name="target">flowcellsummaryframe</xsl:attribute>
+ <xsl:value-of select="$projectDisplayName"/>
+ </xsl:element>
+ </xsl:for-each>
+ </td></tr>
+
+ <xsl:for-each select="Sample[@name != 'all']">
+
+ <xsl:variable name="sampleDisplayName"><xsl:call-template name="getNodeDisplayName"/></xsl:variable>
+
+ <xsl:variable name="items">
+ <xsl:for-each select="Barcode[@name != 'all']">
+ <xsl:variable name="displayName"><xsl:call-template name="getNodeDisplayName"/></xsl:variable>
+
+ <xsl:variable name="sortName">
+ <xsl:choose>
+ <xsl:when test="@name='unknown'">1</xsl:when>
+ <xsl:otherwise><xsl:value-of select="concat('2', @name)"/></xsl:otherwise>
+ </xsl:choose>
+ </xsl:variable>
+
+ <tr>
+ <xsl:attribute name="sortname"><xsl:value-of select="$sortName"/></xsl:attribute>
+ <td/><td/><td/><td>
+ <xsl:element name="a">
+ <xsl:attribute name="href"><xsl:call-template name="getFlowcellSampleBarcodeSimpleLocalPath"/></xsl:attribute>
+ <xsl:attribute name="target">flowcellsummaryframe</xsl:attribute>
+ <xsl:value-of select="$displayName"/>
+ </xsl:element>
+ </td>
+ </tr>
+ </xsl:for-each>
+ </xsl:variable>
+ <tr><td/><td/><td colspan="2">
+ <xsl:for-each select="Barcode[@name = 'all']">
+ <xsl:element name="a">
+ <xsl:attribute name="href"><xsl:call-template name="getFlowcellSampleBarcodeSimpleLocalPath"/></xsl:attribute>
+ <xsl:attribute name="target">flowcellsummaryframe</xsl:attribute>
+ <xsl:value-of select="$sampleDisplayName"/>
+ </xsl:element>
+ </xsl:for-each>
+ </td></tr>
+ <xsl:for-each select="exsl:node-set($items)/tr">
+ <xsl:sort select="@sortname"/>
+ <xsl:copy-of select="."/>
+ </xsl:for-each>
+ </xsl:for-each>
+ </xsl:for-each>
+ </xsl:for-each>
+</table>
+ </exsl:document>
+ <xsl:apply-templates/>
+</xsl:template>
+
+<xsl:template match="/Stats/Flowcell/Project/Sample/Barcode">
+ <xsl:variable name="flowcellSummaryPageFilePath"><xsl:call-template name="getFlowcellSampleBarcodeGlobalPath"/></xsl:variable>
+ <xsl:variable name="flowcellSimpleSummaryPageFilePath"><xsl:call-template name="getFlowcellSampleBarcodeSimpleGlobalPath"/></xsl:variable>
+
+ <exsl:document href="{$flowcellSummaryPageFilePath}" method="html" version="4.0" indent="yes">
+ <xsl:call-template name="generateFlowcellSummaryPage">
+ <xsl:with-param name="showBarcodes" select="../../../@flowcell-id!='all' and @name='all'"/>
+ <xsl:with-param name="alternativeViewRef">../../../../<xsl:call-template name="getFlowcellSampleBarcodeSimpleLocalPath"/></xsl:with-param>
+ <xsl:with-param name="alternativeViewText" select="'hide barcodes'"/>
+ </xsl:call-template>
+ </exsl:document>
+
+ <exsl:document href="{$flowcellSimpleSummaryPageFilePath}" method="html" version="4.0" indent="yes">
+ <xsl:call-template name="generateFlowcellSummaryPage">
+ <xsl:with-param name="showBarcodes" select="false()"/>
+ <xsl:with-param name="alternativeViewRef">../../../../<xsl:call-template name="getFlowcellSampleBarcodeLocalPath"/></xsl:with-param>
+ <xsl:with-param name="alternativeViewText" select="'show barcodes'"/>
+ </xsl:call-template>
+ </exsl:document>
+
+</xsl:template>
+
+<xsl:template match="/Stats/Flowcell/Project[@name='all']/Sample[@name='all']/Barcode[@name='all']">
+ <xsl:variable name="flowcellSummaryPageFilePath"><xsl:call-template name="getFlowcellSampleBarcodeGlobalPath"/></xsl:variable>
+ <xsl:variable name="flowcellSimpleSummaryPageFilePath"><xsl:call-template name="getFlowcellSampleBarcodeSimpleGlobalPath"/></xsl:variable>
+
+ <exsl:document href="{$flowcellSummaryPageFilePath}" method="html" version="4.0" indent="yes">
+ <xsl:call-template name="generateFlowcellSummaryPage">
+ <xsl:with-param name="showBarcodes" select="true()"/>
+ <xsl:with-param name="alternativeViewRef">../../../../<xsl:call-template name="getFlowcellSampleBarcodeSimpleLocalPath"/></xsl:with-param>
+ <xsl:with-param name="alternativeViewText" select="'hide barcodes'"/>
+ </xsl:call-template>
+ </exsl:document>
+
+ <exsl:document href="{$flowcellSimpleSummaryPageFilePath}" method="html" version="4.0" indent="yes">
+ <xsl:call-template name="generateFlowcellSummaryPage">
+ <xsl:with-param name="showBarcodes" select="false()"/>
+ <xsl:with-param name="alternativeViewRef">../../../../<xsl:call-template name="getFlowcellSampleBarcodeLocalPath"/></xsl:with-param>
+ <xsl:with-param name="alternativeViewText" select="'show barcodes'"/>
+ </xsl:call-template>
+ </exsl:document>
+
+</xsl:template>
+
+</xsl:stylesheet>
diff --git a/src/xsl/demux/LaneSummary.xsl b/src/xsl/demux/LaneSummary.xsl
new file mode 100644
index 0000000..0c83294
--- /dev/null
+++ b/src/xsl/demux/LaneSummary.xsl
@@ -0,0 +1,238 @@
+<?xml version="1.0"?>
+<!--
+/**
+ ** BCL to FASTQ file converter
+ ** Copyright (c) 2007-2015 Illumina, Inc.
+ **
+ ** This software is covered by the accompanying EULA
+ ** and certain third party copyright/licenses, and any user of this
+ ** source file is bound by the terms therein.
+ **
+ ** \file LaneSummary.xsl
+ **
+ ** \author Roman Petrovski
+ ** \author Mauricio Varea
+ **/
+-->
+<xsl:stylesheet version="1.0"
+xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+xmlns:str="http://exslt.org/strings"
+xmlns:math="http://exslt.org/math"
+xmlns:bcl2fastq="http://www.illumina.com/bcl2fastq"
+>
+
+<xsl:template name="generateConversionStatsCells">
+ <xsl:param name="flowcellNode" select="../../../.."/>
+ <xsl:param name="projectId" select="../../../@name"/>
+ <xsl:param name="sampleId" select="../../@name"/>
+ <xsl:param name="barcodeId" select="../@name"/>
+ <xsl:variable name="laneNumber" select="@number"/>
+
+ <xsl:variable name="conversionStatsFlowcellNode"
+ select="/Stats/Flowcell[@flowcell-id=$flowcellNode/@flowcell-id]"/>
+
+ <xsl:variable name="conversionStatsBarcodeLaneNode" select="$conversionStatsFlowcellNode/Project[@name=$projectId]/Sample[@name=$sampleId]/Barcode[@name=$barcodeId]/Lane[@number=$laneNumber]"/>
+
+ <xsl:variable name="conversionStatsLaneNode" select="$conversionStatsFlowcellNode/Project[@name='all']/Sample[@name='all']/Barcode[@name='all']/Lane[@number=$laneNumber]"/>
+
+ <xsl:variable name="clustersRaw" select="sum($conversionStatsBarcodeLaneNode/Tile/Raw/ClusterCount)"/>
+ <xsl:variable name="clustersPF" select="sum($conversionStatsBarcodeLaneNode/Tile/Pf/ClusterCount)"/>
+
+ <xsl:variable name="yieldPF" select="sum($conversionStatsBarcodeLaneNode/Tile/Pf/Read/Yield)"/>
+
+ <xsl:variable name="yieldPFQ30" select="sum($conversionStatsBarcodeLaneNode/Tile/Pf/Read/YieldQ30)"/>
+
+ <xsl:variable name="qualityScoreSumPF" select="sum($conversionStatsBarcodeLaneNode/Tile/Pf/Read/QualityScoreSum)"/>
+
+ <td><xsl:value-of select="format-number(round($yieldPF div 1000000), '###,###,###,###,###')"/></td>
+ <td><xsl:if test="0 != $clustersRaw"><xsl:value-of select="format-number($clustersPF div $clustersRaw * 100, '0.00')"/></xsl:if></td>
+ <td><xsl:if test="0 != $yieldPF"><xsl:value-of select="format-number($yieldPFQ30 div $yieldPF * 100, '0.00')"/></xsl:if></td>
+ <td><xsl:if test="0 != $yieldPF"><xsl:value-of select="format-number($qualityScoreSumPF div $yieldPF, '0.00')"/></xsl:if></td>
+
+</xsl:template>
+
+<xsl:template name="generateDemultiplexingStatsCells">
+ <xsl:param name="flowcellNode" select="../../../.."/>
+ <xsl:param name="projectId" select="../../../@name"/>
+ <xsl:param name="sampleId" select="../../@name"/>
+ <xsl:param name="barcodeId" select="../@name"/>
+ <xsl:variable name="laneNumber" select="@number"/>
+
+ <xsl:variable name="conversionStatsFlowcellNode"
+ select="/Stats/Flowcell[@flowcell-id=$flowcellNode/@flowcell-id]"/>
+ <xsl:variable name="conversionStatsBarcodeLaneNode" select="$conversionStatsFlowcellNode/Project[@name=$projectId]/Sample[@name=$sampleId]/Barcode[@name=$barcodeId]/Lane[@number=$laneNumber]"/>
+
+ <xsl:variable name="demuxStatsFlowcellNode"
+ select="$DEMULTIPLEXING_STATS_XML/Stats/Flowcell[@flowcell-id=$flowcellNode/@flowcell-id]"/>
+
+ <xsl:variable name="demuxStatsBarcodeLaneNode" select="$demuxStatsFlowcellNode/
+Project[@name=$projectId]/Sample[@name=$sampleId]/Barcode[@name=$barcodeId]/Lane[@number=$laneNumber]"/>
+
+ <xsl:variable name="demuxStatsLaneNode" select="$demuxStatsFlowcellNode/
+Project[@name='all']/Sample[@name='all']/Barcode[@name='all']/Lane[@number=$laneNumber]"/>
+
+ <xsl:variable name="clustersPF">
+ <xsl:choose>
+ <xsl:when test="0 != $demuxStatsBarcodeLaneNode/BarcodeCount"><xsl:value-of select="$demuxStatsBarcodeLaneNode/BarcodeCount"/></xsl:when>
+ <xsl:otherwise><xsl:value-of select="sum($conversionStatsBarcodeLaneNode/Tile/PF/ClusterCount)"/></xsl:otherwise>
+ </xsl:choose>
+ </xsl:variable>
+ <xsl:variable name="laneClustersPF">
+ <xsl:choose>
+ <xsl:when test="0 != $demuxStatsLaneNode/BarcodeCount"><xsl:value-of select="$demuxStatsLaneNode/BarcodeCount"/></xsl:when>
+ <xsl:otherwise><xsl:value-of select="$clustersPF"/></xsl:otherwise>
+ </xsl:choose>
+ </xsl:variable>
+ <xsl:variable name="perfectBarcodePF" select="$demuxStatsBarcodeLaneNode/PerfectBarcodeCount"/>
+ <xsl:variable name="oneMismatchBarcodePF" select="$demuxStatsBarcodeLaneNode/OneMismatchBarcodeCount"/>
+
+ <td><xsl:value-of select="format-number($clustersPF, '###,###,###,###,###')"/></td>
+ <td><xsl:if test="0 != $laneClustersPF"><xsl:value-of select="format-number($clustersPF div $laneClustersPF * 100, '0.00')"/></xsl:if></td>
+ <xsl:element name="td">
+ <xsl:choose>
+ <xsl:when test="$sampleId='unknown'">N/A</xsl:when>
+ <xsl:otherwise>
+ <xsl:if test="0 != $clustersPF">
+ <xsl:value-of select="format-number($perfectBarcodePF div $clustersPF * 100, '0.00')"/>
+ </xsl:if>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:element>
+ <xsl:element name="td">
+ <xsl:choose>
+ <xsl:when test="$sampleId='unknown'">N/A</xsl:when>
+ <xsl:otherwise>
+ <xsl:if test="0 != $clustersPF">
+ <xsl:value-of select="format-number($oneMismatchBarcodePF div $clustersPF * 100, '0.00')"/>
+ </xsl:if>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:element>
+
+</xsl:template>
+
+<xsl:template name="generateExpandedLaneResultsSummaryTable">
+ <xsl:param name="flowcellNode" select="../../.."/>
+ <xsl:param name="projectId" select="../../@name"/>
+ <xsl:param name="sampleId" select="../@name"/>
+ <xsl:param name="barcodeId" select="@name"/>
+ <xsl:param name="showBarcodes"/>
+
+ <table border="1" ID="ReportTable">
+
+ <tr>
+ <th>Lane</th>
+ <xsl:if test="$showBarcodes and $projectId='all'">
+ <th>Project</th>
+ </xsl:if>
+ <xsl:if test="$showBarcodes and $sampleId='all'">
+ <th>Sample</th>
+ </xsl:if>
+ <xsl:if test="$showBarcodes and $barcodeId='all'">
+ <th>Barcode sequence</th>
+ </xsl:if>
+
+ <th>PF Clusters</th>
+ <th>% of the<br/>lane</th>
+ <th>% Perfect<br/>barcode</th>
+ <th>% One mismatch<br/>barcode</th>
+
+ <th>Yield (Mbases)</th>
+ <th>% PF<br/>Clusters</th>
+ <th>% >= Q30<br/>bases</th>
+ <th>Mean Quality<br/>Score</th>
+ </tr>
+
+ <xsl:for-each select="$flowcellNode/
+Project[(true()=$showBarcodes and (($projectId='all' and @name!='all') or ($projectId!='all' and @name=$projectId))) or
+(false()=$showBarcodes and @name=$projectId)]/
+Sample[(true()=$showBarcodes and (($sampleId='all' and @name!='all') or ($sampleId!='all' and @name=$sampleId))) or
+(false()=$showBarcodes and @name=$sampleId)]/
+Barcode[(true()=$showBarcodes and (($barcodeId='all' and @name!='all') or ($barcodeId!='all' and @name=$barcodeId))) or
+(false()=$showBarcodes and @name=$barcodeId)]/
+Lane">
+ <xsl:sort select="@number"/>
+ <xsl:sort select="../../../@name"/>
+ <xsl:sort select="../../@name"/>
+ <xsl:sort select="../@name"/>
+
+ <tr>
+ <td><xsl:value-of select="@number"/></td>
+ <xsl:if test="$showBarcodes and $projectId='all'">
+ <td><xsl:value-of select="../../../@name"/></td>
+ </xsl:if>
+ <xsl:if test="$showBarcodes and $sampleId='all'">
+ <!--td><xsl:value-of select="../../@name"/></td-->
+ <xsl:element name="td">
+ <xsl:if test="../../@name='unknown'">
+ <xsl:attribute name="style">font-style:italic</xsl:attribute>
+ </xsl:if>
+ <xsl:value-of select="../../@name"/>
+ </xsl:element>
+ </xsl:if>
+ <xsl:if test="$showBarcodes and $barcodeId='all'">
+ <xsl:element name="td">
+ <xsl:if test="../@name='unknown'">
+ <xsl:attribute name="style">font-style:italic</xsl:attribute>
+ </xsl:if>
+ <xsl:value-of select="../@name"/>
+ </xsl:element>
+ </xsl:if>
+ <xsl:call-template name="generateDemultiplexingStatsCells"/>
+ <xsl:call-template name="generateConversionStatsCells">
+ </xsl:call-template>
+ </tr>
+ </xsl:for-each>
+
+ </table>
+
+</xsl:template>
+
+
+<xsl:template name="generateUnknownBarcodesTable">
+ <xsl:param name="flowcellNode" select="../../.."/>
+
+<table border="1" ID="ReportTable">
+
+<tr>
+ <xsl:for-each select="/Stats/Flowcell[@flowcell-id=$flowcellNode/@flowcell-id]/Lane">
+ <xsl:element name="th">
+ <xsl:if test="position()!=1">
+ <xsl:attribute name="style">border-left:double</xsl:attribute>
+ </xsl:if>
+ Lane
+ </xsl:element>
+ <th>Count</th>
+ <th>Sequence</th>
+ </xsl:for-each>
+</tr>
+<tr>
+ <xsl:for-each select="/Stats/Flowcell[@flowcell-id=$flowcellNode/@flowcell-id]/Lane">
+ <xsl:element name="th">
+ <xsl:attribute name="rowspan"><xsl:value-of select="1+count(TopUnknownBarcodes/Barcode)"/></xsl:attribute>
+ <xsl:if test="position()!=1">
+ <xsl:attribute name="style">border-left:double</xsl:attribute>
+ </xsl:if>
+ <xsl:value-of select="@number"/>
+ </xsl:element>
+ <td><xsl:value-of select="format-number(TopUnknownBarcodes/Barcode[position()=1]/@count,'###,###,###,###,###')"/></td>
+ <td><xsl:value-of select="TopUnknownBarcodes/Barcode[position()=1]/@sequence"/></td>
+ </xsl:for-each>
+</tr>
+ <xsl:for-each select="/Stats/Flowcell[@flowcell-id=$flowcellNode/@flowcell-id]/Lane[position()=1]/TopUnknownBarcodes/Barcode">
+ <xsl:variable name="position" select="position()"/>
+<tr>
+ <xsl:if test="position()!=1">
+ <xsl:for-each select="/Stats/Flowcell[@flowcell-id=$flowcellNode/@flowcell-id]/Lane">
+ <td><xsl:value-of select="format-number(TopUnknownBarcodes/Barcode[position()=$position]/@count,'###,###,###,###,###')"/></td>
+ <td><xsl:value-of select="TopUnknownBarcodes/Barcode[position()=$position]/@sequence"/></td>
+ </xsl:for-each>
+ </xsl:if>
+</tr>
+ </xsl:for-each>
+
+</table>
+
+</xsl:template>
+
+</xsl:stylesheet>
diff --git a/src/xsl/demux/NameUtils.xsl b/src/xsl/demux/NameUtils.xsl
new file mode 100644
index 0000000..3340ee3
--- /dev/null
+++ b/src/xsl/demux/NameUtils.xsl
@@ -0,0 +1,70 @@
+<?xml version="1.0"?>
+<!--
+/**
+ ** BCL to FASTQ file converter
+ ** Copyright (c) 2007-2015 Illumina, Inc.
+ **
+ ** This software is covered by the accompanying EULA
+ ** and certain third party copyright/licenses, and any user of this
+ ** source file is bound by the terms therein.
+ **
+ ** \file NameUtils.xsl
+ **
+ ** \author Roman Petrovski
+ ** \author Mauricio Varea
+ **/
+-->
+<xsl:stylesheet version="1.0"
+xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+xmlns:str="http://exslt.org/strings"
+xmlns:math="http://exslt.org/math"
+xmlns:bcl2fastq="http://www.illumina.com/bcl2fastq"
+>
+<xsl:template name="getNodeDisplayName">
+ <xsl:param name="node" select="."/>
+
+ <xsl:choose>
+ <xsl:when test="name()='Flowcell'">
+ <xsl:choose>
+ <xsl:when test="@flowcell-id=''">[unknown flowcell]</xsl:when>
+ <xsl:otherwise><xsl:value-of select="@flowcell-id"/></xsl:otherwise>
+ </xsl:choose>
+ </xsl:when>
+ <xsl:when test="name()='Project'">
+ <xsl:choose>
+ <xsl:when test="@name='all'">[all projects]</xsl:when>
+ <xsl:when test="@name='default'">[default project]</xsl:when>
+ <xsl:otherwise><xsl:value-of select="@name"/></xsl:otherwise>
+ </xsl:choose>
+ </xsl:when>
+ <xsl:when test="name()='Sample'">
+ <xsl:choose>
+ <xsl:when test="@name='all'">[all samples]</xsl:when>
+ <xsl:when test="@name='unknown'">[unknown sample]</xsl:when>
+ <xsl:otherwise><xsl:value-of select="@name"/></xsl:otherwise>
+ </xsl:choose>
+ </xsl:when>
+ <xsl:when test="name()='Barcode'">
+ <xsl:choose>
+ <xsl:when test="@name='all'">[all barcodes]</xsl:when>
+ <xsl:when test="@name='unknown'">[unknown barcode]</xsl:when>
+ <xsl:otherwise><xsl:value-of select="@name"/></xsl:otherwise>
+ </xsl:choose>
+ </xsl:when>
+
+ <xsl:otherwise>
+ <xsl:value-of select="@name"/>
+ </xsl:otherwise>
+ </xsl:choose>
+</xsl:template>
+
+<xsl:template name="getBarcodeDisplayPath">
+ <p>
+ <xsl:for-each select="../../.."><xsl:call-template name="getNodeDisplayName"/></xsl:for-each> /
+ <xsl:for-each select="../.."><xsl:call-template name="getNodeDisplayName"/></xsl:for-each> /
+ <xsl:for-each select=".."><xsl:call-template name="getNodeDisplayName"/></xsl:for-each> /
+ <xsl:for-each select="."><xsl:call-template name="getNodeDisplayName"/></xsl:for-each>
+ </p>
+</xsl:template>
+
+</xsl:stylesheet>
diff --git a/src/xsl/demux/PathUtils.xsl b/src/xsl/demux/PathUtils.xsl
new file mode 100644
index 0000000..28a64ec
--- /dev/null
+++ b/src/xsl/demux/PathUtils.xsl
@@ -0,0 +1,63 @@
+<?xml version="1.0"?>
+<!--
+/**
+ ** BCL to FASTQ file converter
+ ** Copyright (c) 2007-2015 Illumina, Inc.
+ **
+ ** This software is covered by the accompanying EULA
+ ** and certain third party copyright/licenses, and any user of this
+ ** source file is bound by the terms therein.
+ **
+ ** \file PathUtils.xsl
+ **
+ ** \author Roman Petrovski
+ ** \author Mauricio Varea
+ **/
+-->
+<xsl:stylesheet version="1.0"
+xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+xmlns:str="http://exslt.org/strings"
+xmlns:math="http://exslt.org/math"
+xmlns:bcl2fastq="http://www.illumina.com/bcl2fastq"
+>
+
+
+<xsl:template name="getFlowcellSampleBarcodeSimpleLocalPath">
+ <xsl:param name="flowcellId" select="../../../@flowcell-id"/>
+ <xsl:param name="projectId" select="../../@name"/>
+ <xsl:param name="sampleId" select="../@name"/>
+ <xsl:param name="barcodeId" select="@name"/>
+
+ <xsl:value-of select="concat($flowcellId, '/', $projectId, '/', $sampleId, '/', $barcodeId, '/lane.html')"/>
+</xsl:template>
+
+<xsl:template name="getFlowcellSampleBarcodeSimpleGlobalPath">
+ <xsl:variable name="flowcellSummaryPageFileName" ><xsl:call-template name="getFlowcellSampleBarcodeSimpleLocalPath"/></xsl:variable>
+ <xsl:value-of select="concat($OUTPUT_DIRECTORY_HTML_PARAM, '/', $flowcellSummaryPageFileName)"/>
+</xsl:template>
+
+<xsl:template name="getFlowcellSampleBarcodeLocalPath">
+ <xsl:param name="flowcellId" select="../../../@flowcell-id"/>
+ <xsl:param name="projectId" select="../../@name"/>
+ <xsl:param name="sampleId" select="../@name"/>
+ <xsl:param name="barcodeId" select="@name"/>
+
+ <xsl:value-of select="concat($flowcellId, '/', $projectId, '/', $sampleId, '/', $barcodeId, '/laneBarcode.html')"/>
+</xsl:template>
+
+<xsl:template name="getFlowcellSampleBarcodeGlobalPath">
+ <xsl:variable name="flowcellSummaryPageFileName" ><xsl:call-template name="getFlowcellSampleBarcodeLocalPath"/></xsl:variable>
+ <xsl:value-of select="concat($OUTPUT_DIRECTORY_HTML_PARAM, '/', $flowcellSummaryPageFileName)"/>
+</xsl:template>
+
+
+<xsl:template name="getFlowcellRefsLocalPath">
+ <xsl:value-of select="'tree.html'"/>
+</xsl:template>
+
+<xsl:template name="getFlowcellRefsGlobalPath">
+ <xsl:variable name="flowcellRefsLocalPath" ><xsl:call-template name="getFlowcellRefsLocalPath"/></xsl:variable>
+ <xsl:value-of select="concat($OUTPUT_DIRECTORY_HTML_PARAM, '/', $flowcellRefsLocalPath)"/>
+</xsl:template>
+
+</xsl:stylesheet>
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/bcl2fastq2.git
More information about the debian-med-commit
mailing list