[med-svn] [ray] 01/03: Imported Upstream version 2.3.0
Sébastien Boisvert
sboisvert-guest at moszumanska.debian.org
Mon Feb 3 19:22:28 UTC 2014
This is an automated email from the git hooks/post-receive script.
sboisvert-guest pushed a commit to branch master
in repository ray.
commit a38ff74f5ae4661448065fc30c46c243c66c26c9
Author: Sébastien Boisvert <sebastien.boisvert.3 at ulaval.ca>
Date: Mon Feb 3 14:13:11 2014 -0500
Imported Upstream version 2.3.0
---
CMakeLists.txt | 293 +--
Documentation/454.txt | 0
Documentation/AMAZON_EC2.txt | 0
Documentation/Blue_Gene_Q.txt | 5 +
Documentation/COMPILERS.txt | 0
Documentation/CodingStyle.txt | 15 +-
Documentation/Distributed-Storage-Engine.txt | 15 +-
Documentation/Illumina.txt | 0
Documentation/IonTorrent.txt | 0
Documentation/Migration-to-the-Actor-Model.txt | 56 +
Documentation/PacBio.txt | 0
Documentation/Profiling.txt | 2 +-
Documentation/RELEASE_PROCEDURE.txt | 12 +-
Documentation/Ray-Surveyor.md | 58 +
Documentation/Releases.txt | 54 +-
Documentation/Routing.txt | 57 +-
Documentation/SOLiD.txt | 0
Documentation/VISUAL_STUDIO.txt | 0
Documentation/Very-Large-Jobs.txt | 5 +-
Documentation/index.markdown | 3 +
MANUAL_PAGE.txt | 78 +-
Makefile | 260 +--
README.md | 54 +-
RayPlatform/Documentation/Actor-Model.txt | 13 +
RayPlatform/Documentation/DebugMode.txt | 18 +
RayPlatform/Documentation/Gates.txt | 41 +
RayPlatform/Documentation/Handlers.txt | 18 +-
RayPlatform/Documentation/MiniRanks.txt | 329 ++++
RayPlatform/Documentation/Torus.txt | 37 +
RayPlatform/Documentation/plugins.txt | 70 +
RayPlatform/GPL-3.0.txt | 6 +
RayPlatform/LICENSE | 3 +
RayPlatform/Makefile | 25 +-
RayPlatform/Makefile.shared-gcc | 3 -
RayPlatform/{README => README.md} | 20 +-
RayPlatform/RayPlatform/actors/Actor.cpp | 114 ++
RayPlatform/RayPlatform/actors/Actor.h | 75 +
RayPlatform/RayPlatform/actors/Playground.cpp | 211 ++
RayPlatform/RayPlatform/actors/Playground.h | 81 +
.../communication/BufferedData.cpp | 17 +-
.../{ => RayPlatform}/communication/BufferedData.h | 13 +-
RayPlatform/RayPlatform/communication/Message.cpp | 389 ++++
RayPlatform/RayPlatform/communication/Message.h | 176 ++
.../RayPlatform/communication/MessageQueue.cpp | 291 +++
.../RayPlatform/communication/MessageQueue.h | 169 ++
.../communication/MessageRouter.cpp | 152 +-
.../communication/MessageRouter.h | 36 +-
.../communication/MessagesHandler.cpp | 832 +++++---
.../communication/MessagesHandler.h | 122 +-
.../communication/VirtualCommunicator.cpp | 58 +-
.../communication/VirtualCommunicator.h | 18 +-
.../{ => RayPlatform}/communication/mpi_tags.cpp | 8 +-
.../{ => RayPlatform}/communication/mpi_tags.h | 10 +-
RayPlatform/{ => RayPlatform}/core/ComputeCore.cpp | 751 +++++--
RayPlatform/{ => RayPlatform}/core/ComputeCore.h | 148 +-
.../core/MiniRank.cpp} | 16 +-
.../Worker.h => RayPlatform/core/MiniRank.h} | 41 +-
.../{ => RayPlatform}/core/OperatingSystem.cpp | 115 +-
.../{ => RayPlatform}/core/OperatingSystem.h | 25 +-
RayPlatform/RayPlatform/core/RankProcess.h | 323 ++++
.../{ => RayPlatform}/core/master_modes.cpp | 8 +-
RayPlatform/{ => RayPlatform}/core/master_modes.h | 8 +-
RayPlatform/{ => RayPlatform}/core/slave_modes.cpp | 9 +-
RayPlatform/{ => RayPlatform}/core/slave_modes.h | 8 +-
RayPlatform/{ => RayPlatform}/core/statistics.cpp | 9 +-
RayPlatform/{ => RayPlatform}/core/statistics.h | 6 +-
RayPlatform/{ => RayPlatform}/core/types.h | 19 +-
.../{ => RayPlatform}/cryptography/crypto.cpp | 32 +-
.../cryptography/crypto.h} | 24 +-
RayPlatform/RayPlatform/files/FileReader.cpp | 179 ++
RayPlatform/RayPlatform/files/FileReader.h | 60 +
.../handlers/MasterModeExecutor.cpp | 36 +-
.../handlers/MasterModeExecutor.h | 28 +-
.../RayPlatform/handlers/MasterModeHandler.h | 106 +
.../handlers/MessageTagExecutor.cpp | 34 +-
.../handlers/MessageTagExecutor.h | 31 +-
.../RayPlatform/handlers/MessageTagHandler.h | 101 +
.../handlers/SlaveModeExecutor.cpp | 33 +-
.../{ => RayPlatform}/handlers/SlaveModeExecutor.h | 30 +-
.../RayPlatform/handlers/SlaveModeHandler.h | 102 +
.../memory/ChunkAllocatorWithDefragmentation.cpp | 11 +-
.../memory/ChunkAllocatorWithDefragmentation.h | 11 +-
.../memory/DefragmentationGroup.cpp | 11 +-
.../memory/DefragmentationGroup.h | 9 +-
.../memory/DefragmentationLane.cpp | 9 +-
.../{ => RayPlatform}/memory/DefragmentationLane.h | 8 +-
RayPlatform/RayPlatform/memory/DirtyBuffer.cpp | 72 +
RayPlatform/RayPlatform/memory/DirtyBuffer.h | 56 +
.../{ => RayPlatform}/memory/MyAllocator.cpp | 27 +-
RayPlatform/{ => RayPlatform}/memory/MyAllocator.h | 15 +-
.../memory/ReusableMemoryStore.cpp | 9 +-
.../{ => RayPlatform}/memory/ReusableMemoryStore.h | 6 +-
RayPlatform/RayPlatform/memory/RingAllocator.cpp | 536 +++++
.../{ => RayPlatform}/memory/RingAllocator.h | 73 +-
RayPlatform/{ => RayPlatform}/memory/allocator.cpp | 13 +-
RayPlatform/{ => RayPlatform}/memory/allocator.h | 6 +-
.../{ => RayPlatform}/plugins/CorePlugin.cpp | 8 +-
RayPlatform/{ => RayPlatform}/plugins/CorePlugin.h | 58 +-
.../{ => RayPlatform}/plugins/RegisteredPlugin.cpp | 15 +-
.../{ => RayPlatform}/plugins/RegisteredPlugin.h | 9 +-
.../{ => RayPlatform}/profiling/Derivative.cpp | 15 +-
.../{ => RayPlatform}/profiling/Derivative.h | 9 +-
.../RayPlatform/profiling/ProcessStatus.cpp | 102 +
RayPlatform/RayPlatform/profiling/ProcessStatus.h | 28 +
.../{ => RayPlatform}/profiling/Profiler.cpp | 14 +-
RayPlatform/{ => RayPlatform}/profiling/Profiler.h | 6 +-
.../{ => RayPlatform}/profiling/TickLogger.cpp | 10 +-
.../{ => RayPlatform}/profiling/TickLogger.h | 8 +-
.../{ => RayPlatform}/profiling/TimePrinter.cpp | 39 +-
.../{ => RayPlatform}/profiling/TimePrinter.h | 17 +-
.../{ => RayPlatform}/routing/ConnectionGraph.cpp | 55 +-
.../{ => RayPlatform}/routing/ConnectionGraph.h | 38 +-
.../routing/GraphImplementation.cpp | 9 +-
.../routing/GraphImplementation.h | 8 +-
.../routing/GraphImplementationComplete.cpp | 8 +-
.../routing/GraphImplementationComplete.h | 8 +-
.../routing/GraphImplementationDeBruijn.cpp | 9 +-
.../routing/GraphImplementationDeBruijn.h | 9 +-
.../routing/GraphImplementationExperimental.cpp | 9 +-
.../routing/GraphImplementationExperimental.h | 9 +-
.../routing/GraphImplementationGroup.cpp | 9 +-
.../routing/GraphImplementationGroup.h | 8 +-
.../routing/GraphImplementationKautz.cpp | 9 +-
.../routing/GraphImplementationKautz.h | 9 +-
.../routing/GraphImplementationRandom.cpp | 9 +-
.../routing/GraphImplementationRandom.h | 8 +-
.../routing/Polytope.cpp} | 69 +-
.../Hypercube.h => RayPlatform/routing/Polytope.h} | 21 +-
RayPlatform/RayPlatform/routing/Torus.cpp | 558 ++++++
.../Hypercube.h => RayPlatform/routing/Torus.h} | 77 +-
.../{ => RayPlatform}/scheduling/SwitchMan.cpp | 39 +-
.../{ => RayPlatform}/scheduling/SwitchMan.h | 28 +-
.../{ => RayPlatform}/scheduling/TaskCreator.cpp | 9 +-
.../{ => RayPlatform}/scheduling/TaskCreator.h | 12 +-
.../scheduling/VirtualProcessor.cpp | 18 +-
.../scheduling/VirtualProcessor.h | 13 +-
RayPlatform/{ => RayPlatform}/scheduling/Worker.h | 11 +-
RayPlatform/RayPlatform/store/CarriageableItem.h | 61 +
RayPlatform/RayPlatform/store/KeyValueStore.cpp | 604 ++++++
RayPlatform/RayPlatform/store/KeyValueStore.h | 177 ++
.../RayPlatform/store/KeyValueStoreItem.cpp | 105 +
RayPlatform/RayPlatform/store/KeyValueStoreItem.h | 50 +
.../RayPlatform/store/KeyValueStoreRequest.cpp | 66 +
.../RayPlatform/store/KeyValueStoreRequest.h | 54 +
.../{ => RayPlatform}/structures/MyHashTable.h | 18 +-
.../structures/MyHashTableGroup.h | 9 +-
.../structures/MyHashTableIterator.h | 9 +-
RayPlatform/{ => RayPlatform}/structures/MyStack.h | 8 +-
.../{ => RayPlatform}/structures/SplayNode.h | 8 +-
.../{ => RayPlatform}/structures/SplayTree.h | 20 +-
.../structures/SplayTreeIterator.h | 26 +-
.../{ => RayPlatform}/structures/StaticVector.cpp | 17 +-
.../{ => RayPlatform}/structures/StaticVector.h | 16 +-
RayPlatform/common.mk | 120 +-
RayPlatform/communication/Message.cpp | 85 -
RayPlatform/communication/Message.h | 92 -
RayPlatform/git.txt | 1 -
RayPlatform/github.txt | 1 -
RayPlatform/handlers/MasterModeHandler.h | 42 -
RayPlatform/handlers/MessageTagHandler.h | 46 -
RayPlatform/handlers/SlaveModeHandler.h | 42 -
RayPlatform/memory/RingAllocator.cpp | 222 ---
RayPlatform/tag.txt | 1 -
code/{plugin_Amos => Amos}/Amos.cpp | 59 +-
code/{plugin_Amos => Amos}/Amos.h | 27 +-
code/Amos/Makefile | 3 +
.../CoverageDistribution.cpp | 10 +-
.../CoverageDistribution.h | 8 +-
.../CoverageGatherer.cpp | 148 +-
.../CoverageGatherer.h | 20 +-
code/CoverageGatherer/Makefile | 4 +
.../EdgePurger.cpp | 109 +-
.../{plugin_EdgePurger => EdgePurger}/EdgePurger.h | 37 +-
.../EdgePurgerWorker.cpp | 8 +-
.../EdgePurgerWorker.h | 20 +-
code/EdgePurger/Makefile | 4 +
code/Example/Example.cpp | 273 +++
code/Example/Example.h | 117 ++
code/Example/Makefile | 3 +
.../FusionData.cpp | 95 +-
.../{plugin_FusionData => FusionData}/FusionData.h | 35 +-
code/FusionData/Makefile | 3 +
.../FusionTaskCreator.cpp | 27 +-
.../FusionTaskCreator.h | 43 +-
.../FusionWorker.cpp | 31 +-
.../FusionWorker.h | 20 +-
code/FusionTaskCreator/Makefile | 4 +
.../GeneOntology.cpp | 41 +-
.../GeneOntology.h | 38 +-
.../KeyEncoder.cpp | 11 +-
.../KeyEncoder.h | 8 +-
code/GeneOntology/Makefile | 4 +
code/{plugin_GeneOntology => GeneOntology}/types.h | 0
.../GenomeNeighbourhood.cpp | 53 +-
.../GenomeNeighbourhood.h | 33 +-
code/GenomeNeighbourhood/Makefile | 5 +
.../Neighbour.cpp | 2 +-
.../Neighbour.h | 5 +-
.../NeighbourPair.cpp | 5 +-
.../NeighbourPair.h | 5 +-
.../JoinerTaskCreator.cpp | 32 +-
.../JoinerTaskCreator.h | 39 +-
.../JoinerWorker.cpp | 169 +-
.../JoinerWorker.h | 22 +-
code/JoinerTaskCreator/Makefile | 4 +
.../BloomFilter.cpp | 20 +-
.../BloomFilter.h | 5 +-
.../Kmer.cpp | 188 +-
.../Kmer.h | 94 +-
.../KmerAcademyBuilder.cpp | 48 +-
.../KmerAcademyBuilder.h | 32 +-
code/KmerAcademyBuilder/Makefile | 5 +
code/{plugin_Library => Library}/Library.cpp | 182 +-
code/{plugin_Library => Library}/Library.h | 46 +-
.../LibraryPeakFinder.cpp | 8 +-
.../LibraryPeakFinder.h | 2 +-
code/{plugin_Library => Library}/LibraryWorker.cpp | 22 +-
code/{plugin_Library => Library}/LibraryWorker.h | 18 +-
code/Library/Makefile | 5 +
.../MachineHelper.cpp | 720 ++++---
.../MachineHelper.h | 179 +-
code/MachineHelper/Makefile | 3 +
code/Makefile | 35 -
code/MessageProcessor/Makefile | 3 +
.../MessageProcessor.cpp | 782 ++++----
.../MessageProcessor.h | 307 ++-
code/Mock/Makefile | 5 +
code/Mock/Mock.cpp | 47 +
.../cryptography/crypto.h => code/Mock/Mock.h | 25 +-
code/{application_core => Mock}/Parameters.cpp | 447 ++++-
code/{application_core => Mock}/Parameters.h | 42 +-
.../common_functions.cpp | 110 +-
code/{application_core => Mock}/common_functions.h | 37 +-
code/{application_core => Mock}/constants.h | 82 +-
code/NetworkTest/Makefile | 3 +
.../NetworkTest.cpp | 108 +-
.../NetworkTest.h | 33 +-
code/Partitioner/Makefile | 3 +
.../Partitioner.cpp | 118 +-
.../Partitioner.h | 30 +-
code/PathEvaluator/Makefile | 3 +
code/PathEvaluator/PathEvaluator.cpp | 113 ++
code/PathEvaluator/PathEvaluator.h | 68 +
code/Scaffolder/Makefile | 8 +
.../Scaffolder.cpp | 562 +++++-
.../{plugin_Scaffolder => Scaffolder}/Scaffolder.h | 98 +-
.../ScaffoldingAlgorithm.cpp | 28 +-
.../ScaffoldingAlgorithm.h | 13 +-
.../ScaffoldingEdge.cpp | 5 +-
.../ScaffoldingEdge.h | 8 +-
.../ScaffoldingLink.cpp | 4 +-
.../ScaffoldingLink.h | 2 +-
.../ScaffoldingVertex.cpp | 5 +-
.../ScaffoldingVertex.h | 10 +-
.../SummarizedLink.cpp | 8 +-
.../SummarizedLink.h | 9 +-
code/{plugin_Searcher => Searcher}/ColorSet.cpp | 5 +-
code/{plugin_Searcher => Searcher}/ColorSet.h | 2 +-
.../ColoredPeakFinder.cpp | 8 +-
.../ColoredPeakFinder.h | 2 +-
code/{plugin_Searcher => Searcher}/ContigHit.cpp | 2 +-
code/{plugin_Searcher => Searcher}/ContigHit.h | 8 +-
.../ContigSearchEntry.cpp | 2 +-
.../ContigSearchEntry.h | 6 +-
.../DistributionWriter.cpp | 7 +-
.../DistributionWriter.h | 5 +-
code/Searcher/Makefile | 11 +
.../QualityCaller.cpp | 5 +-
code/{plugin_Searcher => Searcher}/QualityCaller.h | 6 +-
.../SearchDirectory.cpp | 29 +-
.../SearchDirectory.h | 9 +-
code/{plugin_Searcher => Searcher}/Searcher.cpp | 97 +-
code/{plugin_Searcher => Searcher}/Searcher.h | 99 +-
.../VirtualKmerColor.cpp | 5 +-
.../VirtualKmerColor.h | 5 +-
.../BubbleData.h | 15 +-
.../BubbleTool.cpp | 8 +-
.../BubbleTool.h | 11 +-
.../Chooser.cpp | 7 +-
.../Chooser.h | 7 +-
.../DepthFirstSearchData.cpp | 24 +-
.../DepthFirstSearchData.h | 18 +-
.../Direction.cpp | 13 +-
.../Direction.h | 10 +-
.../ExtensionData.cpp | 44 +-
.../ExtensionData.h | 42 +-
.../ExtensionElement.cpp | 37 +-
.../ExtensionElement.h | 24 +-
code/SeedExtender/Makefile | 14 +
.../NovaEngine.cpp | 5 +-
.../NovaEngine.h | 5 +-
.../OpenAssemblerChooser.cpp | 6 +-
.../OpenAssemblerChooser.h | 6 +-
.../ReadFetcher.cpp | 15 +-
.../ReadFetcher.h | 14 +-
.../SeedExtender.cpp | 464 +++--
.../SeedExtender.h | 108 +-
.../TipWatchdog.cpp | 4 +-
.../TipWatchdog.h | 8 +-
.../VertexMessenger.cpp | 25 +-
.../VertexMessenger.h | 15 +-
code/SeedingData/GraphPath.cpp | 743 +++++++
code/SeedingData/GraphPath.h | 154 ++
code/SeedingData/Makefile | 9 +
code/SeedingData/PathHandle.cpp | 128 ++
code/SeedingData/PathHandle.h | 69 +
.../SeedWorker.cpp | 286 ++-
.../SeedWorker.h | 62 +-
.../SeedingData.cpp | 330 ++--
.../SeedingData.h | 56 +-
.../DynamicVector.h | 7 +-
.../IndexerWorker.cpp | 14 +-
.../IndexerWorker.h | 19 +-
code/SequencesIndexer/Makefile | 9 +
.../PairedRead.cpp | 18 +-
.../PairedRead.h | 12 +-
.../ReadAnnotation.cpp | 20 +-
.../ReadAnnotation.h | 10 +-
.../SequencesIndexer.cpp | 38 +-
.../SequencesIndexer.h | 39 +-
.../ArrayOfReads.cpp | 7 +-
.../ArrayOfReads.h | 6 +-
code/SequencesLoader/BufferedReader.cpp | 206 ++
code/SequencesLoader/BufferedReader.h | 66 +
.../BzReader.cpp | 17 +-
.../BzReader.h | 8 +-
.../ColorSpaceDecoder.cpp | 39 +-
.../ColorSpaceDecoder.h | 8 +-
.../ColorSpaceLoader.cpp | 36 +-
.../ColorSpaceLoader.h | 26 +-
code/SequencesLoader/ExportLoader.cpp | 108 ++
.../ExportLoader.h} | 45 +-
code/SequencesLoader/FastaBz2Loader.cpp | 51 +
.../FastaBz2Loader.h} | 37 +-
code/SequencesLoader/FastaGzLoader.cpp | 53 +
.../FastaGzLoader.h} | 42 +-
.../FastaLoader.cpp | 10 +-
.../FastaLoader.h | 23 +-
code/SequencesLoader/FastaLoaderForReads.cpp | 49 +
.../FastaLoaderForReads.h} | 41 +-
.../FastqBz2Loader.cpp | 44 +-
.../FastqBz2Loader.h | 36 +-
code/SequencesLoader/FastqGzLoader.cpp | 305 +++
code/SequencesLoader/FastqGzLoader.h | 75 +
code/SequencesLoader/FastqLoader.cpp | 134 ++
.../FastqLoader.h} | 46 +-
code/SequencesLoader/Loader.cpp | 154 ++
.../Loader.h | 64 +-
code/SequencesLoader/LoaderFactory.cpp | 50 +
code/SequencesLoader/LoaderFactory.h | 77 +
code/SequencesLoader/LoaderInterface.cpp | 54 +
.../LoaderInterface.h} | 41 +-
code/SequencesLoader/Makefile | 25 +
.../Read.cpp | 52 +-
.../Read.h | 18 +-
code/SequencesLoader/ReadHandle.cpp | 100 +
code/SequencesLoader/ReadHandle.h | 58 +
code/SequencesLoader/SequenceFileDetector.cpp | 380 ++++
code/SequencesLoader/SequenceFileDetector.h | 59 +
.../SequencesLoader.cpp | 185 +-
.../SequencesLoader.h | 41 +-
.../SffLoader.cpp | 35 +-
.../SffLoader.h | 29 +-
.../SpuriousSeedAnnihilator/AnnihilationWorker.cpp | 614 ++++++
code/SpuriousSeedAnnihilator/AnnihilationWorker.h | 141 ++
code/SpuriousSeedAnnihilator/AnnotationFetcher.cpp | 154 ++
code/SpuriousSeedAnnihilator/AnnotationFetcher.h | 70 +
code/SpuriousSeedAnnihilator/AttributeFetcher.cpp | 98 +
code/SpuriousSeedAnnihilator/AttributeFetcher.h | 107 +
.../SpuriousSeedAnnihilator/GossipAssetManager.cpp | 433 +++++
code/SpuriousSeedAnnihilator/GossipAssetManager.h | 73 +
code/SpuriousSeedAnnihilator/GraphExplorer.cpp | 491 +++++
code/SpuriousSeedAnnihilator/GraphExplorer.h | 112 ++
code/SpuriousSeedAnnihilator/GraphSearchResult.cpp | 267 +++
code/SpuriousSeedAnnihilator/GraphSearchResult.h | 89 +
code/SpuriousSeedAnnihilator/Makefile | 13 +
code/SpuriousSeedAnnihilator/NanoMerger.cpp | 167 ++
code/SpuriousSeedAnnihilator/NanoMerger.h | 136 ++
.../SeedFilteringWorkflow.cpp | 138 ++
.../SeedFilteringWorkflow.h | 94 +
code/SpuriousSeedAnnihilator/SeedGossipSolver.cpp | 371 ++++
code/SpuriousSeedAnnihilator/SeedGossipSolver.h | 64 +
.../SeedMergingWorkflow.cpp | 169 ++
code/SpuriousSeedAnnihilator/SeedMergingWorkflow.h | 103 +
.../SpuriousSeedAnnihilator.cpp | 2042 ++++++++++++++++++++
.../SpuriousSeedAnnihilator.h | 363 ++++
code/Surveyor/CoalescenceManager.cpp | 425 ++++
code/Surveyor/CoalescenceManager.h | 82 +
code/Surveyor/ExperimentVertex.cpp | 49 +
code/Surveyor/ExperimentVertex.h | 58 +
code/Surveyor/GenomeGraphReader.cpp | 251 +++
code/Surveyor/GenomeGraphReader.h | 75 +
code/Surveyor/Makefile | 8 +
code/Surveyor/MatrixOwner.cpp | 232 +++
code/Surveyor/MatrixOwner.h | 68 +
code/Surveyor/Mother.cpp | 482 +++++
code/Surveyor/Mother.h | 118 ++
code/Surveyor/StoreKeeper.cpp | 460 +++++
code/Surveyor/StoreKeeper.h | 99 +
.../GenomeToTaxonLoader.cpp | 4 +-
.../GenomeToTaxonLoader.h | 8 +-
code/TaxonomyViewer/Makefile | 6 +
.../TaxonNameLoader.cpp | 4 +-
.../TaxonNameLoader.h | 7 +-
.../TaxonomicTreeLoader.cpp} | 10 +-
.../TaxonomicTreeLoader.h} | 13 +-
.../TaxonomyViewer.cpp} | 117 +-
.../TaxonomyViewer.h} | 52 +-
.../types.h | 0
.../GridTable.cpp | 43 +-
.../GridTable.h | 16 +-
.../GridTableIterator.cpp | 10 +-
.../GridTableIterator.h | 12 +-
code/VerticesExtractor/Makefile | 9 +
.../Vertex.cpp | 128 +-
.../Vertex.h | 97 +-
.../VerticesExtractor.cpp | 94 +-
.../VerticesExtractor.h | 39 +-
code/application_core/Machine.cpp | 444 +++--
code/application_core/Machine.h | 164 +-
code/application_core/Makefile | 5 +-
code/application_core/ray_main.cpp | 16 +-
code/plugin_Amos/Makefile | 7 -
code/plugin_CoverageGatherer/Makefile | 8 -
code/plugin_EdgePurger/Makefile | 8 -
code/plugin_FusionData/Makefile | 5 -
code/plugin_FusionTaskCreator/Makefile | 6 -
code/plugin_GeneOntology/Makefile | 6 -
code/plugin_GenomeNeighbourhood/Makefile | 11 -
code/plugin_JoinerTaskCreator/Makefile | 8 -
code/plugin_KmerAcademyBuilder/Makefile | 8 -
code/plugin_Library/Makefile | 8 -
code/plugin_MachineHelper/Makefile | 7 -
code/plugin_MessageProcessor/Makefile | 7 -
code/plugin_NetworkTest/Makefile | 7 -
code/plugin_Partitioner/Makefile | 6 -
code/plugin_PhylogenyViewer/Makefile | 8 -
code/plugin_Scaffolder/Makefile | 13 -
code/plugin_Searcher/Makefile | 13 -
code/plugin_SeedExtender/Makefile | 14 -
code/plugin_SeedingData/AssemblySeed.cpp | 160 --
code/plugin_SeedingData/AssemblySeed.h | 66 -
code/plugin_SeedingData/Makefile | 8 -
code/plugin_SequencesIndexer/Makefile | 9 -
code/plugin_SequencesLoader/FastqGzLoader.cpp | 81 -
code/plugin_SequencesLoader/FastqLoader.cpp | 76 -
code/plugin_SequencesLoader/Loader.cpp | 207 --
code/plugin_SequencesLoader/Makefile | 13 -
code/plugin_VerticesExtractor/Makefile | 9 -
git.txt | 1 -
github.txt | 1 -
scripts/Build-Link-Time-Optimization.sh | 10 +-
scripts/ShipProduct.sh | 34 +-
scripts/install.sh | 33 -
tag.txt | 1 -
455 files changed, 27908 insertions(+), 6295 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 06a3d79..ea890aa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,130 +1,175 @@
cmake_minimum_required(VERSION 2.6 )
set( CMAKE_CXX_COMPILER mpicxx )
-set( CMAKE_CXX_FLAGS "-O3 -Wall -ansi" )
-include_directories( RayPlatform code )
+set( CMAKE_CXX_FLAGS "-O3 -Wall -std=c++98 -g" )
+include_directories( . RayPlatform )
add_executable( Ray
-code/plugin_MachineHelper/MachineHelper.cpp
-code/plugin_Library/Library.cpp
-code/plugin_Library/LibraryPeakFinder.cpp
-code/plugin_Library/LibraryWorker.cpp
-code/plugin_FusionTaskCreator/FusionTaskCreator.cpp
-code/plugin_FusionTaskCreator/FusionWorker.cpp
-code/plugin_NetworkTest/NetworkTest.cpp
-code/plugin_SequencesLoader/SequencesLoader.cpp
-code/plugin_SequencesLoader/ArrayOfReads.cpp
-code/plugin_SequencesLoader/ColorSpaceDecoder.cpp
-code/plugin_SequencesLoader/FastqGzLoader.cpp
-code/plugin_SequencesLoader/SffLoader.cpp
-code/plugin_SequencesLoader/Read.cpp
-code/plugin_SequencesLoader/FastaLoader.cpp
-code/plugin_SequencesLoader/BzReader.cpp
-code/plugin_SequencesLoader/FastqBz2Loader.cpp
-code/plugin_SequencesLoader/Loader.cpp
-code/plugin_SequencesLoader/FastqLoader.cpp
-code/plugin_SequencesLoader/ColorSpaceLoader.cpp
-code/plugin_VerticesExtractor/Vertex.cpp
-code/plugin_VerticesExtractor/GridTable.cpp
-code/plugin_VerticesExtractor/GridTableIterator.cpp
-code/plugin_VerticesExtractor/VerticesExtractor.cpp
-code/plugin_JoinerTaskCreator/JoinerTaskCreator.cpp
-code/plugin_JoinerTaskCreator/JoinerWorker.cpp
-code/plugin_SeedingData/SeedingData.cpp
-code/plugin_SeedingData/SeedWorker.cpp
-code/plugin_SeedingData/AssemblySeed.cpp
-code/plugin_FusionData/FusionData.cpp
-code/plugin_CoverageGatherer/CoverageGatherer.cpp
-code/plugin_CoverageGatherer/CoverageDistribution.cpp
-code/plugin_SeedExtender/Direction.cpp
-code/plugin_SeedExtender/TipWatchdog.cpp
-code/plugin_SeedExtender/Chooser.cpp
-code/plugin_SeedExtender/ExtensionElement.cpp
-code/plugin_SeedExtender/SeedExtender.cpp
-code/plugin_SeedExtender/ExtensionData.cpp
-code/plugin_SeedExtender/VertexMessenger.cpp
-code/plugin_SeedExtender/ReadFetcher.cpp
-code/plugin_SeedExtender/BubbleTool.cpp
-code/plugin_SeedExtender/NovaEngine.cpp
-code/plugin_SeedExtender/DepthFirstSearchData.cpp
-code/plugin_SeedExtender/OpenAssemblerChooser.cpp
-code/plugin_MessageProcessor/MessageProcessor.cpp
-code/plugin_Partitioner/Partitioner.cpp
-code/plugin_SequencesIndexer/SequencesIndexer.cpp
-code/plugin_SequencesIndexer/PairedRead.cpp
-code/plugin_SequencesIndexer/ReadAnnotation.cpp
-code/plugin_SequencesIndexer/IndexerWorker.cpp
-code/application_core/Parameters.cpp
+
+
+
code/application_core/ray_main.cpp
-code/application_core/common_functions.cpp
code/application_core/Machine.cpp
-code/plugin_Amos/Amos.cpp
-code/plugin_KmerAcademyBuilder/Kmer.cpp
-code/plugin_KmerAcademyBuilder/KmerAcademyBuilder.cpp
-code/plugin_KmerAcademyBuilder/BloomFilter.cpp
-code/plugin_EdgePurger/EdgePurger.cpp
-code/plugin_EdgePurger/EdgePurgerWorker.cpp
-code/plugin_Searcher/ColorSet.cpp
-code/plugin_Searcher/ContigHit.cpp
-code/plugin_Searcher/Searcher.cpp
-code/plugin_Searcher/ContigSearchEntry.cpp
-code/plugin_Searcher/VirtualKmerColor.cpp
-code/plugin_Searcher/DistributionWriter.cpp
-code/plugin_Searcher/ColoredPeakFinder.cpp
-code/plugin_Searcher/QualityCaller.cpp
-code/plugin_Searcher/SearchDirectory.cpp
-code/plugin_Scaffolder/Scaffolder.cpp
-code/plugin_Scaffolder/SummarizedLink.cpp
-code/plugin_Scaffolder/ScaffoldingAlgorithm.cpp
-code/plugin_Scaffolder/ScaffoldingVertex.cpp
-code/plugin_Scaffolder/ScaffoldingEdge.cpp
-code/plugin_Scaffolder/ScaffoldingLink.cpp
-code/plugin_GeneOntology/KeyEncoder.cpp
-code/plugin_GeneOntology/GeneOntology.cpp
-code/plugin_GenomeNeighbourhood/GenomeNeighbourhood.cpp
-code/plugin_GenomeNeighbourhood/NeighbourPair.cpp
-code/plugin_GenomeNeighbourhood/Neighbour.cpp
-code/plugin_PhylogenyViewer/GenomeToTaxonLoader.cpp
-code/plugin_PhylogenyViewer/PhylogenyViewer.cpp
-code/plugin_PhylogenyViewer/PhylogeneticTreeLoader.cpp
-code/plugin_PhylogenyViewer/TaxonNameLoader.cpp
-RayPlatform/handlers/MasterModeExecutor.cpp
-RayPlatform/handlers/SlaveModeExecutor.cpp
-RayPlatform/handlers/MessageTagExecutor.cpp
-RayPlatform/core/master_modes.cpp
-RayPlatform/core/ComputeCore.cpp
-RayPlatform/core/slave_modes.cpp
-RayPlatform/core/OperatingSystem.cpp
-RayPlatform/core/statistics.cpp
-RayPlatform/routing/GraphImplementationExperimental.cpp
-RayPlatform/routing/GraphImplementationKautz.cpp
-RayPlatform/routing/ConnectionGraph.cpp
-RayPlatform/routing/GraphImplementation.cpp
-RayPlatform/routing/GraphImplementationGroup.cpp
-RayPlatform/routing/GraphImplementationComplete.cpp
-RayPlatform/routing/GraphImplementationDeBruijn.cpp
-RayPlatform/routing/GraphImplementationRandom.cpp
-RayPlatform/routing/Hypercube.cpp
-RayPlatform/scheduling/TaskCreator.cpp
-RayPlatform/scheduling/SwitchMan.cpp
-RayPlatform/scheduling/VirtualProcessor.cpp
-RayPlatform/profiling/Derivative.cpp
-RayPlatform/profiling/TimePrinter.cpp
-RayPlatform/profiling/Profiler.cpp
-RayPlatform/profiling/TickLogger.cpp
-RayPlatform/cryptography/crypto.cpp
-RayPlatform/structures/StaticVector.cpp
-RayPlatform/memory/ReusableMemoryStore.cpp
-RayPlatform/memory/ChunkAllocatorWithDefragmentation.cpp
-RayPlatform/memory/DefragmentationGroup.cpp
-RayPlatform/memory/RingAllocator.cpp
-RayPlatform/memory/MyAllocator.cpp
-RayPlatform/memory/allocator.cpp
-RayPlatform/memory/DefragmentationLane.cpp
-RayPlatform/plugins/CorePlugin.cpp
-RayPlatform/plugins/RegisteredPlugin.cpp
-RayPlatform/communication/VirtualCommunicator.cpp
-RayPlatform/communication/MessagesHandler.cpp
-RayPlatform/communication/MessageRouter.cpp
-RayPlatform/communication/Message.cpp
-RayPlatform/communication/mpi_tags.cpp
-RayPlatform/communication/BufferedData.cpp
+code/Partitioner/Partitioner.cpp
+code/Searcher/DistributionWriter.cpp
+code/Searcher/Searcher.cpp
+code/Searcher/ContigHit.cpp
+code/Searcher/ContigSearchEntry.cpp
+code/Searcher/SearchDirectory.cpp
+code/Searcher/ColorSet.cpp
+code/Searcher/VirtualKmerColor.cpp
+code/Searcher/QualityCaller.cpp
+code/Searcher/ColoredPeakFinder.cpp
+code/KmerAcademyBuilder/KmerAcademyBuilder.cpp
+code/KmerAcademyBuilder/Kmer.cpp
+code/KmerAcademyBuilder/BloomFilter.cpp
+code/SequencesLoader/BzReader.cpp
+code/SequencesLoader/FastaGzLoader.cpp
+code/SequencesLoader/ExportLoader.cpp
+code/SequencesLoader/LoaderFactory.cpp
+code/SequencesLoader/FastqLoader.cpp
+code/SequencesLoader/FastqGzLoader.cpp
+code/SequencesLoader/LoaderInterface.cpp
+code/SequencesLoader/FastaBz2Loader.cpp
+code/SequencesLoader/SffLoader.cpp
+code/SequencesLoader/FastaLoader.cpp
+code/SequencesLoader/ColorSpaceLoader.cpp
+code/SequencesLoader/Loader.cpp
+code/SequencesLoader/ColorSpaceDecoder.cpp
+code/SequencesLoader/FastqBz2Loader.cpp
+code/SequencesLoader/ArrayOfReads.cpp
+code/SequencesLoader/BufferedReader.cpp
+code/SequencesLoader/ReadHandle.cpp
+code/SequencesLoader/FastaLoaderForReads.cpp
+code/SequencesLoader/SequenceFileDetector.cpp
+code/SequencesLoader/Read.cpp
+code/SequencesLoader/SequencesLoader.cpp
+code/JoinerTaskCreator/JoinerTaskCreator.cpp
+code/JoinerTaskCreator/JoinerWorker.cpp
+code/SeedExtender/ExtensionElement.cpp
+code/SeedExtender/ReadFetcher.cpp
+code/SeedExtender/OpenAssemblerChooser.cpp
+code/SeedExtender/VertexMessenger.cpp
+code/SeedExtender/TipWatchdog.cpp
+code/SeedExtender/NovaEngine.cpp
+code/SeedExtender/BubbleTool.cpp
+code/SeedExtender/Chooser.cpp
+code/SeedExtender/SeedExtender.cpp
+code/SeedExtender/Direction.cpp
+code/SeedExtender/ExtensionData.cpp
+code/SeedExtender/DepthFirstSearchData.cpp
+code/Library/Library.cpp
+code/Library/LibraryWorker.cpp
+code/Library/LibraryPeakFinder.cpp
+code/Mock/Mock.cpp
+code/Mock/Parameters.cpp
+code/Mock/common_functions.cpp
+code/Example/Example.cpp
+code/TaxonomyViewer/TaxonomyViewer.cpp
+code/TaxonomyViewer/TaxonomicTreeLoader.cpp
+code/TaxonomyViewer/TaxonNameLoader.cpp
+code/TaxonomyViewer/GenomeToTaxonLoader.cpp
+code/GeneOntology/KeyEncoder.cpp
+code/GeneOntology/GeneOntology.cpp
+code/MessageProcessor/MessageProcessor.cpp
+code/PathEvaluator/PathEvaluator.cpp
+code/VerticesExtractor/VerticesExtractor.cpp
+code/VerticesExtractor/Vertex.cpp
+code/VerticesExtractor/GridTableIterator.cpp
+code/VerticesExtractor/GridTable.cpp
+code/SpuriousSeedAnnihilator/AttributeFetcher.cpp
+code/SpuriousSeedAnnihilator/SeedFilteringWorkflow.cpp
+code/SpuriousSeedAnnihilator/AnnotationFetcher.cpp
+code/SpuriousSeedAnnihilator/GraphSearchResult.cpp
+code/SpuriousSeedAnnihilator/NanoMerger.cpp
+code/SpuriousSeedAnnihilator/SeedMergingWorkflow.cpp
+code/SpuriousSeedAnnihilator/SpuriousSeedAnnihilator.cpp
+code/SpuriousSeedAnnihilator/GraphExplorer.cpp
+code/SpuriousSeedAnnihilator/SeedGossipSolver.cpp
+code/SpuriousSeedAnnihilator/GossipAssetManager.cpp
+code/SpuriousSeedAnnihilator/AnnihilationWorker.cpp
+code/MachineHelper/MachineHelper.cpp
+code/Surveyor/MatrixOwner.cpp
+code/Surveyor/StoreKeeper.cpp
+code/Surveyor/CoalescenceManager.cpp
+code/Surveyor/ExperimentVertex.cpp
+code/Surveyor/Mother.cpp
+code/Surveyor/GenomeGraphReader.cpp
+code/FusionTaskCreator/FusionWorker.cpp
+code/FusionTaskCreator/FusionTaskCreator.cpp
+code/Amos/Amos.cpp
+code/Scaffolder/Scaffolder.cpp
+code/Scaffolder/ScaffoldingEdge.cpp
+code/Scaffolder/ScaffoldingVertex.cpp
+code/Scaffolder/ScaffoldingLink.cpp
+code/Scaffolder/SummarizedLink.cpp
+code/Scaffolder/ScaffoldingAlgorithm.cpp
+code/GenomeNeighbourhood/GenomeNeighbourhood.cpp
+code/GenomeNeighbourhood/Neighbour.cpp
+code/GenomeNeighbourhood/NeighbourPair.cpp
+code/EdgePurger/EdgePurger.cpp
+code/EdgePurger/EdgePurgerWorker.cpp
+code/NetworkTest/NetworkTest.cpp
+code/SequencesIndexer/ReadAnnotation.cpp
+code/SequencesIndexer/PairedRead.cpp
+code/SequencesIndexer/IndexerWorker.cpp
+code/SequencesIndexer/SequencesIndexer.cpp
+code/FusionData/FusionData.cpp
+code/CoverageGatherer/CoverageGatherer.cpp
+code/CoverageGatherer/CoverageDistribution.cpp
+code/SeedingData/SeedingData.cpp
+code/SeedingData/SeedWorker.cpp
+code/SeedingData/PathHandle.cpp
+code/SeedingData/GraphPath.cpp
+RayPlatform/RayPlatform/cryptography/crypto.cpp
+RayPlatform/RayPlatform/plugins/RegisteredPlugin.cpp
+RayPlatform/RayPlatform/plugins/CorePlugin.cpp
+RayPlatform/RayPlatform/actors/Playground.cpp
+RayPlatform/RayPlatform/actors/Actor.cpp
+RayPlatform/RayPlatform/profiling/Profiler.cpp
+RayPlatform/RayPlatform/profiling/Derivative.cpp
+RayPlatform/RayPlatform/profiling/ProcessStatus.cpp
+RayPlatform/RayPlatform/profiling/TickLogger.cpp
+RayPlatform/RayPlatform/profiling/TimePrinter.cpp
+RayPlatform/RayPlatform/store/KeyValueStore.cpp
+RayPlatform/RayPlatform/store/KeyValueStoreItem.cpp
+RayPlatform/RayPlatform/store/KeyValueStoreRequest.cpp
+RayPlatform/RayPlatform/memory/ReusableMemoryStore.cpp
+RayPlatform/RayPlatform/memory/MyAllocator.cpp
+RayPlatform/RayPlatform/memory/allocator.cpp
+RayPlatform/RayPlatform/memory/RingAllocator.cpp
+RayPlatform/RayPlatform/memory/ChunkAllocatorWithDefragmentation.cpp
+RayPlatform/RayPlatform/memory/DefragmentationLane.cpp
+RayPlatform/RayPlatform/memory/DirtyBuffer.cpp
+RayPlatform/RayPlatform/memory/DefragmentationGroup.cpp
+RayPlatform/RayPlatform/core/statistics.cpp
+RayPlatform/RayPlatform/core/ComputeCore.cpp
+RayPlatform/RayPlatform/core/MiniRank.cpp
+RayPlatform/RayPlatform/core/master_modes.cpp
+RayPlatform/RayPlatform/core/OperatingSystem.cpp
+RayPlatform/RayPlatform/core/slave_modes.cpp
+RayPlatform/RayPlatform/handlers/MessageTagExecutor.cpp
+RayPlatform/RayPlatform/handlers/MasterModeExecutor.cpp
+RayPlatform/RayPlatform/handlers/SlaveModeExecutor.cpp
+RayPlatform/RayPlatform/scheduling/TaskCreator.cpp
+RayPlatform/RayPlatform/scheduling/SwitchMan.cpp
+RayPlatform/RayPlatform/scheduling/VirtualProcessor.cpp
+RayPlatform/RayPlatform/files/FileReader.cpp
+RayPlatform/RayPlatform/structures/StaticVector.cpp
+RayPlatform/RayPlatform/communication/BufferedData.cpp
+RayPlatform/RayPlatform/communication/Message.cpp
+RayPlatform/RayPlatform/communication/MessageQueue.cpp
+RayPlatform/RayPlatform/communication/MessagesHandler.cpp
+RayPlatform/RayPlatform/communication/VirtualCommunicator.cpp
+RayPlatform/RayPlatform/communication/mpi_tags.cpp
+RayPlatform/RayPlatform/communication/MessageRouter.cpp
+RayPlatform/RayPlatform/routing/ConnectionGraph.cpp
+RayPlatform/RayPlatform/routing/Torus.cpp
+RayPlatform/RayPlatform/routing/Polytope.cpp
+RayPlatform/RayPlatform/routing/GraphImplementation.cpp
+RayPlatform/RayPlatform/routing/GraphImplementationKautz.cpp
+RayPlatform/RayPlatform/routing/GraphImplementationComplete.cpp
+RayPlatform/RayPlatform/routing/GraphImplementationExperimental.cpp
+RayPlatform/RayPlatform/routing/GraphImplementationRandom.cpp
+RayPlatform/RayPlatform/routing/GraphImplementationDeBruijn.cpp
+RayPlatform/RayPlatform/routing/GraphImplementationGroup.cpp
+
)
diff --git a/Documentation/454.txt b/Documentation/454.txt
old mode 100755
new mode 100644
diff --git a/Documentation/AMAZON_EC2.txt b/Documentation/AMAZON_EC2.txt
old mode 100755
new mode 100644
diff --git a/Documentation/Blue_Gene_Q.txt b/Documentation/Blue_Gene_Q.txt
new file mode 100644
index 0000000..2c7488a
--- /dev/null
+++ b/Documentation/Blue_Gene_Q.txt
@@ -0,0 +1,5 @@
+On the IBM Blue Gene/Q:
+
+module load mpich2
+make CXXFLAGS="-qminimaltoc -g -O3 -qarch=qp -qtune=qp "
+
diff --git a/Documentation/COMPILERS.txt b/Documentation/COMPILERS.txt
old mode 100755
new mode 100644
diff --git a/Documentation/CodingStyle.txt b/Documentation/CodingStyle.txt
index 99d1b5c..ac908c9 100644
--- a/Documentation/CodingStyle.txt
+++ b/Documentation/CodingStyle.txt
@@ -5,14 +5,15 @@
* With TAB symbols.
-
== New lines ==
* Only use '\n'
-
== Symbols ==
+* Class names are names, not verbs.
+* Method names are verbs, not names.
+
* Attribute members of a class are prefixed with 'm_'.
* No acronyms (except mpi).
@@ -23,4 +24,14 @@
* Message-passing interface tags are prefixed with 'RAY_MPI_TAG_'.
+== Headers ==
+
+Include the headers in this order:
+
+* plugin headers
+* other plugin headers
+* RayPlatform headers
+* C++ headers
+* C headers
+* POSIX headers
diff --git a/Documentation/Distributed-Storage-Engine.txt b/Documentation/Distributed-Storage-Engine.txt
index dcef420..5c1b0f0 100644
--- a/Documentation/Distributed-Storage-Engine.txt
+++ b/Documentation/Distributed-Storage-Engine.txt
@@ -1,5 +1,16 @@
-The distributed storage engine used by Ray is a distributed sparse hash table that
-uses these features:
+3 types of objects are distributed:
+
+- input sequence reads, the key is the ReadHandle. A read handle is a global
+ integer
+- graph vertices, the key is the DNA sequence (2-bit). hashing is used to find
+ the owner
+- paths. the key is a PathHandle. a path handle contains a rank and a local
+ identifier
+
+
+
+The distributed storage engine used by Ray is a distributed sparse hash table
+(from RayPlatform) that uses these features:
- incremental resizing
- double hashing
diff --git a/Documentation/Illumina.txt b/Documentation/Illumina.txt
old mode 100755
new mode 100644
diff --git a/Documentation/IonTorrent.txt b/Documentation/IonTorrent.txt
old mode 100755
new mode 100644
diff --git a/Documentation/Migration-to-the-Actor-Model.txt b/Documentation/Migration-to-the-Actor-Model.txt
new file mode 100644
index 0000000..181a8f5
--- /dev/null
+++ b/Documentation/Migration-to-the-Actor-Model.txt
@@ -0,0 +1,56 @@
+Roadmap for refactoring of Ray and RayPlatform to use the actor model.
+Sébastien Boisvert
+2013-09-26
+
+# Estimated time effort: 1 month full-time
+
+Link: http://en.wikipedia.org/wiki/Actor_model
+Prototype of the model in C++/MPI: https://github.com/sebhtml/BioActors
+Erlang example: https://github.com/sebhtml/erlang-tests/
+
+# 3 things that an actor can do when receiving a message:
+
+1. send messages to other actors (including possibly himself
+2. spawn new actors
+3. change its behavior for next message (via state in C++, in Erlang, this means
+that you recurse with different arguments to further the life of the actor)
+
+
+# Advantages over the current RayPlatform model:
+
+- slave mode calls are useless most of the time since things (like
+m_inbox->hasMessage) returns false probably 99% of the time.
+- Code modularity is achieved without ugly RayPlatform macros. Instead,
+it is achieved with Actor registering themselves.
+- With this model, There are N MPI ranks, and any number of actors. actors are mapped to
+a rank.
+
+
+# Principles for the migration
+
+- only messages are allowed (no more slave modes or master modes)
+- each plugin will effectively become an actor with an address
+- Each actor (old plugins) will receive directly its messages with receive
+- new actor method called send
+- The old plugin classes with inherit from ComputeCore and Actor
+
+
+# Roadmap
+
+
+1. remove calls to allocateSlaveModeHandle
+
+2. remove calls to allocateMasterModeHandle
+
+3. at this point, this satisfy rule 1. of the actor model, that is an actor can only react
+to messages. Each MPI rank is an actor. But we can not spawn new actors at this point.
+
+4. remove call to processData in RayPlatform
+
+5. Ship a Ray version with this !
+
+6. rename registerPlugin to Spawn actor.
+
+7. Actor model *replaces* mini-ranks.
+
+
diff --git a/Documentation/PacBio.txt b/Documentation/PacBio.txt
old mode 100755
new mode 100644
diff --git a/Documentation/Profiling.txt b/Documentation/Profiling.txt
index 022bc3b..0be128c 100644
--- a/Documentation/Profiling.txt
+++ b/Documentation/Profiling.txt
@@ -51,7 +51,7 @@ all communications (send and receive operations).
The option -run-profiler will run Ray in slow mode, but the supervisor
will collect a lot of profiling information. There is also a compile
time option to add collectors in the code. The option is
-CONFIG_PROFILER_COLLECT=y (or -D CONFIG_PROFILER_COLLECT).
+PROFILER_COLLECT=y (or -D CONFIG_PROFILER_COLLECT).
## Running time
diff --git a/Documentation/RELEASE_PROCEDURE.txt b/Documentation/RELEASE_PROCEDURE.txt
index 809dd4f..c424f43 100644
--- a/Documentation/RELEASE_PROCEDURE.txt
+++ b/Documentation/RELEASE_PROCEDURE.txt
@@ -1,13 +1,19 @@
* run regression tests (+backup on colosse)
* run unit tests
-* check version in Makefile
-* ./Ray -help > MANUAL_PAGE.txt
* compile with make
* compile with cmake
+
+* update release list (Ray & RayPlatform)
+* check version in Makefile
+* ./Ray -help > MANUAL_PAGE.txt
* git tag vx.y.z
* do a tar.bz2 (ShipProduct), compile it (make + make install) and test it
+
* upload the tarball on sf
* update the web site (download + manual)
+
* post an email on the list including changes (dump-changes)
* update the seqanswer thread
-* update release list (Ray & RayPlatform)
+* Google+
+* Twitter
+* Facebook
diff --git a/Documentation/Ray-Surveyor.md b/Documentation/Ray-Surveyor.md
new file mode 100644
index 0000000..75b62cd
--- /dev/null
+++ b/Documentation/Ray-Surveyor.md
@@ -0,0 +1,58 @@
+Ray Surveyor: compare genomic content between samples
+
+Usage:
+
+<pre>
+mpiexec -n 512 \
+Ray \
+-k 31 \
+-output RaySurveyorResults \
+-run-surveyor \
+-read-sample-graph ID033401 ./1996/ID033401-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID033406 ./1996/ID033406-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID033399 ./1996/ID033399-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID033398 ./1996/ID033398-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID119442 ./Patients/KID119442-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID119444 ./Patients/KID119444-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID119445 ./Patients/KID119445-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID119536 ./Patients/KID119536-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID119537 ./Patients/KID119537-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID119788 ./Patients/KID119788-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID119957 ./Patients/KID119957-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID119958 ./Patients/KID119958-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID119960 ./Patients/KID119960-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID120069 ./Patients/KID120069-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID120070 ./Patients/KID120070-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID120111 ./Patients/KID120111-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID120112 ./Patients/KID120112-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID120113 ./Patients/KID120113-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID120114 ./Patients/KID120114-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID120206 ./Patients/ID120206-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID120292 ./Patients/ID120292-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID120368 ./Patients/ID120368-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID120369 ./Patients/ID120369-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID120371 ./Patients/ID120371-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID120713 ./Patients/ID120713-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID120169 ./Contemporains/ID120169-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID119921 ./Contemporains/ID119921-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID120882 ./Contemporains/ID120882-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID119784 ./Contemporains/KID119784-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID120143 ./Environnement/KID120143-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID119787 ./Environnement/KID119787-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID120086 ./Environnement/KID120086-Ray-2013-10-07/kmers.txt \
+-read-sample-graph ID120329 ./Environnement/ID120329-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID120094 ./Environnement/KID120094-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID119685 ./Environnement/KID119685-Ray-2013-10-07/kmers.txt \
+-read-sample-graph KID120090 ./Environnement/KID120090-Ray-2013-10-07/kmers.txt \
+</pre>
+
+The files kmers.txt can be generated by Ray. Furthermore, there will probably be
+also an option called: -read-sample-reads <SampleDirectory>
+
+This will generate:
+
+RayOutput/Surveyor/SimilarityMatrix.tsv
+RayOutput/Surveyor/DistanceMatrix.tsv
+
+
+This file will contain pairwise similarity between graphs.
diff --git a/Documentation/Releases.txt b/Documentation/Releases.txt
index ff7ff72..a26a751 100644
--- a/Documentation/Releases.txt
+++ b/Documentation/Releases.txt
@@ -1,22 +1,32 @@
-Ray v2.1.0 "Ancient Granularity of Epochs" with RayPlatform v1.1.0 "Chariot of Complexity"
-Ray v2.0.0 "Dark Astrocyte of Knowledge" with RayPlatform v1.0.3 "Gray Pylon of Wisdom"
-Ray v1.7.0 "Magical Luminary of Celestial Bodies"
-Ray v1.6.1
-Ray v1.6.0
-Ray v1.4.0
-Ray v1.3.0
-Ray v1.2.3
-Ray v1.2.2
-Ray v1.2.1
-Ray v1.2.0
-Ray v1.0.0
-Ray v1.0.0
-Ray v0.1.0
-Ray v0.0.7
-Ray v0.0.6
-Ray v0.0.5
-Ray v0.0.4
-Ray v0.0.3
-Ray v0.0.2
-Ray v0.0.1
-Ray v0.0.0
+This file contains releases of Ray
+available at
+- http://sourceforge.net/projects/denovoassembler/files/
+- https://github.com/sebhtml/Ray-Releases
+
+2013-10-31 Ray v2.3.0 "Distributed Pumpkin" RayPlatform v2.0.0 "Actor Nest"
+2013-04-17 Ray v2.2.0 "Cloudy Sky of Wishes" RayPlatform v1.1.1 "Nested Droids of Luck"
+2012-10-30 Ray v2.1.0 "Ancient Granularity of Epochs" RayPlatform v1.1.0 "Chariot of Complexity"
+2012-06-24 Ray v2.0.0 "Dark Astrocyte of Knowledge" RayPlatform v1.0.3 "Gray Pylon of Wisdom"
+2012-05-29 Ray v2.0.0-rc8
+2012-05-17 Ray v2.0.0-rc7
+2012-03-28 Ray v2.0-rc5
+2011-10-03 Ray v1.7 "Magical Luminary of Celestial Bodies"
+2011-07-15 Ray v1.6.1 "sunray"
+2011-06-13 Ray v1.6.0
+2011-05-30 Ray v1.4.0
+2011-03-22 Ray v1.3.0
+2011-02-23 Ray v1.2.4
+2011-02-05 Ray v1.2.3 "cRAYfish"
+2011-01-11 Ray v1.2.2 "X-ray"
+2010-12-30 Ray v1.2.1 "stringray"
+2010-12-03 Ray v1.2.0
+2010-11-25 Ray v1.0.0
+2010-11-04 Ray v0.1.0
+2010-04-24 Ray v0.0.7
+2010-04-10 Ray v0.0.6
+2010-03-29 Ray v0.0.5
+2010-03-23 Ray v0.0.4
+2010-03-10 Ray v0.0.3
+ Ray v0.0.2
+ Ray v0.0.1 "Osiris"
+ Ray v0.0.0 "crocodile"
diff --git a/Documentation/Routing.txt b/Documentation/Routing.txt
index 7a09d9c..748bc64 100644
--- a/Documentation/Routing.txt
+++ b/Documentation/Routing.txt
@@ -2,7 +2,7 @@ For very large jobs, routing should be enabled.
Message routing reduces the latency and memory usage.
Also, the network will by happier.
-This guide contains network architecture for routing messages with Ray.
+This guide contains virtual network architectures for routing messages with Ray.
Vertices is the number of computer cores to use (mpiexec -n ###)
@@ -11,44 +11,77 @@ See also Documentation/Very-Large-Jobs.txt.
## Convex regular polytope
Messages can be routed with a polytope using
-the polytope connection type. A hypercube is a polytope with alphabetSize
+the polytope connection type. A hypercube is a polytope with radix
of 2. The connection types 'polytope' and 'hypercube' use the same code
path so they are equivalent.
-The basic relation for edges is that only 1 symbol can be different.
+The basic relation for edges is that only 1 dimension can be different.
Example: <0,1,2> -> <0,2,2>
For the convex regular polytope, we have
-vertices:= alphabetSize^wordLength
-degree:= (alphabetSize-1)*wordLength
+Vertices:= Radix^Dimension
+Degree:= (Radix-1)*Dimension
Vertices: 64
Type: Polytope
+Radix: 8
+Dimension: 2
Degree: 14
-Diameter: 2
Options: -route-messages -connection-type polytope -routing-graph-degree 14
Vertices: 512
Type: Polytope
+Radix: 8
+Dimension: 3 (8^3=512)
Degree: 21=(8-1)*3
-Diameter: 3 (8^3=512)
Options: -route-messages -connection-type polytope -routing-graph-degree 21
-
-Vertices: 1024
+Vertices: 1024 = 32*32
Type: Polytope
-Degree: 62=(32-1)*2
-Diameter: 2 (32^2=1024)
+Radix: 32
+Dimension: 2 (32^2=1024)
+Degree: 62 = (32-1)*2
Options: -route-messages -connection-type polytope -routing-graph-degree 62
Vertices: 1024
Type: Polytope
+Radix: 2
+Dimension: 10 (2^10=1024)
Degree: 10
-Diameter: 10 (2^10=1024)
Options: -route-messages -connection-type polytope -routing-graph-degree 10
+Vertices: 2025 (45 ^ 2)
+Type: Polytope
+Radix: 45
+Dimension: 2
+Degree: 88 ((45 - 1 ) * 2)
+Options: -route-messages -connection-type polytope -routing-graph-degree 88
+
+Vertices: 4225 = 65*65
+Type: Polytope
+Degree: 128 = (65-1)*2
+Diameter: 2 (65^2=1024)
+Options: -route-messages -connection-type polytope -routing-graph-degree 128
+
+
+## Torus
+
+Type: torus
+Radix: 8
+Dimension: 3
+Vertices: Radix^Dimension = 512
+Degree: 2*Dimension = 6
+Options: -route-messages -connection-type torus -routing-graph-degree 6
+
+Type: torus
+Radix: 4
+Dimension: 5
+Vertices: Radix^Dimension = 1024
+Degree: 2*Dimension = 10
+Options: -route-messages -connection-type torus -routing-graph-degree 10
+
## de Bruijn
diff --git a/Documentation/SOLiD.txt b/Documentation/SOLiD.txt
old mode 100755
new mode 100644
diff --git a/Documentation/VISUAL_STUDIO.txt b/Documentation/VISUAL_STUDIO.txt
old mode 100755
new mode 100644
diff --git a/Documentation/Very-Large-Jobs.txt b/Documentation/Very-Large-Jobs.txt
index eab88fc..f723c10 100644
--- a/Documentation/Very-Large-Jobs.txt
+++ b/Documentation/Very-Large-Jobs.txt
@@ -1,7 +1,8 @@
-For very large jobs, routing should be enabled.
-Message routing reduces the latency.
+For very large jobs, routing should be enabled unless you are using a good
+interconnect.
+Message routing reduces the latency for bad interconnects.
mpiexec -n 512 Ray \
diff --git a/Documentation/index.markdown b/Documentation/index.markdown
new file mode 100644
index 0000000..99bc1ab
--- /dev/null
+++ b/Documentation/index.markdown
@@ -0,0 +1,3 @@
+---
+title: Welcome to the Ray documentation !
+---
diff --git a/MANUAL_PAGE.txt b/MANUAL_PAGE.txt
index 3989eba..0a2afac 100644
--- a/MANUAL_PAGE.txt
+++ b/MANUAL_PAGE.txt
@@ -2,9 +2,13 @@ NAME
Ray - assemble genomes in parallel using the message-passing interface
SYNOPSIS
- mpiexec -n NUMBER_OF_RANKS Ray -k KMERLENGTH -p l1_1.fastq l1_2.fastq -p l2_1.fastq l2_2.fastq -o test
+ mpiexec -n 80 Ray -k 31 -p l1_1.fastq l1_2.fastq -p l2_1.fastq l2_2.fastq -o test
- mpiexec -n NUMBER_OF_RANKS Ray Ray.conf # with commands in a file
+ mpiexec -n 80 Ray Ray.conf # with commands in a file
+
+ mpiexec -n 80 Ray -k 31 -detect-sequence-files SampleDirectory # auto-detection
+
+ mpiexec -n 10 Ray -mini-ranks-per-rank 7 Ray.conf # with mini-ranks
DESCRIPTION:
@@ -29,6 +33,19 @@ DESCRIPTION:
-version
Displays Ray version and compilation options.
+ Run Ray in pure MPI mode
+
+ mpiexec -n 80 Ray ...
+
+ Run Ray with mini-ranks on 10 machines, 8 cores / machine (MPI and IEEE POSIX threads)
+
+ mpiexec -n 10 Ray -mini-ranks-per-rank 7 ...
+
+ Run Ray on one core only (still needs MPI)
+
+ Ray ...
+
+
Using a configuration file
Ray can be launched with
@@ -40,14 +57,19 @@ DESCRIPTION:
-k kmerLength
Selects the length of k-mers. The default value is 21.
It must be odd because reverse-complement vertices are stored together.
- The maximum length is defined at compilation by MAXKMERLENGTH
+ The maximum length is defined at compilation by CONFIG_MAXKMERLENGTH
Larger k-mers utilise more memory.
Inputs
+ -detect-sequence-files SampleDirectory
+ Detects files in a directory automatically.
+ This option can generate these commands automatically for you: LoadPairedEndReads (-p) and LoadSingleEndReads (-s)
+
-p leftSequenceFile rightSequenceFile [averageOuterDistance standardDeviation]
Provides two files containing paired-end reads.
averageOuterDistance and standardDeviation are automatically computed if not provided.
+ LoadPairedEndReads is equivalent to -p
-i interleavedSequenceFile [averageOuterDistance standardDeviation]
Provides one file containing interleaved paired-end reads.
@@ -55,11 +77,13 @@ DESCRIPTION:
-s sequenceFile
Provides a file containing single-end reads.
+ LoadSingleEndReads is equivalent to -s
Outputs
-o outputDirectory
Specifies the directory for outputted files. Default is RayOutput
+ Other name: -output
Assembly options (defaults work well)
@@ -73,6 +97,15 @@ DESCRIPTION:
Sébastien Boisvert, Élénie Godzaridis, François Laviolette & Jacques Corbeil.
First Annual RECOMB Satellite Workshop on Massively Parallel Sequencing, March 26-27 2011, Vancouver, BC, Canada.
+ -debug-recycling
+ Debugs the recycling events
+
+ -ignore-seeds
+ Disables assembly by ignoring seeds.
+
+ -merge-seeds
+ Merges seeds initially to reduce running time.
+
-disable-scaffolder
Disables the scaffolder.
@@ -95,7 +128,7 @@ DESCRIPTION:
-bloom-filter-bits bits
Sets the number of bits for the Bloom filter
- Default is 268435456 bits, 0 bits disables the Bloom filter.
+ Default is auto bits (adaptive), 0 bits disables the Bloom filter.
-hash-table-buckets buckets
Sets the initial number of buckets. Must be a power of 2 !
@@ -215,12 +248,13 @@ DESCRIPTION:
-connection-type type
Sets the connection type for routes.
Accepted values are debruijn, hypercube, polytope, group, random, kautz and complete. Default is debruijn.
- debruijn: a full de Bruijn graph a given alphabet and diameter
- hypercube: a hypercube, alphabet is {0,1} and the vertices is a power of 2
+ torus: a k-ary n-cube, radix: k, dimension: n, degree: 2*dimension, vertices: radix^dimension
polytope: a convex regular polytope, alphabet is {0,1,...,B-1} and the vertices is a power of B
+ hypercube: a hypercube, alphabet is {0,1} and the vertices is a power of 2
+ debruijn: a full de Bruijn graph a given alphabet and diameter
+ kautz: a full de Kautz graph, which is a subgraph of a de Bruijn graph
group: silly model where one representative per group can communicate with outsiders
random: Erdős–Rényi model
- kautz: a full de Kautz graph, which is a subgraph of a de Bruijn graph
complete: a full graph with all the possible connections
With the type debruijn, the number of ranks must be a power of something.
Examples: 256 = 16*16, 512=8*8*8, 49=7*7, and so on.
@@ -251,6 +285,12 @@ DESCRIPTION:
Checks message data reliability for any non-empty message.
add '-D CONFIG_SSE_4_2' in the Makefile to use hardware instruction (SSE 4.2)
+ -write-scheduling-data
+ Writes RayPlatform scheduling information to RayOutput/Scheduling/
+
+ -write-plugin-data
+ Writes data for plugins registered with the RayPlatform API to RayOutput/Plugins
+
-run-profiler
Runs the profiler as the code runs. By default, only show granularity warnings.
Running the profiler increases running times.
@@ -258,6 +298,9 @@ DESCRIPTION:
-with-profiler-details
Shows number of messages sent and received in each methods during in each time slices (epochs). Needs -run-profiler.
+ -debug
+ Turns on -run-profiler and -with-profiler-details for debugging
+
-show-communication-events
Shows all messages sent and received.
@@ -286,13 +329,22 @@ FILES
Note: file format is determined with file extension.
.fasta
+ .fa
.fasta.gz (needs HAVE_LIBZ=y at compilation)
+ .fa.gz (needs HAVE_LIBZ=y at compilation)
.fasta.bz2 (needs HAVE_LIBBZ2=y at compilation)
+ .fa.bz2 (needs HAVE_LIBBZ2=y at compilation)
.fastq
+ .fq
.fastq.gz (needs HAVE_LIBZ=y at compilation)
+ .fq.gz (needs HAVE_LIBZ=y at compilation)
.fastq.bz2 (needs HAVE_LIBBZ2=y at compilation)
+ .fq.bz2 (needs HAVE_LIBBZ2=y at compilation)
+ export.txt
+ qseq.txt
.sff (paired reads must be extracted manually)
.csfasta (color-space reads)
+ .csfa (color-space reads)
Outputted files
@@ -349,7 +401,7 @@ FILES
RayOutput/LibraryStatistics.txt
Estimation of outer distances for paired reads
- RayOutput/Library<LibraryNumber>.txt
+ RayOutput/LibraryData.xml
Frequencies for observed outer distances (insert size + read lengths)
Partition
@@ -362,9 +414,11 @@ FILES
Ray software
RayOutput/RayVersion.txt
- The version of Ray
+ The version of Ray
RayOutput/RayCommand.txt
- The exact same command provided
+ The exact same command provided
+ RayOutput/RaySmartCommand.txt
+ The smart command generated by Ray
AMOS
@@ -373,8 +427,6 @@ FILES
Communication
- RayOutput/MessagePassingInterface.txt
- Number of messages sent
RayOutput/NetworkTest.txt
Latencies in microseconds
RayOutput/Rank<rank>NetworkTestData.txt
@@ -408,4 +460,4 @@ COPYRIGHT
You have received a copy of the GNU General Public License
along with this program (see LICENSE).
-Ray 2.1.0
+Ray 2.3.0
diff --git a/Makefile b/Makefile
index b6a8dad..1703af9 100644
--- a/Makefile
+++ b/Makefile
@@ -1,20 +1,45 @@
VERSION = 2
-PATCHLEVEL = 1
+PATCHLEVEL = 3
SUBLEVEL = 0
EXTRAVERSION =
-NAME = Ancient Granularity of Epochs
+NAME = Distributed Pumpkin
-# number of cores to use for compilation
-J=1
-
-RAY_VERSION = $(VERSION).$(PATCHLEVEL).$(SUBLEVEL)$(EXTRAVERSION)
+# Like in Linux, stuff that must be propagated to
+# the source code (C++ here) or other Makefiles
+# are prefixed with CONFIG_
# author: Sébastien Boisvert
# Makefile for the ray assembler
# Objects appended to obj-y are compiled and linked.
# Objects appended to obj-n are not compiled and linked.
-#############################################
-# compilation options
+#
+# Based on http://www.ravnborg.org/kbuild/makefiles.html
+#
+# The code is distributed in a small Ray core that is built on top
+# of the RayPlatform.
+#
+# Then, plugins (interface CorePlugin in the RayPlatform)
+# are simply added onto the core (class ComputeCore in the
+# RayPlatform).
+
+# this can be changed with make MPICXX=...
+MPICXX = mpicxx
+
+# CXXFLAGS can be changed by the end user with make CXXFLAGS="..."
+CXXFLAGS = -O3 -std=c++98 -Wall -g
+
+RM = rm
+CD = cd
+MAKE = make
+ECHO = echo
+MKDIR = mkdir
+CP = cp
+
+CONFIG_RAY_VERSION = $(VERSION).$(PATCHLEVEL).$(SUBLEVEL)$(EXTRAVERSION)
+
+#######################################################################
+# Compilation options
+#######################################################################
# installation prefix with make install
PREFIX=install-prefix
@@ -41,12 +66,6 @@ HAVE_LIBZ = n
# y/n
HAVE_LIBBZ2 = n
-# use Intel's compiler
-# the name of the Intel MPI C++ compiler is mpiicpc
-# Open-MPI and MPICH2 utilise mpic++ for the name.
-# y/n
-INTEL_COMPILER = n
-
# pack structures to reduce memory usage
# will work on x86 and x86_64
# won't work on Itanium and on Sparc
@@ -66,143 +85,146 @@ ASSERT = n
# collect profiling information with -run-profiler
# if set to n, the code is not even compiled in
-CONFIG_PROFILER_COLLECT=n
+PROFILER_COLLECT=n
# use the precision clock
# needs -l rt too
-CONFIG_CLOCK_GETTIME=n
-
-# OS detection based on git Makefile
-uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
-
-# optimize
-OPTIMIZE = y
-
-# add -g to gcc
-# see "Is there a downside to leaving in debug symbols in release builds?"
-# http://stackoverflow.com/questions/5569644/is-there-a-downside-to-leaving-in-debug-symbols-in-release-builds
-# in short: the executable is larger, but symbols are in a different section (thus the code is not slower)
-DEBUG = n
+CLOCK_GETTIME=n
-# profiling
-GPROF = n
+# use MPI I/O for file operations
-ifeq ($(GPROF),y)
- OPTIMIZE = n
- FORCE_PACKING = n
-endif
+MPI_IO=n
-PEDANTIC = n
+#######################################################################
+# Don't edit below this this point ---> .
+#######################################################################
-MPICXX-y = mpicxx
+# Build configuration options correctly for the source code
-# mpic++ from an MPI implementation must be reachable with the PATH
-# tested implementations of MPI include Open-MPI and MPICH2
-CXXFLAGS =
+CONFIG_ASSERT=$(ASSERT)
+CONFIG_HAVE_LIBZ=$(HAVE_LIBZ)
+CONFIG_HAVE_LIBBZ2=$(HAVE_LIBBZ2)
+CONFIG_FORCE_PACKING=$(FORCE_PACKING)
+CONFIG_PROFILER_COLLECT=$(PROFILER_COLLECT)
+CONFIG_CLOCK_GETTIME=$(CLOCK_GETTIME)
+CONFIG_MPI_IO=$(MPI_IO)
-# optimization
-CXXFLAGS-$(OPTIMIZE) += -O3
+# These 2 are used by an other Makefile
+export CONFIG_HAVE_LIBZ
+export CONFIG_HAVE_LIBBZ2
-ifeq ($(INTEL_COMPILER),n)
-# g++ options
-ifeq ($(uname_S),Linux)
- #-std=c++98
- CXXFLAGS += -Wall -std=c++98
- CXXFLAGS-$(PEDANTIC) += -pedantic -Wextra
-endif
-endif
+#######################################################################
-# profiling
-CXXFLAGS-$(GPROF) += -g -pg
-LDFLAGS-$(GPROF) += -pg -g
-
-# if you use Intel's mpiicpc, uncomment the following lines
-MPICXX-$(INTEL_COMPILER) = mpiicpc
-CXXFLAGS-$(INTEL_COMPILER) += -DMPICH_IGNORE_CXX_SEEK -DMPICH_SKIP_MPICXX
+# Build the CONFIG_FLAGS
+# This could be stored in a config.h too.
+CONFIG_FLAGS-y=
#maximum k-mer length
-CXXFLAGS-y += -D MAXKMERLENGTH=$(MAXKMERLENGTH)
-CXXFLAGS-y += $(EXTRA)
-LDFLAGS-y += $(EXTRA)
+CONFIG_FLAGS-y += -D CONFIG_MAXKMERLENGTH=$(MAXKMERLENGTH)
# compile assertions
-CXXFLAGS-$(ASSERT) += -DASSERT
+CONFIG_FLAGS-$(CONFIG_ASSERT) += -D CONFIG_ASSERT -D ASSERT
-#compile with libz
-CXXFLAGS-$(HAVE_LIBZ) += -DHAVE_LIBZ
-LDFLAGS-$(HAVE_LIBZ) += -lz
+#compile with zlib
+CONFIG_FLAGS-$(CONFIG_HAVE_LIBZ) += -D CONFIG_HAVE_LIBZ
+LDFLAGS-$(CONFIG_HAVE_LIBZ) += -lz
#compile with libbz2
-CXXFLAGS-$(HAVE_LIBBZ2) += -DHAVE_LIBBZ2
-LDFLAGS-$(HAVE_LIBBZ2) += -lbz2
-
-#debug flag
-CXXFLAGS-$(DEBUG) += -g
-LDFLAGS-$(DEBUG) += -g
+CONFIG_FLAGS-$(CONFIG_HAVE_LIBBZ2) += -D CONFIG_HAVE_LIBBZ2
+LDFLAGS-$(CONFIG_HAVE_LIBBZ2) += -lbz2
# pack data in memory to save space
-CXXFLAGS-$(FORCE_PACKING) += -DFORCE_PACKING
+CONFIG_FLAGS-$(CONFIG_FORCE_PACKING) += -D CONFIG_FORCE_PACKING
-CXXFLAGS-$(CONFIG_PROFILER_COLLECT) += -D CONFIG_PROFILER_COLLECT
-CXXFLAGS-$(CONFIG_CLOCK_GETTIME) += -D CONFIG_CLOCK_GETTIME
+# use MPI I/O
+CONFIG_FLAGS-$(CONFIG_MPI_IO) += -D CONFIG_MPI_IO
+CONFIG_FLAGS-$(CONFIG_PROFILER_COLLECT) += -D CONFIG_PROFILER_COLLECT
+CONFIG_FLAGS-$(CONFIG_CLOCK_GETTIME) += -D CONFIG_CLOCK_GETTIME
LDFLAGS-$(CONFIG_CLOCK_GETTIME) += -l rt
-CXXFLAGS-y += -D RAY_VERSION=\\\"$(RAY_VERSION)\\\"
-
-CXXFLAGS += $(CXXFLAGS-y)
-LDFLAGS += $(LDFLAGS-y)
-
-MPICXX = $(MPICXX-y)
+CONFIG_FLAGS-y += -D CONFIG_RAY_VERSION=\"$(CONFIG_RAY_VERSION)\"
-# object files
-
-#################################
+# CONFIG_FLAGS is separate from CXXFLAGS
+# This eases building the package in distributions
+LDFLAGS = $(LDFLAGS-y)
+CONFIG_FLAGS=$(CONFIG_FLAGS-y)
+Q=@
+#######################################################################
+# Build rules.
# the target is Ray
all: Ray
-showOptions:
- @echo ""
- @echo "Compilation options (you can change them of course)"
- @echo ""
- @echo PREFIX = $(PREFIX)
- @echo MAXKMERLENGTH = $(MAXKMERLENGTH)
- @echo FORCE_PACKING = $(FORCE_PACKING)
- @echo ASSERT = $(ASSERT)
- @echo HAVE_LIBZ = $(HAVE_LIBZ)
- @echo HAVE_LIBBZ2 = $(HAVE_LIBBZ2)
- @echo INTEL_COMPILER = $(INTEL_COMPILER)
- @echo MPICXX = $(MPICXX)
- @echo GPROF = $(GPROF)
- @echo OPTIMIZE = $(OPTIMIZE)
- @echo DEBUG = $(DEBUG)
- @echo ""
- @echo "Compilation and linking flags (generated automatically)"
- @echo ""
- @echo CXXFLAGS = $(CXXFLAGS)
- @echo LDFLAGS = $(LDFLAGS)
- @echo ""
- @touch showOptions
+# inference rule
+%.o: %.cpp
+ $(Q)$(ECHO) " CXX $@"
+ $(Q)$(MPICXX) $(CXXFLAGS) $(CONFIG_FLAGS) -I. -IRayPlatform -c $< -o $@
+
+include code/*/Makefile
+
+showOptions:
+ $(Q)echo ""
+ $(Q)echo "Compilation options (you can change them of course)"
+ $(Q)echo ""
+ $(Q)echo PREFIX = $(PREFIX)
+ $(Q)echo MPICXX = $(MPICXX)
+ $(Q)echo MAXKMERLENGTH = $(MAXKMERLENGTH)
+ $(Q)echo FORCE_PACKING = $(FORCE_PACKING)
+ $(Q)echo ASSERT = $(ASSERT)
+ $(Q)echo HAVE_LIBZ = $(HAVE_LIBZ)
+ $(Q)echo HAVE_LIBBZ2 = $(HAVE_LIBBZ2)
+ $(Q)echo ""
+ $(Q)echo "Compilation and linking flags (generated automatically)"
+ $(Q)echo ""
+ $(Q)echo CXXFLAGS = $(CXXFLAGS)
+ $(Q)echo CONFIG_FLAGS = $(CONFIG_FLAGS)
+ $(Q)echo LDFLAGS = $(LDFLAGS)
+ $(Q)echo ""
+ $(Q)touch showOptions
# how to make Ray
-Ray: showOptions RayPlatform/libRayPlatform.a code/TheRayGenomeAssembler.a
- $(MPICXX) $(LDFLAGS) code/TheRayGenomeAssembler.a RayPlatform/libRayPlatform.a -o $@
- @echo $(PREFIX) > PREFIX
- @echo Ray > TARGETS
+Ray: code/application_core/ray_main.o libRay.a libRayPlatform.a
+ $(Q)$(ECHO) " LD $@"
+ $(Q)$(MPICXX) $^ -o$@ $(LDFLAGS)
+ $(Q)$(ECHO) $(PREFIX) > PREFIX
+
+libRay.a: $(obj-y)
+ $(Q)$(ECHO) " AR $@"
+ $(Q)$(AR) rcs $@ $^
-code/TheRayGenomeAssembler.a:
- @cd code; make MPICXX="$(MPICXX)" CXXFLAGS="$(CXXFLAGS)" -j $(J) all ; cd ..
+libRayPlatform.a: RayPlatform/libRayPlatform.a
+ $(Q)$(CP) $^ $@
RayPlatform/libRayPlatform.a:
- @cd RayPlatform; make MPICXX="$(MPICXX)" CXXFLAGS="$(CXXFLAGS)" -j $(J) ; cd ..
+ $(Q)$(MAKE) $(MFLAGS) -C RayPlatform
clean:
- @rm -f Ray showOptions PREFIX TARGETS
- @echo "Cleaning Ray Application"
- @(cd code; make clean; cd ..)
- @echo "Cleaning Ray Platform"
- @cd RayPlatform;make clean; cd ..
- @echo CLEAN
-
-install:
- @scripts/install.sh
+ $(Q)$(MAKE) $(MFLAGS) -C RayPlatform clean
+ $(Q)$(ECHO) CLEAN Ray plugins
+ $(Q)$(RM) -f Ray PREFIX $(obj-y) libRay.a libRayPlatform.a code/application_core/ray_main.o
+
+install:
+ $(eval PREFIX=$(shell cat PREFIX))
+
+ $(Q)$(MKDIR) -p $(PREFIX)
+
+ $(Q)$(ECHO) ""
+ $(Q)$(ECHO) "Installing Ray to $(PREFIX)"
+ $(Q)$(ECHO) ""
+
+ $(Q)cp LICENSE.txt $(PREFIX)
+ $(Q)cp gpl-3.0.txt $(PREFIX)
+ $(Q)cp RayPlatform/lgpl-3.0.txt $(PREFIX)
+
+ $(Q)cp Ray $(PREFIX)
+ $(Q)cp -r Documentation $(PREFIX)
+ $(Q)cp README.md $(PREFIX)
+ $(Q)cp MANUAL_PAGE.txt $(PREFIX)
+ $(Q)cp AUTHORS $(PREFIX)
+ $(Q)cp -r scripts $PREFIX
+
+ $(Q)mkdir $(PREFIX)/RayPlatform
+ $(Q)cp RayPlatform/AUTHORS $(PREFIX)/RayPlatform
+ $(Q)cp RayPlatform/README $(PREFIX)/RayPlatform
+ $(Q)cp -r RayPlatform/Documentation $(PREFIX)/RayPlatform
+
diff --git a/README.md b/README.md
index 02a04bc..b8276a1 100644
--- a/README.md
+++ b/README.md
@@ -14,17 +14,48 @@ Ray is documented in
## Solutions (all bundled in a single Product called Ray)
-- de novo genome assembly
-- de novo meta-genome assembly (with Ray Méta)
+Standard:
+
+- de novo genome assembly (works by default)
+ => http://online.liebertpub.com/doi/abs/10.1089/cmb.2009.0238
+ => Documentation/README-heuristics
+- quantification of contig abundances (works by default)
+
+Metagenomics:
+
+- Ray Meta: de novo metagenome assembly (works by default)
+ => http://genomebiology.com/2012/13/12/R122
+- Ray Communities: quantification of microbiome consortia members (with Ray Communities with -search)
+ => Documentation/BiologicalAbundances.txt
+- Ray Communities: taxonomy profiling of samples (with -search and -with-taxonomy)
+ => Documentation/Taxonomy.txt
+- Ray Ontology: gene ontology profiling of samples (with -search and -gene-ontology)
+ => Documentation/GeneOntology.txt
+- Ray Surveyor: compare genomic content between samples (with -run-surveyor)
+ => Documentation/Ray-Surveyor.md
+
+Transcriptomics:
+
- de novo transcriptome assembly (works, but not tested a lot)
-- quantification of contig abundances
-- quantification of microbiome consortia members (with Ray Communities)
- quantification of transcript expression
-- taxonomy profiling of samples (with Ray Communities)
-- gene ontology profiling of samples (with Ray Ontologies)
-see Documentation/BiologicalAbundances.txt
+
+# Distributors
+
+- Geeknet, Inc. http://sourceforge.net/projects/denovoassembler/
+- GitHub, Inc. https://github.com/sebhtml/ray
+- Debian (Software in the Public Interest, Inc.) http://packages.debian.org/sid/ray
+- Ubuntu (Canonical Ltd.) https://launchpad.net/ubuntu/+source/ray
+- archlinux (AUR Development Team.) https://aur.archlinux.org/packages/ray/
+- DNAnexus, Inc. https://platform.dnanexus.com/app/ray
+- CloudBioLinux https://github.com/chapmanb/cloudbiolinux/blob/master/cloudbio/custom/bio_nextgen.py
+
+In progress:
+
+- Fedora (Red Hat, Inc.) https://bugzilla.redhat.com/show_bug.cgi?id=872783 (in progress)
+- Galaxy (Galaxy Team) http://user.list.galaxyproject.org/How-do-I-add-Ray-to-Galaxy-Central-in-the-tool-shed-td4655623.html#none
+- BaseSpace (Illumina, Inc.)
# Website
@@ -48,8 +79,6 @@ the code changes.
- http://github.com/sebhtml/ray-logo (artworks)
-- http://github.com/sebhtml/RayKmerSearchDevel (development scripts)
-
# Mailing lists
- Users: denovoassembler-users AT lists.sourceforge.net
@@ -88,6 +117,11 @@ You need a C++ compiler (supporting C++ 1998), make, an implementation of MPI (s
Tested C++ compilers: see Documentation/COMPILERS.txt
+## Parallel I/O
+
+To compile with MPI I/O, use this:
+
+ make MPI_IO=y
## Faster execution
@@ -98,7 +132,7 @@ the compilation.
make PREFIX=Build.native DEBUG=n ASSERT=n EXTRA=" -march=native"
make install
-The best way to build Ray is to use whole-program optimization.
+Another way to build Ray is to use whole-program optimization.
With gcc, use this script:
./scripts/Build-Link-Time-Optimization.sh
diff --git a/RayPlatform/Documentation/Actor-Model.txt b/RayPlatform/Documentation/Actor-Model.txt
new file mode 100644
index 0000000..8cf7895
--- /dev/null
+++ b/RayPlatform/Documentation/Actor-Model.txt
@@ -0,0 +1,13 @@
+With N ranks, actors numbered from 0 to N - 1 are the ranks.
+Actors can send and receive messages from them using this.
+
+Implemented:
+
+- Actor::receive
+- Actor::send
+- Actor::spawn
+- routing works
+- Actor::die
+
+To implement
+
diff --git a/RayPlatform/Documentation/DebugMode.txt b/RayPlatform/Documentation/DebugMode.txt
new file mode 100644
index 0000000..60c8a08
--- /dev/null
+++ b/RayPlatform/Documentation/DebugMode.txt
@@ -0,0 +1,18 @@
+The debug mode if RayPlatform can be enabled by calling
+ComputeCore::enableProfiler().
+
+For instance, Ray is a software that uses RayPlatform and its option
+-debug does just that.
+
+Furthermore, the RayPlatform debug mode can be activated in the middle
+of a computation by sending SIGUSR1 to processes that should activate
+debug mode. Sending SIGUSR1 again will disable the debug mode. Basically,
+Sending SIGUSR1 toggles the debug mode.
+
+In the future, sending SIGUSR1 to the RayPlatform core will cause it
+to read (or attempt to) a command file named Commands.txt in the
+current directory (PWD).
+
+See also Documentation/profiling.txt
+
+
diff --git a/RayPlatform/Documentation/Gates.txt b/RayPlatform/Documentation/Gates.txt
new file mode 100644
index 0000000..2df6b87
--- /dev/null
+++ b/RayPlatform/Documentation/Gates.txt
@@ -0,0 +1,41 @@
+Gates for non-linear scheduling
+2012-12-04
+
+
+Currently, SwitchMan only allows linear scheduling of master modes.
+
+o---->o---->o---->o---->o---->o---->o---->o
+
+
+However, it would be more general to allow non-linear scheduling like this:
+
+
+o---->o---->o---->o---->o---->o---->o---->o
+ \
+ \
+ \
+ ->o---->o---->o---->o---->o---->o
+
+
+Or like this:
+
+
+o---->o---->o---->o---->o---->o---->o---->o
+ \ /
+ \ /
+ \ /
+ ->o---->o---->o---->o---->o---->o
+
+
+The current way of setting the next master mode for a current master mode is:
+
+ void ComputeCore::setMasterModeNextMasterMode(PluginHandle plugin,MasterMode mode,MasterMode next);
+
+So the new prototype is still:
+
+ void CompureCore::setMasterModeNextMasterMode(PluginHandle plugin,MasterMode mode,MasterMode next);
+
+But there is this new method (called somewhere where the decision is made):
+
+ void CompureCore::setSelectNextMasterMode(PluginHandle plugin,MasterMode masterMode, MasterMode selectedNextMasterMode);
+
diff --git a/RayPlatform/Documentation/Handlers.txt b/RayPlatform/Documentation/Handlers.txt
index 62fe5b6..f34ff37 100644
--- a/RayPlatform/Documentation/Handlers.txt
+++ b/RayPlatform/Documentation/Handlers.txt
@@ -38,23 +38,9 @@ There are 3 handler classes:
- SlaveModeHandler
-This makes Ray (or any application using the Ray platform) modular.
+This makes Ray (or any application using the RayPlatform) modular.
== MACROS ==
-__CreatePlugin
- declares a static pointer to a plugin
-
-__CreateMasterModeAdapter
- generates an implementation for an adapter
-
-__CreateSlaveModeAdapter
- generates an implementation for an adapter
-
-__CreateMessageTagAdapter
- generates an implementation for an adapter
-
-__BindPlugin
- sets the value of the static pointer to the plugin
-
+see plugins.txt
diff --git a/RayPlatform/Documentation/MiniRanks.txt b/RayPlatform/Documentation/MiniRanks.txt
new file mode 100644
index 0000000..aae2f5f
--- /dev/null
+++ b/RayPlatform/Documentation/MiniRanks.txt
@@ -0,0 +1,329 @@
+Mini-ranks
+Sébastien Boisvert
+2012-10-17, Argonne National Laboratory
+
+Disclaimer: this document contains ideas and blueprints, but the true design
+is in the source code with corresponding comments therein.
+
+2012-10-17 10:00 UTC/GMT-5:
+
+This idea was discussed during a meeting with Professor Rick Stevens.
+
+The concept of mini-rank in electronics was introduced in 2008 in the paper
+
+"Mini-rank: Adaptive DRAM architecture for improving memory power efficiency"
+http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.182.4308&rep=rep1&type=pdf
+
+
+Here, we use the concept in the context of MPI ranks, not memory ranks.
+
+The processing machinery of a computer is hierarchical with
+this progression:
+
+ socket -> processor -> core -> thread
+
+The MPI programming model only gives MPI ranks that send messages to each
+other. Usually a MPI Rank is mapped to a single process. And a process runs
+on a single core.
+
+Figure 1: The MPI programming model.
+
+ +--------------------+
+ | MPI_COMM_WORLD | MPI communicator
+ +---------+----------+
+ |
+ +------+------+---+--+------+------+
+ | | | | | |
+ +---+ +---+ +---+ +---+ +---+ +---+
+ | 0 | | 1 | | 2 | | 3 | | 4 | | 5 | MPI ranks
+ +---+ +---+ +---+ +---+ +---+ +---+
+
+
+Mini ranks can be thought as ranks within ranks.
+
+
+Figure 2: The MPI programming model, with mini ranks.
+
+ +--------------------+
+ | MPI_COMM_WORLD | MPI communicator
+ +---------+----------+
+ |
+ +------+------+---+--+------+------+
+ | | | | | |
+ +---+ +---+ +---+ +---+ +---+ +---+
+ | 0 | | 1 | | 2 | | 3 | | 4 | | 5 | MPI ranks (1 RankProcess.cpp instance per rank)
+ +---+ +---+ +---+ +---+ +---+ +---+ with the thread from main() for MPI calls
+ | | | | | | | | | | | |
+ | 0 | | 4 | | 8 | |12 | |16 | |20 | |
+ | 1 | | 5 | | 9 | |13 | |17 | |21 | | => mini-ranks
+ | 2 | | 6 | |10 | |14 | |18 | |22 | |
+ | 3 | | 7 | |11 | |15 | |19 | |23 | | (1 MiniRank.cpp instance per minirank (in 1 pthread))
+ | | | | | | | | | | | | | (1 Application running with its RayPlatform ComputeCore.cpp)
+ +---+ +---+ +---+ +---+ +---+ +---+
+
+Current programming model:
+
+Each MPI rank runs one
+
+class Machine.cpp (in Ray)
+this class create application plugins and starts the main loop
+
+the main loop is in RayPlatform.
+
+
+
+
+Future software design:
+
+
+class VirtualMachine.cpp (runs as 1 MPI rank)
+
+ (1 pthread doing the MPI calls)
+ - picks up stuff from mini rank outbox
+ - copy these buffers into another one that will be used by
+ Mailman
+ - deposit incoming messages into the inbox of miniranks
+
+
+ class Machine.cpp (running a mini rank in 1 pthread)
+ plugins see only a inbox and a outbox
+
+ buffers of mini ranks are pristine, they can not be dirty
+ they can also call barrier on RayPlatform
+
+ class ComputeCore.cpp (with the main loop)
+
+ - add locks on inbox and outbox.
+
+
+As of Ray v2.0.0, each MPI rank is running as a Machine.cpp
+
+Machine.cpp is not in RayPlatform however.
+
+
+## Adapters
+
+For adapters, the current implementation is that each plugin is a singleton.
+This must be changed. Instead, each static variable in a plugin must be an
+array of plugin.
+
+Then, the callbacks will take an extra parameter: the index of the minirank
+within the rank.
+
+The current code was reverted to the virtual methods.
+
+
+## Message reception
+
+Designed in collaboration with Fangfang Xia
+
+1. rank receives message with MPI
+2. rank locks the corresponding minirank inbox
+3. rank push the message to the minirank inbox
+4. rank unlocks the minirank inbox
+5. The minirank locks its inbox
+6. The minirank consumes its message
+7. The minirank unlocks its inbox
+
+## Sending a message
+
+Designed in collaboration with Fangfang Xia
+
+1. The minirank puts a message in its outbox
+2. The minirank locks the rank outbox
+3. The minirank pushes onto the rank outbox
+4. The minirank unlocks the rank outbox
+5. The rank locks its outbox
+6. The rank pushes messages onto the network
+7. The rank unlocks its outbox
+
+
+
+1 important aspect is that the rank has no buffering system at all.
+All the buffering systems are in the miniranks. For MPI_Recv, the virtual
+machine uses the buffer from the corresponding minirank.
+For MPI_Isend, the virtual machine uses the buffer from the minirank.
+The dirty/cleaning subsystem is local to each minirank.
+
+
+For a system with 8 nodes, 2 processors per node, and with 4 cores per processor,
+we have 16 processors and a total of 64 cores.
+
+Case 1:
+It can be run like with the classic MPI programming model using 1 MPI rank per
+core -- 64 MPI ranks. Thus we would have 64 processes.
+
+
+mpiexec -n 64 Ray -mini-ranks-per-rank 1
+
+ (the default for -mini-ranks-per-rank is 1)
+
+ - 8 nodes
+ - 16 processors
+ - 64 cores
+ - 64 MPI ranks
+ - 64 processes
+ - 64 threads (1-to-1)
+
+Case 2:
+An alternative is to have 1 MPI rank per node. Each MPI rank has 7 miniranks.
+Each minirank runs in a thread. There is one extra thread for communication too.
+
+mpiexec -n 8 -bynode Ray -mini-ranks-per-rank 7
+
+ - 8 nodes
+ - 16 processors
+ - 64 cores
+ - 8 MPI ranks
+ - 8 processes
+ - 8*7=54 mini-rank threads
+ - 8 comm. threads (1-to-1)
+
+Case 3:
+We can also use 1 MPI rank per processor, and 3 miniranks per rank.
+
+mpiexec -n 16 -bynode Ray -mini-ranks-per-rank 3
+
+ - 8 nodes
+ - 16 processors
+ - 64 cores
+ - 16 MPI ranks
+ - 16 processes
+ - 16*3=48 mini-rank threads
+ - 16*1=16 comm. threads (1-to-1)
+
+
+With Open-MPI, the --bind-to-socket will be very nice.
+The -bynode option is necessary.
+
+2012-10-17 20:00 UTC/GMT-5:00:
+
+Tasks:
+
+- [DONE]add VirtualMachine
+- [DONE] add MiniRank
+- propagate mini rank numbers instead of ranks
+- compile Ray without errors
+- port plugin to the old adapter architecture
+- implement the case with 1 minirank per rank (no pthread)
+
+
+2012-10-18 10:44 UTF/GMT -5:00:
+
+
+
+An example of layout on the Colosse super computer at Laval University:
+
+mpiexec -n 8 -bynode Ray -mini-ranks-per-rank 7
+
+
+- assumes no HyperThreading(R)
+- assumes 8 nodes, 2 sockets / node, 1 CPU / socket, 4 cores / CPU, 1 thread / core
+- assumes 8 MPI ranks, 7 mini-ranks / MPI rank
+- total of 56 miniranks, 8 MPI ranks
+
+The plugin code only know about mini-ranks. So the plugin code is portable.
+
+ MPI rank 0
+ control thread (communication thread, don't use pthread because it's the main thread)
+ mini-rank 0 (addr [0,0], 1 pthread)
+ mini-rank 1 (addr [0,1], 1 pthread)
+ mini-rank 2 (addr [0,2], 1 pthread)
+ mini-rank 3 (addr [0,3], 1 pthread)
+ mini-rank 4 (addr [0,4], 1 pthread)
+ mini-rank 5 (addr [0,5], 1 pthread)
+ mini-rank 6 (addr [0,6], 1 pthread)
+
+ MPI rank 1
+ control thread (communication thread, don't use pthread because it's the main thread)
+ mini-rank 7 (addr [1,0], 1 pthread)
+ mini-rank 8 (addr [1,1], 1 pthread)
+ mini-rank 9 (addr [1,2], 1 pthread)
+ mini-rank 10 (addr [1,3], 1 pthread)
+ mini-rank 11 (addr [1,4], 1 pthread)
+ mini-rank 12 (addr [1,5], 1 pthread)
+ mini-rank 13 (addr [1,6], 1 pthread)
+
+ MPI rank 2
+ control thread (communication thread, don't use pthread because it's the main thread)
+ mini-rank 14 (addr [2,0], 1 pthread)
+ mini-rank 15 (addr [2,1], 1 pthread)
+ mini-rank 16 (addr [2,2], 1 pthread)
+ mini-rank 17 (addr [2,3], 1 pthread)
+ mini-rank 18 (addr [2,4], 1 pthread)
+ mini-rank 19 (addr [2,5], 1 pthread)
+ mini-rank 20 (addr [2,6], 1 pthread)
+
+ MPI rank 3
+ control thread (communication thread, don't use pthread because it's the main thread)
+ mini-rank 21 (addr [3,0], 1 pthread)
+ mini-rank 22 (addr [3,1], 1 pthread)
+ mini-rank 23 (addr [3,2], 1 pthread)
+ mini-rank 24 (addr [3,3], 1 pthread)
+ mini-rank 25 (addr [3,4], 1 pthread)
+ mini-rank 26 (addr [3,5], 1 pthread)
+ mini-rank 27 (addr [3,6], 1 pthread)
+
+ MPI rank 4
+ control thread (communication thread, don't use pthread because it's the main thread)
+ mini-rank 28 (addr [4,0], 1 pthread)
+ mini-rank 29 (addr [4,1], 1 pthread)
+ mini-rank 30 (addr [4,2], 1 pthread)
+ mini-rank 31 (addr [4,3], 1 pthread)
+ mini-rank 32 (addr [4,4], 1 pthread)
+ mini-rank 33 (addr [4,5], 1 pthread)
+ mini-rank 34 (addr [4,6], 1 pthread)
+
+ MPI rank 5
+ control thread (communication thread, don't use pthread because it's the main thread)
+ mini-rank 35 (addr [5,0], 1 pthread)
+ mini-rank 36 (addr [5,1], 1 pthread)
+ mini-rank 37 (addr [5,2], 1 pthread)
+ mini-rank 38 (addr [5,3], 1 pthread)
+ mini-rank 39 (addr [5,4], 1 pthread)
+ mini-rank 40 (addr [5,5], 1 pthread)
+ mini-rank 41 (addr [5,6], 1 pthread)
+
+ MPI rank 6
+ control thread (communication thread, don't use pthread because it's the main thread)
+ mini-rank 42 (addr [6,0], 1 pthread)
+ mini-rank 43 (addr [6,1], 1 pthread)
+ mini-rank 44 (addr [6,2], 1 pthread)
+ mini-rank 45 (addr [6,3], 1 pthread)
+ mini-rank 46 (addr [6,4], 1 pthread)
+ mini-rank 47 (addr [6,5], 1 pthread)
+ mini-rank 48 (addr [6,6], 1 pthread)
+
+ MPI rank 7
+ control thread (communication thread, don't use pthread because it's the main thread)
+ mini-rank 49 (addr [7,0], 1 pthread)
+ mini-rank 50 (addr [7,1], 1 pthread)
+ mini-rank 51 (addr [7,2], 1 pthread)
+ mini-rank 52 (addr [7,3], 1 pthread)
+ mini-rank 53 (addr [7,4], 1 pthread)
+ mini-rank 54 (addr [7,5], 1 pthread)
+ mini-rank 55 (addr [7,6], 1 pthread)
+
+
+2012-10-29 GMT -4:00
+
+The secret sauce is about synchronization.
+
+This 2003 Ph.D. thesis pretty much summarizes the state-of-the-art (in 2003).
+
+ http://www.cse.chalmers.se/~tsigas/papers/Yi-Thesis.pdf
+
+
+As a starting point, there is a lot of explanations on this project, with associated code
+in the public domain.
+
+ http://www.codeproject.com/Articles/43510/Lock-Free-Single-Producer-Single-Consumer-Circular
+
+
+The future of high-performance computing is with lockless non-blocking distributed algorithms.
+
+- lockless => choose atomic operations over locks;
+- non-blocking => choose trylock over lock
+- distributed => choose message passing over shared structures
+
+
diff --git a/RayPlatform/Documentation/Torus.txt b/RayPlatform/Documentation/Torus.txt
new file mode 100644
index 0000000..910598c
--- /dev/null
+++ b/RayPlatform/Documentation/Torus.txt
@@ -0,0 +1,37 @@
+A Torus with radix k and with dimension n has k^n vertices.
+
+Vertices <x_1,x_2,x_3,...,x_n> and <y_1,y_2,y_3,...,y_n> are linked
+if and only if only one dimension i \in \{1,...,n\} exists such as
+
+ inc(x_i) = y_i or inc(y_i) = x_i
+
+ where inc(v) = (v+1) mod k
+
+
+Complete
+ diameter = 0
+ vertices = 4096
+ degree = 4096
+ edges = vertices * degree = 16777216
+
+Polytope
+ radix = 64
+ diameter = 2
+ vertices = radix ^ diameter = 4096
+ degree = (radix - 1) * diameter = 126
+ edges = vertices * degree = 524288
+
+Polytope
+ radix = 4
+ diameter = 6
+ vertices = radix ^ diameter = 4096
+ degree = (radix - 1) * diameter = 18
+ edges = vertices * degree = 73728
+
+Torus
+ radix = 4
+ dimension = 6
+ vertices = radix ^ dimension = 4096
+ degree = 2 * dimension = 12
+ edges = vertices * degree = 49152
+ diameter = 6
diff --git a/RayPlatform/Documentation/plugins.txt b/RayPlatform/Documentation/plugins.txt
index 2d3b052..bccb8b0 100644
--- a/RayPlatform/Documentation/plugins.txt
+++ b/RayPlatform/Documentation/plugins.txt
@@ -1,4 +1,74 @@
The application programming interface for plugins is all in the class ComputeCore.
+## Relationships:
+
+CorePlugin <---> MasterModeHandler (an adapter) <---> ComputeCore
+CorePlugin <---> SlaveModeHandler (an adapter) <---> ComputeCore
+CorePlugin <---> MessageTagHandler (an adapter) <---> ComputeCore
+
see RayPlatform/core/ComputeCore.h
+
+Macros can be used to push adapters on the ComputeCore instance.
+Otherwise, the code must be written for adapters (declaration, implementation, binding).
+
+
+## in the .h file of a plugin, before the class declaration
+
+__DeclarePlugin(PluginExample) -> declares a plugin. this is required because adapters need a forward declaration
+
+__DeclareMasterModeAdapter(PluginExample,MASTER_MODE_EXAMPLE) -> declares a master mode adapter for a plugin
+
+__DeclareSlaveModeAdapter(PluginExample,SLAVE_MODE_EXAMPLE) -> declares a slave mode adapter for a plugin
+
+__DeclareMessageTagAdapter(PluginExample,MESSAGE_TAG_EXAMPLE) -> declares a message tag adapter for a plugin
+
+
+## in the .h file of a plugin, inside the class declaration
+
+__AddAdapter(PluginExample,MASTER_MODE_EXAMPLE)
+
+__AddAdapter(PluginExample,SLAVE_MODE_EXAMPLE)
+
+__AddAdapter(PluginExample,MESSAGE_TAG_EXAMPLE)
+
+
+## in the .cpp file of a plugin, before the first method implementation
+
+__CreatePlugin(PluginExample) -> this is used when mini-ranks are not enabled.
+
+__CreateMasterModeAdapter(PluginExample,MASTER_MODE_EXAMPLE) -> creates the actual code for a master mode adapter
+
+__CreateSlaveModeAdapter(PluginExample,SLAVE_MODE_EXAMPLE) -> creates the actual code for a slave mode adapter
+
+__CreateSlaveModeAdapter(PluginExample,MESSAGE_TAG_EXAMPLE) -> creates the actual code for a message tag adapter
+
+
+## in the .cpp file of a plugin, inside registerPlugin or resolveSymbols
+
+__GetAdapter(PluginExample,MASTER_MODE_EXAMPLE) -> gets the reference to an adapter
+
+__GetAdapter(PluginExample,SLAVE_MODE_EXAMPLE) -> gets the reference to an adapter
+
+__GetAdapter(PluginExample,MESSAGE_TAG_EXAMPLE) -> gets the reference to an adapter
+
+__BindPlugin(PluginExample) -> binds a plugin
+
+__BindAdapter(PluginExample,MASTER_MODE_EXAMPLE) -> binds a adapter
+
+__BindAdapter(PluginExample,SLAVE_MODE_EXAMPLE) -> binds a adapter
+
+__BindAdapter(PluginExample,MESSAGE_TAG_EXAMPLE) -> binds a adapter
+
+
+
+## Simplified API
+
+It is possible to use only a single API call to configure a given handler.
+
+
+__ConfigureMasterModeHandler(PluginExample, MASTER_MODE_EXAMPLE); -> configure a handler
+
+__ConfigureSlaveModeHandler(PluginExample, SLAVE_MODE_EXAMPLE); -> configure a handler
+
+__ConfigureMessageTagHandler(PluginExample, MESSAGE_TAG_EXAMPLE); -> configure a handler
diff --git a/RayPlatform/GPL-3.0.txt b/RayPlatform/GPL-3.0.txt
new file mode 100644
index 0000000..203fe3f
--- /dev/null
+++ b/RayPlatform/GPL-3.0.txt
@@ -0,0 +1,6 @@
+The Ray Application is licensed under the GNU General Public License version 3.
+see gpl-3.0.txt
+
+The Ray Platform is licensed under the GNU Lesser General Public License version 3.
+see RayPlatform/lgpl-3.0.txt
+
diff --git a/RayPlatform/LICENSE b/RayPlatform/LICENSE
new file mode 100644
index 0000000..8fb1ca9
--- /dev/null
+++ b/RayPlatform/LICENSE
@@ -0,0 +1,3 @@
+This framework is distributed under the GNU Lesser General Public License, version 3.
+
+See lgpl-3.0.txt
diff --git a/RayPlatform/Makefile b/RayPlatform/Makefile
index 2db2a9d..2c13fc3 100644
--- a/RayPlatform/Makefile
+++ b/RayPlatform/Makefile
@@ -3,11 +3,11 @@
# based on http://www.ravnborg.org/kbuild/makefiles.html
#
-VERSION = 1
-PATCHLEVEL = 1
+VERSION = 2
+PATCHLEVEL = 0
SUBLEVEL = 0
-EXTRAVERSION =
-NAME = Chariot of Complexity
+EXTRAVERSION =
+NAME = Actor Nest
RAYPLATFORM_VERSION = $(VERSION).$(PATCHLEVEL).$(SUBLEVEL)$(EXTRAVERSION)
@@ -16,12 +16,23 @@ include common.mk
all: libRayPlatform.a
libRayPlatform.a: $(obj-y)
- $(AR) rcs $@ $^
+ $(Q)$(ECHO) " AR $@"
+ $(Q)$(AR) rcs $@ $^
+
+# compile assertions
+
+CONFIG_ASSERT=$(ASSERT)
+
+CONFIG_FLAGS-y=
+CONFIG_FLAGS-$(CONFIG_ASSERT) += -D CONFIG_ASSERT -D ASSERT
+CONFIG_FLAGS=$(CONFIG_FLAGS-y)
# inference rule
%.o: %.cpp
- $(MPICXX) $(CXXFLAGS) -D RAYPLATFORM_VERSION=\"$(RAYPLATFORM_VERSION)\" -I. -c -o $@ $<
+ $(Q)$(ECHO) " CXX $@"
+ $(Q)$(MPICXX) $(CXXFLAGS) $(CONFIG_FLAGS) -D RAYPLATFORM_VERSION=\"$(RAYPLATFORM_VERSION)\" -I. -c -o $@ $<
clean:
- $(RM) -f libRayPlatform.a $(obj-y)
+ $(Q)$(ECHO) CLEAN RayPlatform
+ $(Q)$(RM) -f libRayPlatform.a $(obj-y)
diff --git a/RayPlatform/Makefile.shared-gcc b/RayPlatform/Makefile.shared-gcc
index fcfa947..b2b0232 100644
--- a/RayPlatform/Makefile.shared-gcc
+++ b/RayPlatform/Makefile.shared-gcc
@@ -1,7 +1,4 @@
# author: Sébastien Boisvert
-#
-# based on http://www.ravnborg.org/kbuild/makefiles.html
-#
include common.mk
diff --git a/RayPlatform/README b/RayPlatform/README.md
similarity index 87%
rename from RayPlatform/README
rename to RayPlatform/README.md
index a52cc5b..ba5f010 100644
--- a/RayPlatform/README
+++ b/RayPlatform/README.md
@@ -7,8 +7,23 @@ added on the RayPlatform compute engine.
It uses the message-passing interface for interprocess communication, but this is
transparent to the developer.
-## Illustration
+## Models
+
+Mainly two models are available:
+
+- Rank model (with RayPlatform Plugin API)
+- Actor Model (with the RayPlatform Actor Playground API)
+
+Also: RayPlatform implements a "mini-ranks" programming model. It is a hybrid
+programming model where MPI ranks and mini-ranks exist. Mini-ranks run in separate
+threads inside a MPI rank process. This eases the usage of a superior hybrid
+model with MPI and threads (pthread). But the actor model is superior in every aspect.
+
+
+## Illustration of the RayPlatform Plugin API
+
+```
+--------------------------------------------------------------------------+
| |
| Application |
@@ -30,10 +45,11 @@ transparent to the developer.
| Message Passing Interface |
| |
+--------------------------------------------------------------------------+
+```
## Projects using RayPlatform
-- The Ray genome assembler
+- The Ray genome assembler (and also Ray Meta, Ray Communities, Ray Surveyor)
http://github.com/sebhtml/ray
- RayPlatform example
http://github.com/sebhtml/RayPlatform-example
diff --git a/RayPlatform/RayPlatform/actors/Actor.cpp b/RayPlatform/RayPlatform/actors/Actor.cpp
new file mode 100644
index 0000000..9879b3e
--- /dev/null
+++ b/RayPlatform/RayPlatform/actors/Actor.cpp
@@ -0,0 +1,114 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of the actor model implementation in RayPlatform.
+
+ RayPlatform is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ RayPlatform is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with RayPlatform. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+// \author Sébastien Boisvert
+// License: GPLv3
+// Université Laval 2013-10-10
+
+#include "Actor.h"
+
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/cryptography/crypto.h>
+
+#include <iostream>
+using namespace std;
+
+Actor::Actor() {
+}
+
+Actor::~Actor() {
+}
+
+void Actor::send(int destination, Message * message) {
+
+ message->setSourceActor(getName());
+ message->setDestinationActor(destination);
+
+ // if the buffer is not NULL, allocate a RayPlatform
+ // buffer and copy stuff in it.
+ if(message->getBuffer() != NULL) {
+
+ // this call return MAXIMUM_MESSAGE_SIZE_IN_BYTES + padding
+ char * newBuffer = (char*)m_core->getOutboxAllocator()->allocate(42);
+ int bytes = message->getNumberOfBytes();
+ char * oldBuffer = (char*)message->getBufferBytes();
+
+ memcpy(newBuffer, oldBuffer, bytes * sizeof(char));
+
+ message->setBuffer(newBuffer);
+ }
+
+#if 0
+ cout << "DEBUG Actor::send CRC32: ";
+ cout << computeCyclicRedundancyCode32((uint8_t*) message->getBufferBytes(),
+ message->getNumberOfBytes());
+ cout << endl;
+#endif
+
+ // deleguate the message to ComputeCore..
+
+ m_core->sendActorMessage(message);
+}
+
+void Actor::send(int destination, Message & message) {
+
+ send(destination, &message);
+}
+
+void Actor::spawn(Actor * actor) {
+
+ m_core->spawnActor(actor);
+}
+
+void Actor::configureStuff(int name, Playground * kernel) {
+
+ m_name = name;
+ m_core = kernel;
+
+ m_dead = false;
+}
+
+void Actor::die() {
+ m_dead = true;
+}
+
+bool Actor::isDead() const {
+
+ return m_dead;
+}
+
+int Actor::getName() const {
+ return m_name;
+}
+
+void Actor::printName() const {
+
+ cout << "/actors/" << getName();
+ cout << " -> ";
+}
+
+int Actor::getRank() const {
+ return m_core->getRank();
+}
+
+int Actor::getSize() const {
+ return m_core->getSize();
+}
diff --git a/RayPlatform/RayPlatform/actors/Actor.h b/RayPlatform/RayPlatform/actors/Actor.h
new file mode 100644
index 0000000..577078e
--- /dev/null
+++ b/RayPlatform/RayPlatform/actors/Actor.h
@@ -0,0 +1,75 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of the actor model implementation in RayPlatform.
+
+ RayPlatform is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ RayPlatform is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with RayPlatform. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+// author: Sébastien Boisvert
+// Université Laval 2013-10-10
+
+#ifndef ActorHeader
+
+#define ActorHeader
+
+class ComputeCore;
+class Message;
+class Playground;
+
+#include <RayPlatform/communication/Message.h>
+
+/**
+ * actor model.
+ *
+ * This is part of the
+ * RayPlatform Actor Playground API
+ *
+ * \author Sébastien Boisvert
+ * \see https://github.com/sebhtml/BioActors
+ */
+class Actor {
+
+private:
+ int m_name;
+ Playground * m_core;
+ bool m_dead;
+
+ void send(int destination, Message * message);
+public:
+
+ enum {
+ FIRST_TAG = 10000,
+ BOOT
+ };
+
+ Actor();
+ virtual ~Actor();
+
+ virtual void receive(Message & message) = 0;
+ void send(int destination, Message & message);
+ void spawn(Actor * actor);
+ void configureStuff(int name, Playground * kernel);
+ int getName() const;
+ void printName() const;
+
+ int getRank() const;
+ int getSize() const;
+
+ void die();
+ bool isDead() const;
+};
+
+#endif
diff --git a/RayPlatform/RayPlatform/actors/Playground.cpp b/RayPlatform/RayPlatform/actors/Playground.cpp
new file mode 100644
index 0000000..b454856
--- /dev/null
+++ b/RayPlatform/RayPlatform/actors/Playground.cpp
@@ -0,0 +1,211 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform: a message-passing development framework
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#include "Playground.h"
+
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/memory/RingAllocator.h>
+
+#include <stdlib.h>
+
+#ifdef CONFIG_ASSERT
+#include <assert.h>
+#endif
+
+Playground::Playground() {
+
+ m_actorIterator = 0;
+ m_aliveActors = 0;
+}
+
+Playground::~Playground() {
+
+}
+
+void Playground::spawnActor(Actor * actor) {
+
+ // actor 0 on rank 0 is the rank itself !
+ // actor i on rank i is the rank itself too.
+ if(m_actorIterator == 0) {
+ m_actorIterator++;
+ Actor * actor = NULL;
+ m_actors.push_back(actor);
+ }
+
+ int identifier = m_computeCore->getRank() + m_actorIterator * getSize();
+ actor->configureStuff(identifier, this);
+
+ //cout << "DEBUG ... spawnActor name= " << identifier << endl;
+
+ m_actors.push_back(actor);
+
+ m_actorIterator++;
+
+ m_aliveActors ++;
+}
+
+void Playground::sendActorMessage(Message * message) {
+
+ int sourceActor = message->getSourceActor();
+ int destinationActor = message->getDestinationActor();
+
+ int sourceRank = getActorRank(sourceActor);
+ int destinationRank = getActorRank(destinationActor);
+
+#ifdef CONFIG_ASSERT
+ assert(sourceActor >= 0);
+ assert(destinationActor >= 0);
+ assert(sourceRank >= 0);
+ assert(destinationRank >= 0);
+ assert(sourceRank < getSize());
+ assert(destinationRank < getSize());
+#endif
+
+#if 0
+ cout << "DEBUG ... " << message << " sendActorMessage sourceActor= ";
+ cout << sourceActor << " destinationActor= ";
+ cout << destinationActor << " sourceRank= " << sourceRank;
+ cout << " tag= " << message->getTag();
+ cout << " bytes= " << message->getNumberOfBytes();
+ cout << " destinationRank= " << destinationRank << endl;
+#endif
+
+ message->setSource(sourceRank);
+ message->setDestination(destinationRank);
+
+ getComputeCore()->send(message);
+}
+
+int Playground::getActorRank(int name) const {
+
+ return name % getSize();
+}
+
+int Playground::getRank() const {
+ return m_computeCore->getRank();
+}
+
+int Playground::getSize() const {
+
+ return m_computeCore->getSize();
+}
+
+void Playground::receiveActorMessage(Message * message) {
+
+ int actorName = message->getDestinationActor();
+
+ int index = actorName / getSize();
+
+#ifdef CONFIG_ASSERT
+ if(index < 0) {
+ cout << "Error: negative actor name.";
+ cout << " actorName " << actorName;
+ cout << " getSize " << getSize();
+ cout << " index " << index;
+ message->printActorMetaData();
+ cout << endl;
+ }
+ assert(index >= 0);
+#endif
+
+#if 0
+ cout << "DEBUG .... Rank= " << getRank();
+ cout << " tag= " << message->getTag();
+ cout << " receiveActorMessage actorName= " << actorName;
+ cout << " index=> " << index;
+ message->printActorMetaData();
+ cout << endl;
+#endif
+
+ if(!(index < (int) m_actors.size()))
+ return;
+
+ Actor * actor = m_actors[index];
+
+ if(actor == NULL)
+ return;
+
+#ifdef CONFIG_ASSERT
+ assert( message != NULL );
+#endif
+
+ actor->receive(*message);
+
+ if(actor->isDead()) {
+
+ m_aliveActors --;
+
+ /*
+ cout << "DEBUG Playground actor " << actor->getName() << " died, remaining: ";
+ cout << m_aliveActors << endl;
+ */
+
+ delete actor;
+ actor = NULL;
+ m_actors[index] = NULL;
+ }
+}
+
+bool Playground::hasAliveActors() const {
+
+ return m_aliveActors > 0;
+}
+
+void Playground::bootActors() {
+
+ //cout << "DEBUG ... bootActors" << endl;
+
+ int toBoot = m_actors.size();
+
+ for(int i = 0 ; i < toBoot ; ++i) {
+
+ Actor * actor = m_actors[i];
+ if(actor == NULL) {
+ continue;
+ }
+
+ int name = actor->getName();
+
+ Message message;
+ message.setTag(Actor::BOOT);
+ message.setSource(m_computeCore->getRank());
+ message.setDestination(m_computeCore->getRank());
+ message.setSourceActor(name);
+ message.setDestinationActor(name);
+
+ //cout << "DEBUG booting actor # " << name << endl;
+
+ receiveActorMessage(&message);
+ }
+}
+
+void Playground::initialize(ComputeCore * core) {
+
+ m_computeCore = core;
+}
+
+ComputeCore * Playground::getComputeCore() {
+ return m_computeCore;
+}
+RingAllocator * Playground::getOutboxAllocator() {
+
+ return getComputeCore()->getOutboxAllocator();
+}
diff --git a/RayPlatform/RayPlatform/actors/Playground.h b/RayPlatform/RayPlatform/actors/Playground.h
new file mode 100644
index 0000000..411e5fa
--- /dev/null
+++ b/RayPlatform/RayPlatform/actors/Playground.h
@@ -0,0 +1,81 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform: a message-passing development framework
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#ifndef PlaygroundHeader
+#define PlaygroundHeader
+
+#include <RayPlatform/actors/Actor.h>
+
+#include <vector>
+using namespace std;
+
+class ComputeCore;
+class RingAllocator;
+
+/**
+ * This is part of the
+ * RayPlatform Actor Playground API
+ */
+class Playground {
+
+private:
+
+ vector<Actor*> m_actors;
+ int m_actorIterator;
+ int m_aliveActors;
+ ComputeCore * m_computeCore;
+
+public:
+
+ Playground();
+ ~Playground();
+
+ void initialize(ComputeCore * core);
+ bool hasAliveActors() const;
+
+ // actor stuff !!!
+
+ /**
+ * assign a unique identifier to the actor
+ * and add it to the party team.
+ */
+ void spawnActor(Actor * actor);
+
+ /**
+ * send a message to an actor.
+ */
+ void sendActorMessage(Message * message);
+
+ /**
+ * receive a message for an actor.
+ */
+ void receiveActorMessage(Message * message);
+ void bootActors();
+ int getActorRank(int name) const;
+
+ RingAllocator * getOutboxAllocator();
+ ComputeCore * getComputeCore();
+
+ int getRank() const;
+ int getSize() const;
+};
+
+#endif
diff --git a/RayPlatform/communication/BufferedData.cpp b/RayPlatform/RayPlatform/communication/BufferedData.cpp
similarity index 92%
rename from RayPlatform/communication/BufferedData.cpp
rename to RayPlatform/RayPlatform/communication/BufferedData.cpp
index fca4c1d..92ec9b3 100644
--- a/RayPlatform/communication/BufferedData.cpp
+++ b/RayPlatform/RayPlatform/communication/BufferedData.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,10 +19,11 @@
*/
-#include <communication/BufferedData.h>
-#include <memory/RingAllocator.h>
-#include <memory/allocator.h>
-#include <structures/StaticVector.h>
+#include "BufferedData.h"
+
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/memory/allocator.h>
+#include <RayPlatform/structures/StaticVector.h>
#include <string.h>
#include <stdio.h>
@@ -175,7 +176,7 @@ bool BufferedData::flush(int destination,int period,int tag,RingAllocator*outbox
message[i]=getAt(destination,i);
}
Message aMessage(message,amount,destination,tag,rank);
- outbox->push_back(aMessage);
+ outbox->push_back(&aMessage);
m_flushedMessages++;
reset(destination);
return true;
diff --git a/RayPlatform/communication/BufferedData.h b/RayPlatform/RayPlatform/communication/BufferedData.h
similarity index 90%
rename from RayPlatform/communication/BufferedData.h
rename to RayPlatform/RayPlatform/communication/BufferedData.h
index f872c9e..6258b05 100644
--- a/RayPlatform/communication/BufferedData.h
+++ b/RayPlatform/RayPlatform/communication/BufferedData.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,10 +22,11 @@
#ifndef _BufferedData
#define _BufferedData
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/memory/MyAllocator.h>
+#include <RayPlatform/structures/StaticVector.h>
+
#include <stdint.h>
-#include <memory/MyAllocator.h>
-#include <structures/StaticVector.h>
-#include <memory/RingAllocator.h>
/**
* This class accumulates messages and flush them when the threshold is reached.
diff --git a/RayPlatform/RayPlatform/communication/Message.cpp b/RayPlatform/RayPlatform/communication/Message.cpp
new file mode 100644
index 0000000..ddb2b23
--- /dev/null
+++ b/RayPlatform/RayPlatform/communication/Message.cpp
@@ -0,0 +1,389 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#include "Message.h"
+#include "mpi_tags.h"
+
+#include <RayPlatform/cryptography/crypto.h>
+
+#include <iostream>
+using namespace std;
+
+#include <string.h>
+
+#ifdef CONFIG_ASSERT
+#include <assert.h>
+#endif
+
+#define ACTOR_MODEL_NOBODY -1
+
+/**
+ * We always pad the buffer with actor source and actor destination.
+ * If routing is enabled, we also pad with the route source and the route destination.
+ * Finally, the buffer may be padded with a checksum too !
+ *
+
+ */
+#define MESSAGE_META_DATA_ACTOR_SOURCE 0
+#define MESSAGE_META_DATA_ACTOR_DESTINATION 4
+#define MESSAGE_META_DATA_ROUTE_SOURCE 8
+#define MESSAGE_META_DATA_ROUTE_DESTINATION 12
+#define MESSAGE_META_DATA_CHECKSUM 16
+
+
+Message::Message() {
+
+ initialize();
+}
+
+void Message::initialize() {
+ m_buffer = NULL;
+ //m_count = 0;
+ m_bytes = 0;
+ m_tag = 0;
+ m_source = 0;
+ m_destination = 0;
+ m_sourceActor = ACTOR_MODEL_NOBODY;
+ m_destinationActor = ACTOR_MODEL_NOBODY;
+}
+
+Message::~Message() {
+}
+
+/** buffer must be allocated or else it will CORE DUMP. */
+Message::Message(MessageUnit*b,int c,Rank dest,MessageTag tag,Rank source){
+
+ initialize();
+
+ m_buffer=b;
+ //m_count=c;
+ m_bytes = c * sizeof(MessageUnit);
+ m_destination=dest;
+ m_tag=tag;
+ m_source=source;
+
+ /*
+ cout << "DEBUG in constructor -> ";
+ printActorMetaData();
+ cout << endl;
+ */
+}
+
+MessageUnit*Message::getBuffer(){
+ return (MessageUnit*)m_buffer;
+}
+
+char * Message::getBufferBytes() {
+ return (char*) m_buffer;
+}
+
+int Message::getCount() const{
+ return m_bytes / sizeof(MessageUnit);
+}
+
+Rank Message::getDestination() const{
+ return m_destination;
+}
+
+MessageTag Message::getTag() const{
+ return m_tag;
+}
+
+//Message::Message(){}
+
+int Message::getSource() const{
+ return m_source;
+}
+
+void Message::print(){
+ uint8_t shortTag=getTag();
+
+ cout<<"Source: "<<getSource()<<" Destination: "<<getDestination();
+
+ if(isActorModelMessage()) {
+ cout << " ActorModel: Yes.";
+ } else {
+ cout <<" Tag: "<<MESSAGE_TAGS[shortTag];
+ }
+
+ cout<<" RealTag: "<<getTag();
+ cout<<" Count: "<<getCount();
+
+ if(getCount() > 0){
+ cout<<" Overlay: "<<getBuffer()[0];
+ }
+}
+
+void Message::setBuffer(void *buffer){
+ m_buffer = buffer;
+}
+
+void Message::setTag(MessageTag tag){
+ m_tag=tag;
+}
+
+void Message::setCount(int count){
+ //m_count=count;
+ m_bytes = count * sizeof(MessageUnit);
+}
+
+void Message::setSource(Rank source){
+ m_source=source;
+}
+
+void Message::setDestination(Rank destination){
+ m_destination=destination;
+}
+
+bool Message::isActorModelMessage() const {
+
+ return ( m_sourceActor != ACTOR_MODEL_NOBODY
+ && m_destinationActor != ACTOR_MODEL_NOBODY );
+}
+
+
+int Message::getDestinationActor() const {
+ return m_destinationActor;
+}
+
+int Message::getSourceActor() const {
+ return m_sourceActor;
+}
+
+void Message::setSourceActor(int sourceActor) {
+ m_sourceActor = sourceActor;
+}
+
+void Message::setDestinationActor(int destinationActor) {
+ m_destinationActor = destinationActor;
+}
+
+void Message::saveActorMetaData() {
+
+ // write rank numbers for these.
+ // MPI ranks actually have actor names too !
+ if(m_sourceActor < 0 && m_destinationActor < 0) {
+ m_sourceActor = m_source;
+ m_destinationActor = m_destination;
+ }
+
+#ifdef CONFIG_ASSERT
+ int bytes = getNumberOfBytes();
+ uint32_t checksumBefore = computeCyclicRedundancyCode32((uint8_t*)getBuffer(), bytes);
+
+ if(m_sourceActor < 0) {
+ cout << "Error m_sourceActor " << m_sourceActor;
+ printActorMetaData();
+ }
+ assert(m_sourceActor >= 0);
+ assert(m_destinationActor >= 0);
+#endif
+ //cout << "DEBUG saveActorMetaData tag " << getTag() << endl;
+
+ int offset = getNumberOfBytes();
+
+ // actor metadata must be the first to be saved.
+ offset -= 0;
+
+ char * memory = (char*) getBuffer();
+
+ memcpy(memory + offset + MESSAGE_META_DATA_ACTOR_SOURCE, &m_sourceActor, sizeof(int));
+ memcpy(memory + offset + MESSAGE_META_DATA_ACTOR_DESTINATION, &m_destinationActor, sizeof(int));
+ //printActorMetaData();
+
+ setNumberOfBytes(getNumberOfBytes() + 2 * sizeof(int));
+ //setCount(m_count + 1);
+ //m_count += 1; // add 1 uint64_t
+
+ /*
+ cout << "DEBUG after saveActorMetaData ";
+ printActorMetaData();
+ cout << endl;
+ */
+
+#ifdef CONFIG_ASSERT
+ uint32_t checksumAfter = computeCyclicRedundancyCode32((uint8_t*)getBuffer(), bytes);
+
+ assert(checksumBefore == checksumAfter);
+#endif
+}
+
+void Message::printActorMetaData() {
+ cout << "DEBUG printActorMetaData tag= " << getTag();
+ cout << " m_sourceActor = " << getSourceActor();
+ cout << " m_destinationActor = " << getDestinationActor() << " ";
+ cout << " bytes= " << getNumberOfBytes();
+
+}
+
+void Message::loadActorMetaData() {
+/*
+ cout << "DEBUG before loadActorMetaData -> ";
+ printActorMetaData();
+ cout << endl;
+*/
+ // m_count already contains the actor metadata.
+ int offset = getNumberOfBytes();
+ offset -= 2 * sizeof(int);
+
+ char * memory = (char*) getBuffer();
+
+ memcpy(&m_sourceActor, memory + offset + MESSAGE_META_DATA_ACTOR_SOURCE, sizeof(int));
+ memcpy(&m_destinationActor, memory + offset + MESSAGE_META_DATA_ACTOR_DESTINATION, sizeof(int));
+
+
+ /*
+ cout << "DEBUG loadActorMetaData m_sourceActor= " << m_sourceActor;
+ cout << " m_destinationActor= " << m_destinationActor << endl;
+*/
+ // remove 1 uint64_t
+
+ setNumberOfBytes(getNumberOfBytes() - 2 * sizeof(int));
+ //setCount(m_count - 1);
+ //m_count -= 1;
+/*
+ cout << "DEBUG after loadActorMetaData -> ";
+ printActorMetaData();
+ cout << endl;
+ */
+
+#ifdef CONFIG_ASSERT
+ assert(m_sourceActor >= 0);
+ assert(m_destinationActor >= 0);
+#endif
+
+}
+
+int Message::getMetaDataSize() const {
+ return 4 * sizeof(int);
+}
+
+void Message::setRoutingSource(int source) {
+
+ m_routingSource = source;
+}
+
+void Message::setRoutingDestination(int destination) {
+
+ m_routingDestination = destination;
+}
+
+void Message::saveRoutingMetaData() {
+
+#ifdef CONFIG_ASSERT
+ assert(m_routingSource >= 0);
+ assert(m_routingDestination >= 0);
+#endif
+
+ /*
+ cout << "DEBUG saveRoutingMetaData m_routingSource " << m_routingSource;
+ cout << " m_routingDestination " << m_routingDestination << endl;
+*/
+ int offset = getNumberOfBytes();
+ char * memory = (char*) getBuffer();
+
+ // the count already includes the actor addresses
+ // this is stupid, but hey, nobody aside me is touching this code
+ // with a ten-foot stick
+ offset -= 2 * sizeof(int);
+
+ memcpy(memory + offset + MESSAGE_META_DATA_ROUTE_SOURCE, &m_routingSource, sizeof(int));
+ memcpy(memory + offset + MESSAGE_META_DATA_ROUTE_DESTINATION, &m_routingDestination, sizeof(int));
+ //printActorMetaData();
+
+ setNumberOfBytes(getNumberOfBytes() + 2 * sizeof(int));
+ //m_count += 1; // add 1 uint64_t
+
+ //cout << "DEBUG saved routing metadata at offset " << offset << " new count " << m_count << endl;
+ //displayMetaData();
+}
+
+
+int Message::getRoutingSource() const {
+
+ return m_routingSource;
+}
+
+int Message::getRoutingDestination() const {
+
+ return m_routingDestination;
+}
+
+void Message::loadRoutingMetaData() {
+ //cout << "DEBUG loadRoutingMetaData count " << getNumberOfBytes() << endl;
+
+ int offset = getNumberOfBytes();
+
+ // m_count contains actor metadata *AND* routing metadata (if necessary)
+ offset -= 2 * sizeof(int);
+ offset -= 2 * sizeof(int);
+
+ char * memory = (char*) getBuffer();
+
+ memcpy(&m_routingSource, memory + offset + MESSAGE_META_DATA_ROUTE_SOURCE, sizeof(int));
+ memcpy(&m_routingDestination, memory + offset + MESSAGE_META_DATA_ROUTE_DESTINATION, sizeof(int));
+
+ setNumberOfBytes(getNumberOfBytes() - 2 * sizeof(int));
+ //m_count -= 1;
+
+ /*
+ cout << "DEBUG loadRoutingMetaData ";
+ cout << "loaded m_routingSource ";
+ cout << m_routingSource;
+ cout << " m_routingDestination " << m_routingDestination;
+ cout << " offset " << offset ;
+ printActorMetaData();
+ cout << endl;
+ */
+
+ //displayMetaData();
+
+ // these can not be negative, otherwise
+ // this method would not have been called now.
+#ifdef CONFIG_ASSERT
+ assert(m_routingSource >= 0);
+ assert(m_routingDestination >= 0);
+#endif
+}
+
+void Message::displayMetaData() {
+ Message * aMessage = this;
+
+ cout << " DEBUG displayMetaData count " << aMessage->getNumberOfBytes();
+ cout << " tag " << getTag() << endl;
+
+ int offset = getNumberOfBytes() - 2 * 2 * sizeof(int);
+
+ for(int i = 0 ; i < 4 ; ++i) {
+ cout << " [" << i << " -> " << ((int*)aMessage->getBuffer())[offset + i] << "]";
+ }
+ cout << endl;
+
+}
+
+void Message::setNumberOfBytes(int bytes) {
+
+ m_bytes = bytes;
+}
+
+int Message::getNumberOfBytes() const {
+ return m_bytes;
+}
diff --git a/RayPlatform/RayPlatform/communication/Message.h b/RayPlatform/RayPlatform/communication/Message.h
new file mode 100644
index 0000000..b60b6d3
--- /dev/null
+++ b/RayPlatform/RayPlatform/communication/Message.h
@@ -0,0 +1,176 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#ifndef _Message_H
+#define _Message_H
+
+#include <RayPlatform/core/types.h>
+
+#include <stdint.h>
+
+/**
+ * In Ray, every message is a Message.
+ * the inbox and the outbox are arrays of Message's
+ * All the code in Ray utilise Message to communicate information.
+ * MPI_Datatype is always MPI_UNSIGNED_LONG_LONG
+ * m_count is >=0 and <= MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(uint64_t)
+ * (default is 4000/8 = 500 ).
+ *
+ * in the buffer:
+ *
+ * Application payload (works)
+ * int source actor (works)
+ * int destination actor (works)
+ * int routing source (works)
+ * int routing destination (works)
+ * int checksum (probably broken)
+ *
+ * The offset (see Message.cpp) are relative to the number of bytes *BEFORE*
+ * adding this metadata.
+ * Note that the m_count is the number of MessageUnit for the message
+ * and may include metadata if saveActorMetaData or saveRoutingMetaData
+ * was called.
+ *
+ * Obviously, this is the correct order for calling this methods:
+ *
+ * setSourceActor
+ * setDestinationActor
+ * saveActorMetaData (increases m_count by 1 MessageUnit)
+ * setRoutingSource
+ * setRoutingDestination
+ * saveRoutingMetaData (increases m_count by 1 MessageUnit)
+ *
+ * then, the message can be sent.
+ *
+ * On the receiving side:
+ *
+ * loadActorMetaData
+ * getSourceActor
+ * getDestinationActor (decreases m_count by 1 MessageUnit)
+ * loadRoutingMetaData
+ * getRoutingSource
+ * getRoutingDestination (decreases m_count by 1 MessageUnit)
+ *
+ * All this crap is necessary because messages exposed to actors or to MPI ranks
+ * do not contain these metadata.
+ *
+ * The platform allows MAXIMUM_MESSAGE_SIZE_IN_BYTES for buffers of application messages.
+ * To this, we add 2 * MessageUnit (1 MessageUnit for actors, 1 MessageUnit for routing).
+ *
+ * TODO: the checksum code is probably broken by now. check that out.
+ *
+ * \author Sébastien Boisvert
+ */
+class Message{
+private:
+
+ /** the message body, contains data
+ * if NULL, m_count must be 0 */
+ void * m_buffer;
+
+ /** the number of uint64_t that the m_buffer contains
+ * can be 0 regardless of m_buffer value
+ * */
+ //int m_count;
+ int m_bytes;
+
+ /** the Message-passing interface rank destination
+ * Must be >=0 and <= MPI_Comm_size()-1 */
+ Rank m_destination;
+
+ /**
+ * Ray message-passing interface message tags are named RAY_MPI_TAG_<something>
+ * see mpi_tag_macros.h
+ */
+ MessageTag m_tag;
+
+ /** the message-passing interface rank source
+ * Must be >=0 and <= MPI_Comm_size()-1 */
+ Rank m_source;
+
+ int m_sourceActor;
+ int m_destinationActor;
+
+ int m_routingSource;
+ int m_routingDestination;
+
+ void initialize();
+public:
+ Message();
+ ~Message();
+ Message(MessageUnit * b,int c,Rank dest,MessageTag tag,Rank source);
+ MessageUnit *getBuffer();
+ char * getBufferBytes();
+ int getCount() const;
+/**
+ * Returns the destination MPI rank
+ */
+ Rank getDestination() const;
+
+/**
+ * Returns the message tag (RAY_MPI_TAG_something)
+ */
+ MessageTag getTag() const;
+/**
+ * Gets the source MPI rank
+ */
+ Rank getSource() const;
+
+ void print();
+
+ void setBuffer(void*buffer);
+
+ void setTag(MessageTag tag);
+
+ void setSource(Rank source);
+
+ void setDestination(Rank destination);
+
+ void setCount(int count);
+ void setNumberOfBytes(int bytes);
+ int getNumberOfBytes() const;
+
+ // actor model endpoints
+
+
+ bool isActorModelMessage() const;
+
+ int getDestinationActor() const;
+ int getSourceActor() const;
+ void setSourceActor(int sourceActor);
+ void setDestinationActor(int destinationActor);
+ void saveActorMetaData();
+ void loadActorMetaData();
+ int getMetaDataSize() const;
+ void printActorMetaData();
+
+ void setRoutingSource(int source);
+ void setRoutingDestination(int destination);
+ void saveRoutingMetaData();
+
+ int getRoutingSource() const;
+ int getRoutingDestination() const;
+ void loadRoutingMetaData();
+ void displayMetaData();
+
+};
+
+#endif
diff --git a/RayPlatform/RayPlatform/communication/MessageQueue.cpp b/RayPlatform/RayPlatform/communication/MessageQueue.cpp
new file mode 100644
index 0000000..5d4da08
--- /dev/null
+++ b/RayPlatform/RayPlatform/communication/MessageQueue.cpp
@@ -0,0 +1,291 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2012 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform: a message-passing development framework
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#include "MessageQueue.h"
+
+#include <iostream>
+using namespace std;
+
+#ifdef ASSERT
+#include <assert.h>
+#endif /* ASSERT */
+
+#ifdef CONFIG_COMPARE_AND_SWAP
+
+#if defined(__INTEL_COMPILER) || defined(__GNUC__)
+/* The compiler provides what we need */
+
+#elif defined( __APPLE__)
+#include <libkern/OSAtomic.h>
+
+#elif defined(_WIN32)
+#include <windows.h>
+
+#endif /* _WIN32 */
+
+#endif /* CONFIG_COMPARE_AND_SWAP */
+
+void MessageQueue::constructor(uint32_t bins){
+
+ #ifdef ASSERT
+ assert(bins);
+ assert(bins>0);
+ #endif /* ASSERT */
+
+/*
+ * For the non-blocking lockless algorithm, we need an extra slot.
+ */
+ m_size=bins+1;
+ m_headForPopOperations=0;
+ m_tailForPushOperations=0;
+
+ m_ring=(Message*)__Malloc(m_size*sizeof(Message),"/MessageQueue",false);
+
+ m_dead=false;
+
+ #ifdef ASSERT
+ assert(m_ring!=NULL);
+ assert(m_headForPopOperations<m_size);
+ assert(m_tailForPushOperations<m_size);
+ #endif /* ASSERT */
+
+#ifdef CONFIG_USE_LOCKING
+#ifdef CONFIG_USE_SPINLOCK
+ pthread_spin_init(&m_lock,PTHREAD_PROCESS_PRIVATE);
+#endif /* CONFIG_USE_SPINLOCK */
+
+#ifdef CONFIG_USE_MUTEX
+ pthread_mutex_init(&m_lock,NULL);
+#endif /* CONFIG_USE_MUTEX */
+#endif /* CONFIG_USE_LOCKING */
+
+}
+
+/*
+ * push() is thread-safe if only one thread calls it.
+ * The calling thread can be different from the other thread
+ * that is calling pop().
+ */
+bool MessageQueue::push(Message*message){
+
+ uint32_t nextTail=increment(m_tailForPushOperations); // not atomic
+
+/*
+ * The queue is full.
+ * Maybe there is a pop() call in progress, but the new head
+ * has not been published yet so we must wait.
+ */
+ if(nextTail==m_headForPopOperations)
+ return false;
+
+ m_ring[m_tailForPushOperations]=*message; // not atomic
+
+/*
+ * Publish the change to other threads.
+ * This must be atomic.
+ * Instructions must be executed in-order otherwise,
+ * m_tailForPushOperations will publish a draft of the message.
+ */
+ m_tailForPushOperations=nextTail; // likely atomic if sizeof(SystemBus) >= 32 bits
+
+ return true;
+}
+
+/**
+ * \see http://www.codeproject.com/Articles/43510/Lock-Free-Single-Producer-Single-Consumer-Circular
+ */
+bool MessageQueue::isFull(){
+ return increment(m_tailForPushOperations) == m_headForPopOperations;
+}
+
+/*
+ * Pop is thread-safe if only one thread calls it.
+ * It can be a thread different from the one that calls
+ * push().
+ */
+bool MessageQueue::pop(Message*message){
+
+/*
+ * We can not pop from am empty list.
+ */
+ if(m_headForPopOperations==m_tailForPushOperations)
+ return false;
+
+ *message=m_ring[m_headForPopOperations]; // not atomic
+
+ uint32_t nextHead=increment(m_headForPopOperations); // not atomic
+
+/*
+ * Publish the change.
+ * Instructions must be executed in-order otherwise,
+ * m_tailForPushOperations will publish a draft of the message.
+ */
+ m_headForPopOperations=nextHead; // likely atomic if sizeof(SystemBus) >= 32 bits
+
+ return true;
+}
+
+bool MessageQueue::hasContent(){
+ return m_headForPopOperations!=m_tailForPushOperations;
+}
+
+void MessageQueue::destructor(){
+ if(m_size==0)
+ return;
+
+ #ifdef ASSERT
+ assert(m_size>0);
+ assert(m_ring!=NULL);
+ #endif /* ASSERT */
+
+ m_size=0;
+ m_headForPopOperations=0;
+ m_tailForPushOperations=0;
+ __Free(m_ring,"/MessageQueue",false);
+ m_ring=NULL;
+
+ #ifdef ASSERT
+ assert(m_size==0);
+ assert(m_ring==NULL);
+ #endif /* ASSERT */
+
+#ifdef CONFIG_USE_LOCKING
+
+#ifdef CONFIG_USE_SPINLOCK
+ pthread_spin_destroy(&m_lock);
+#endif /* CONFIG_USE_SPINLOCK */
+
+#ifdef CONFIG_USE_MUTEX
+ pthread_mutex_destroy(&m_lock);
+#endif /* CONFIG_USE_MUTEX */
+
+#endif /* CONFIG_USE_LOCKING */
+}
+
+bool MessageQueue::isDead(){
+ return m_dead;
+}
+
+#ifdef CONFIG_USE_LOCKING
+
+/*
+ * The following 3 locking methods
+ * are not compiled if locking is not enabled.
+ */
+
+void MessageQueue::lock(){
+#ifdef CONFIG_USE_SPINLOCK
+ pthread_spin_lock(&m_lock);
+#elif defined(CONFIG_USE_MUTEX)
+ pthread_mutex_lock(&m_lock);
+#endif /* CONFIG_USE_MUTEX */
+}
+
+bool MessageQueue::tryLock(){
+#ifdef CONFIG_USE_SPINLOCK
+ return pthread_spin_trylock(&m_spinlock)==0;
+#elif defined(CONFIG_USE_MUTEX)
+ pthread_mutex_trylock(&m_lock);
+#endif /* CONFIG_USE_MUTEX */
+}
+
+void MessageQueue::unlock(){
+#ifdef CONFIG_USE_SPINLOCK
+ pthread_spin_unlock(&m_spinlock);
+#elif defined(CONFIG_USE_MUTEX)
+ pthread_mutex_unlock(&m_lock);
+#endif /* CONFIG_USE_MUTEX */
+}
+
+#endif /* CONFIG_USE_LOCKING */
+
+void MessageQueue::sendKillSignal(){
+ m_dead=true;
+}
+
+uint32_t MessageQueue::increment(uint32_t index){
+
+ #ifdef ASSERT
+ assert(index<m_size);
+ assert(m_size!=0);
+ assert(index>=0);
+ #endif /* ASSERT */
+
+ return (index+1)%m_size;
+}
+
+/*
+ * The code below is not used at the moment, but could be an
+ * alternative to CONFIG_USE_LOCKING for some architectures.
+ */
+#ifdef CONFIG_COMPARE_AND_SWAP
+bool CompareAndSwap(uint32_t*memory,uint32_t oldValue,uint32_t newValue){
+ #ifdef __INTEL_COMPILER
+/*
+ * Intel provides a compatibility layer with gcc it seems.
+ * \see Intel® C++ Intrinsic Reference, p.165 (Document Number: 312482-003US)
+ * \see http://software.intel.com/sites/default/files/m/9/4/c/8/e/18072-347603.pdf
+ */
+ return __sync_bool_compare_and_swap(memory,oldValue,newValue);
+
+ #elif defined(__GNUC__)
+/*
+ * gcc provides nice builtins.
+ * http://gcc.gnu.org/onlinedocs/gcc-4.1.1/gcc/Atomic-Builtins.html
+ */
+ return __sync_bool_compare_and_swap(memory,oldValue,newValue);
+
+ #elif defined(_WIN32)
+/*
+ * This code is not tested. From the documentation, the ordering is different.
+ * The documentation is from http://msdn.microsoft.com/en-us/library/windows/desktop/ms683560%28v=vs.85%29.aspx
+ * There is also an intrisic called _InterlockedCompareExchange.
+ */
+ InterlockedCompareExchange(memory,newValue,oldValue);
+/*
+ * This Microsoft API is retarded, InterlockedCompareExchange returns the value of *memory
+ * before the call, which is ridiculous because we need to know if the compare-and-swap
+ * was successful regardless.
+ */
+ return *memory==newValue;
+
+ #elif defined(BLUE_GENE_Q)
+
+/*
+ * For the Blue Gene /Q, the name is the same as in gcc.
+ * \see IBM XL C/C++ for Blue Gene/Q, V12.1, p. 484
+ * \see https://support.scinet.utoronto.ca/wiki/images/d/d5/Bgqccompiler.pdf
+ */
+ return __sync_bool_compare_and_swap(memory,oldValue,newValue);
+
+ #else
+/*
+ * Provide a dummy implementation that will fail sometimes I guess.
+ */
+ if(*memory!=oldValue)
+ return false;
+ *memory=newValue
+ return true;
+ #endif
+}
+
+#endif /* CONFIG_COMPARE_AND_SWAP */
+
diff --git a/RayPlatform/RayPlatform/communication/MessageQueue.h b/RayPlatform/RayPlatform/communication/MessageQueue.h
new file mode 100644
index 0000000..e461ae3
--- /dev/null
+++ b/RayPlatform/RayPlatform/communication/MessageQueue.h
@@ -0,0 +1,169 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2012 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform: a message-passing development framework
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#ifndef _MessageQueue_h
+#define _MessageQueue_h
+
+#include "Message.h"
+
+#include <RayPlatform/memory/allocator.h> /* for __Malloc and __Free */
+
+/*
+ * For uint32_t
+ * Heck, even Microsoft provides this, at least in MSVC 2010.
+ *
+ * \see http://stackoverflow.com/questions/126279/c99-stdint-h-header-and-ms-visual-studio
+ */
+#include <stdint.h>
+
+//#define CONFIG_USE_LOCKING
+//#define CONFIG_USE_MUTEX
+//#define CONFIG_USE_SPINLOCK
+
+#ifdef CONFIG_USE_LOCKING
+#if defined(CONFIG_USE_MUTEX) || defined(CONFIG_USE_SPINLOCK)
+#include <pthread.h>
+#endif
+#endif /* CONFIG_USE_LOCKING */
+
+/**
+ * \brief This is a message queue as its name implies.
+ *
+ * This class is used by the MPI ranks in the mini-ranks model.
+ * The buffers are allocated by the RingAllocator of the corresponding
+ * mini-rank.
+ *
+ * *************************************************************
+ * Here is a user story (where the user is a processor !):
+ *
+ * Number of MPI ranks: 8 (one thread each)
+ * Number of mini-ranks per MPI rank: 23 (one thread each)
+ * Total number of mini-ranks: 8*23 = 184
+ *
+ * - Mini-rank 4 ([0,4]) pushes something for mini-rank 101 ([4,9]) in its StaticVector outbox [NO SPINLOCK] /in the application
+ * - the ComputeCore object of mini-rank 4 pushes the something in its MessageQueue outbox [SPINLOCK] #1 in RayPlatform land (rare)
+ * - the MessagesHandler of RankProcess 0 pushes it with MPI_Isend (src: [0,4], dest: [4,9]) [SPINLOCK] #2 in RayPlatform land (frequent)
+ * - the MessagesHandler of RankProcess 4 pops it with MPI_Iprobe/MPI_Recv [SPINLOCK] #3 in RayPlatform land (rare)
+ * - RankProcess 4 pushes the messages in the MessageQueue inbox of [4,9] (continued)
+ * - Mini-rank 101 pops from its MessageQueue inbox and pushes it into its StaticVector inbox [SPINLOCK] #4 in RayPlatform land (frequent)
+ * - Mini-rank 101 consumes its inbox message [NO SPINLOCK] /in the application
+ *
+ * *************************************************************
+ *
+ * The most important thing is that there are no global lock.
+ *
+ * Locks:
+ *
+ * - a MPI rank needs to lock the MessageQueue outbox of every mini-rank to check for messages.
+ * - a MPI rank does not lock any MessageQueue inbox unless MPI_Iprobe was productive.
+ * - a mini-rank does not lock its MessageQueue outbox unless it has something in its StaticVector outbox
+ * - a mini-rank needs to lock its MessageQueue inbox to check for messages.
+ *
+ * *************************************************************
+ * So this is really a nice design (by Élénie Godzaridids) because
+ *
+ * 1. a MPI rank locks/unlocks endlessly MessageQueue outbox objects, but mini-ranks do
+ * not care unless they need to push a message.
+ * 2. a mini-rank locks/unlocks endlessly its MessageQueue inbox object, but the MPI rank
+ * does not care because it will only lock/unlock it when MPI_Iprobe is productive, which
+ * does not happen so often.
+ *
+ *
+ * => So nobody fight over the same thing really.
+ *
+ * *************************************************************
+ *
+ * The last thing to think about is who is responsible for managing the dirty/available slots
+ * for the RingAllocator objects. Clearly, it is the RankProcess object because MiniRank are
+ * not allowed to call MPI.
+ *
+ * head tail
+ * pop here push here
+ * * *
+ * ---> --->
+ * -------------------------------------------------------------------->
+ *
+ *
+ * \author Élénie Godzaridis (design)
+ * \author Sébastien Boisvert (programming & design)
+ *
+ * \see http://www.codeproject.com/Articles/43510/Lock-Free-Single-Producer-Single-Consumer-Circular
+ *
+ * There is also a Ph.D. thesis by Yi Zhang in 2003 about non-blocking algorithms:
+ *
+ * \see http://www.cse.chalmers.se/~tsigas/papers/Yi-Thesis.pdf
+ */
+class MessageQueue{
+
+#ifdef CONFIG_USE_LOCKING
+
+#if defined(CONFIG_USE_MUTEX)
+ pthread_mutex_t m_lock;
+#elif defined(CONFIG_USE_SPINLOCK)
+ pthread_spinlock_t m_lock;
+#endif /* CONFIG_USE_SPINLOCK */
+
+#endif /* CONFIG_USE_LOCKING */
+
+ Message*m_ring;
+ uint32_t m_size;
+
+/*
+ * The volatile keyword means:
+ *
+ * " Generally speaking, the volatile keyword is intended to prevent the compiler from applying any
+ * optimizations on the code that assume values of variables cannot change "on their own."
+ *
+ * This is necessary to tell the compiler to avoid any reordering of instructions for this
+ * variable.
+ *
+ * \see http://en.wikipedia.org/wiki/Volatile_variable
+ */
+ volatile uint32_t m_headForPopOperations;
+ volatile uint32_t m_tailForPushOperations;
+
+ bool m_dead;
+
+ uint32_t increment(uint32_t index);
+
+public:
+
+ void constructor(uint32_t bins);
+
+ bool push(Message*message);
+ bool pop(Message*message);
+
+ void destructor();
+ bool hasContent();
+
+#ifdef CONFIG_USE_LOCKING
+ void lock();
+ void tryLock();
+ void unlock();
+#endif /* CONFIG_USE_LOCKING */
+
+ void sendKillSignal();
+ bool isDead();
+ bool isFull();
+};
+
+#endif /* _MessageQueue_h */
+
diff --git a/RayPlatform/communication/MessageRouter.cpp b/RayPlatform/RayPlatform/communication/MessageRouter.cpp
similarity index 78%
rename from RayPlatform/communication/MessageRouter.cpp
rename to RayPlatform/RayPlatform/communication/MessageRouter.cpp
index 907109a..9576b48 100644
--- a/RayPlatform/communication/MessageRouter.cpp
+++ b/RayPlatform/RayPlatform/communication/MessageRouter.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -14,7 +14,7 @@
GNU Lesser General Public License for more details.
You have received a copy of the GNU Lesser General Public License
- along with this program (lgpl-3.0.txt).
+ along with this program (lgpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
@@ -28,17 +28,19 @@
* \reviewedBy Elénie Godzaridis 2011-11-05
*/
-#include <communication/MessageRouter.h>
+#include "MessageRouter.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/core/ComputeCore.h>
+
+#include <time.h> /* for time */
#include <string.h> /* for memcpy */
#include <assert.h>
-#include <core/OperatingSystem.h>
-#include <core/ComputeCore.h>
-#include <time.h> /* for time */
using namespace std;
-/*
- * According to the MPI standard, MPI_TAG_UB is >= 32767 (2^15-1)
- * Therefore, the tag must be >=0 and <= 32767
+/*
+ * According to the MPI standard, MPI_TAG_UB is >= 32767 (2^15-1)
+ * Therefore, the tag must be >=0 and <= 32767
* In most applications, there will be not that much tag
* values.
* 2^14 = 16384
@@ -48,7 +50,7 @@ using namespace std;
* A routing tag is >= 16384 and the corresponding real tag is
* tag - 16384.
*/
-#define __ROUTING_TAG_BASE 16384
+#define __ROUTING_TAG_BASE 16384
#define __ROUTING_SOURCE 0
#define __ROUTING_DESTINATION 1
@@ -82,6 +84,7 @@ void MessageRouter::routeOutcomingMessages(){
if(isRoutingTag(communicationTag)){
#ifdef CONFIG_ROUTING_VERBOSITY
cout<<"["<<__func__<<"] Message has already a routing tag."<<endl;
+ aMessage->displayMetaData();
#endif
continue;
}
@@ -94,12 +97,13 @@ void MessageRouter::routeOutcomingMessages(){
if(m_graph.isConnected(trueSource,trueDestination)){
#ifdef CONFIG_ROUTING_VERBOSITY
- cout<<"["<<__func__<<"] Rank "<<trueSource<<" can reach "<<trueDestination<<" without routing"<<endl;
+ cout<<"["<<__func__<<"] Rank "<<trueSource<<" can reach "<<trueDestination<<" without routing";
+ cout << " payload size: " << aMessage->getCount() << endl;
#endif
continue;
}
-
+
// re-route the message by re-writing the tag
MessageTag routingTag=getRoutingTag(communicationTag);
aMessage->setTag(routingTag);
@@ -132,26 +136,45 @@ void MessageRouter::routeOutcomingMessages(){
}
// routing information is stored in 64 bits
- int newCount=aMessage->getCount()+1;
+ int newCount=aMessage->getCount();
aMessage->setCount(newCount);
-
+
#ifdef ASSERT
assert(buffer!=NULL);
#endif
+ aMessage->setRoutingSource(trueSource);
+ aMessage->setRoutingDestination(trueDestination);
+ aMessage->saveRoutingMetaData();
+
+ /*
setSourceInBuffer(buffer,newCount,trueSource);
setDestinationInBuffer(buffer,newCount,trueDestination);
+ */
#ifdef ASSERT
- assert(getSourceFromBuffer(aMessage->getBuffer(),aMessage->getCount())==trueSource);
- assert(getDestinationFromBuffer(aMessage->getBuffer(),aMessage->getCount())==trueDestination);
+ assert(aMessage->getRoutingSource() ==trueSource);
+ assert(aMessage->getRoutingDestination() ==trueDestination);
+ assert(trueSource >= 0);
+ assert(trueDestination >= 0);
+ assert(trueSource < m_size);
+ assert(trueDestination < m_size);
#endif
Rank nextRank=m_graph.getNextRankInRoute(trueSource,trueDestination,m_rank);
aMessage->setDestination(nextRank);
+#ifdef CONFIG_ASSERT
+ assert(nextRank >= 0);
+ assert(nextRank < m_size);
+#endif
+
#ifdef CONFIG_ROUTING_VERBOSITY
- cout<<"["<<__func__<<"] relayed message, trueSource="<<trueSource<<" trueDestination="<<trueDestination<<" to intermediateSource "<<nextRank<<endl;
+ cout<<"["<<__func__<<"] relayed message, trueSource="<<trueSource;
+ cout << "count= " << aMessage->getCount();
+ cout <<" trueDestination="<<trueDestination<<" to intermediateSource "<<nextRank;
+ aMessage->displayMetaData();
+ cout <<endl;
#endif
}
@@ -167,7 +190,7 @@ void MessageRouter::routeOutcomingMessages(){
}
/**
- * route incoming messages
+ * route incoming messages
* \returns true if rerouted something.
*/
bool MessageRouter::routeIncomingMessages(){
@@ -182,16 +205,11 @@ bool MessageRouter::routeIncomingMessages(){
return false;
// otherwise, we have exactly one precious message.
-
+
Message*aMessage=m_inbox->at(0);
+
MessageTag tag=aMessage->getTag();
- MessageUnit*buffer=aMessage->getBuffer();
- int count=aMessage->getCount();
- #ifdef CONFIG_ROUTING_VERBOSITY
- uint8_t printableTag=tag;
- cout<<"[routeIncomingMessages] tag= "<<MESSAGE_TAGS[printableTag]<<" value="<<tag<<endl;
- #endif
// if the message has no routing tag, then we can safely receive it as is
if(!isRoutingTag(tag)){
@@ -203,8 +221,31 @@ bool MessageRouter::routeIncomingMessages(){
return false;
}
+ //cout << "DEBUG message was sent by " << aMessage->getSource();
+
+ //aMessage->displayMetaData();
+
+ // at this point, we know there is a routing activity to perform.
+ //MessageUnit*buffer=aMessage->getBuffer();
+ aMessage->loadRoutingMetaData();
+
+ #ifdef CONFIG_ROUTING_VERBOSITY
+ int count=aMessage->getCount();
+ uint8_t printableTag=tag;
+ cout<<"[routeIncomingMessages] tag= "<<MESSAGE_TAGS[printableTag]<<" value="<<tag;
+ cout << " count= " << count <<endl;
+ #endif
+
+ /*
Rank trueSource=getSourceFromBuffer(buffer,count);
Rank trueDestination=getDestinationFromBuffer(buffer,count);
+ */
+
+ Rank trueSource = aMessage->getRoutingSource();
+ Rank trueDestination = aMessage->getRoutingDestination();
+
+ //cout << "DEBUG after loading metadata: trueDestination= ";
+ //cout << trueDestination << " trueSource " << trueSource << endl;
// this is the final destination
// we have received the message
@@ -216,7 +257,7 @@ bool MessageRouter::routeIncomingMessages(){
// we must update the original source and original tag
aMessage->setSource(trueSource);
-
+
// the original destination is already OK
#ifdef ASSERT
assert(aMessage->getDestination()==m_rank);
@@ -230,7 +271,7 @@ bool MessageRouter::routeIncomingMessages(){
#endif
// remove the routing stuff
- int newCount=aMessage->getCount()-1;
+ int newCount=aMessage->getCount();
#ifdef ASSERT
assert(newCount>=0);
@@ -249,6 +290,12 @@ bool MessageRouter::routeIncomingMessages(){
assert(m_rank!=trueDestination);
#endif
+#ifdef CONFIG_ASSERT
+ assert(trueDestination >= 0 && trueDestination < m_size);
+ assert(trueSource >= 0 && trueSource < m_size);
+ assert(m_rank >= 0 && m_rank < m_size);
+#endif
+
// at this point, we know that we need to forward
// the message to another peer
Rank nextRank=m_graph.getNextRankInRoute(trueSource,trueDestination,m_rank);
@@ -257,7 +304,16 @@ bool MessageRouter::routeIncomingMessages(){
cout<<"["<<__func__<<"] message has been sent to the next one, trueSource="<<trueSource<<" trueDestination= "<<trueDestination;
cout<<" Previous= "<<aMessage->getSource()<<" Current= "<<m_rank<<" Next= "<<nextRank<<endl;
#endif
-
+
+#ifdef CONFIG_ASSERT
+ assert(trueSource >= 0);
+ assert(trueDestination >= 0);
+ assert(trueSource < m_size);
+ assert(trueDestination < m_size);
+ assert(nextRank >= 0);
+ assert(nextRank < m_size);
+#endif
+
// we forward the message
relayMessage(aMessage,nextRank);
@@ -269,7 +325,7 @@ bool MessageRouter::routeIncomingMessages(){
* forward a message to follow a route
*/
void MessageRouter::relayMessage(Message*message,Rank destination){
-
+
int count=message->getCount();
// routed messages always have a payload
@@ -284,8 +340,8 @@ void MessageRouter::relayMessage(Message*message,Rank destination){
message->setBuffer(outgoingMessage);
#ifdef CONFIG_ROUTING_VERBOSITY
- cout<<"[relayMessage] TrueSource="<<getSourceFromBuffer(message->getBuffer(),message->getCount());
- cout<<" TrueDestination="<<getDestinationFromBuffer(message->getBuffer(),message->getCount());
+ cout<<"[relayMessage] TrueSource="<< message->getRoutingSource();
+ cout<<" TrueDestination="<< message->getRoutingDestination();
cout<<" RelaySource="<<m_rank<<" RelayDestination="<<destination<<endl;
#endif
@@ -293,11 +349,14 @@ void MessageRouter::relayMessage(Message*message,Rank destination){
message->setSource(m_rank);
message->setDestination(destination);
+ // package routing metadata for the delivery onto the network !!!
+ message->saveRoutingMetaData();
+
#ifdef ASSERT
assert(m_graph.isConnected(m_rank,destination));
#endif
- m_outbox->push_back(*message);
+ m_outbox->push_back(message);
}
@@ -328,11 +387,14 @@ void MessageRouter::enable(StaticVector*inbox,StaticVector*outbox,RingAllocator*
m_outboxAllocator=outboxAllocator;
m_rank=rank;
m_enabled=true;
+ m_deletionTime=0;
- if(m_rank==MASTER_RANK)
- m_graph.writeFiles(prefix);
+ m_prefix=prefix;
+}
- m_deletionTime=0;
+void MessageRouter::writeFiles(){
+ if(m_rank==MASTER_RANK)
+ m_graph.writeFiles(m_prefix);
}
bool MessageRouter::hasCompletedRelayEvents(){
@@ -411,10 +473,24 @@ Rank MessageRouter::getDestinationFromBuffer(MessageUnit*buffer,int count){
assert(buffer!=NULL);
#endif
- uint32_t*routingInformation=(uint32_t*)(buffer+__ROUTING_OFFSET(count));
+ int offset = __ROUTING_OFFSET(count);
+
+ uint32_t*routingInformation=(uint32_t*)(buffer+ offset);
Rank rank=routingInformation[__ROUTING_DESTINATION];
+ /*
+ int * bufferAsInteger = (int*)buffer;
+ cout << "DEBUG getDestinationFromBuffer count " << count;
+ cout << " offset " << offset ;
+ cout << " destination " << rank << endl;
+
+ for(int i = 0 ; i < 4 ; ++i) {
+ cout << "DEBUG buffer [ " << i << "] -> ";
+ cout << bufferAsInteger[i] << endl;
+ }
+ */
+
#ifdef ASSERT
assert(rank>=0);
assert(rank<m_size);
diff --git a/RayPlatform/communication/MessageRouter.h b/RayPlatform/RayPlatform/communication/MessageRouter.h
similarity index 86%
rename from RayPlatform/communication/MessageRouter.h
rename to RayPlatform/RayPlatform/communication/MessageRouter.h
index 43633cd..24ca807 100644
--- a/RayPlatform/communication/MessageRouter.h
+++ b/RayPlatform/RayPlatform/communication/MessageRouter.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,13 +22,12 @@
#ifndef _MessageRouter_h
#define _MessageRouter_h
-#include <memory/RingAllocator.h>
-#include <structures/StaticVector.h>
-#include <communication/Message.h>
-#include <routing/ConnectionGraph.h>
-#include <core/types.h>
+#include "Message.h"
-class ComputeCore;
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/routing/ConnectionGraph.h>
+#include <RayPlatform/core/types.h>
#include <string>
#include <map>
@@ -36,15 +35,18 @@ class ComputeCore;
#include <time.h> /* for time() */
using namespace std;
+class ComputeCore;
+
/**
- * \author Sébastien Boisvert 2011-11-04
- * \reviewedBy Elénie Godzaridis 2011-11-05
- *
* the MessageRouter makes communication more efficient.
*
+ * \author Sébastien Boisvert 2011-11-04
+ * \reviewedBy Elénie Godzaridis 2011-11-05
*/
class MessageRouter {
+ string m_prefix;
+
/** the extra ticks after the computation has completed. **/
time_t m_deletionTime;
@@ -84,8 +86,6 @@ class MessageRouter {
void relayMessage(Message*message,Rank destination);
- bool isRoutingTag(MessageTag tag);
-
MessageTag getMessageTagFromRoutingTag(MessageTag tag);
Rank getSourceFromBuffer(MessageUnit*buffer,int count);
Rank getDestinationFromBuffer(MessageUnit*buffer,int count);
@@ -127,6 +127,14 @@ string prefix,int numberOfRanks,string type,int degree);
bool hasCompletedRelayEvents();
ConnectionGraph*getGraph();
+
+/**
+ * Write routing information files.
+ */
+ void writeFiles();
+
+ bool isRoutingTag(MessageTag tag);
+
};
#endif /* _MessageRouter_h */
diff --git a/RayPlatform/communication/MessagesHandler.cpp b/RayPlatform/RayPlatform/communication/MessagesHandler.cpp
similarity index 53%
rename from RayPlatform/communication/MessagesHandler.cpp
rename to RayPlatform/RayPlatform/communication/MessagesHandler.cpp
index 79184d0..fb9a403 100644
--- a/RayPlatform/communication/MessagesHandler.cpp
+++ b/RayPlatform/RayPlatform/communication/MessagesHandler.cpp
@@ -1,8 +1,8 @@
/*
- RayPlatform
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,13 +19,13 @@
*/
-#include <communication/MessagesHandler.h>
+#include "MessagesHandler.h"
+#include "MessageRouter.h"
/* for decoding message tags */
-#include <communication/MessageRouter.h>
-#include <memory/allocator.h>
-#include <core/OperatingSystem.h>
-#include <core/ComputeCore.h>
+#include <RayPlatform/memory/allocator.h>
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <fstream>
#include <assert.h>
@@ -35,147 +35,108 @@
using namespace std;
// #define COMMUNICATION_IS_VERBOSE
+//#define CONFIG_DEBUG_MPI_RANK
+//#define CONFIG_DEBUG_MINI_RANK_COMMUNICATION
-#define __NOT_SET -1
+#define CONFIG_MESSAGE_QUEUE_RETRY_WARNING 1024
+
+#ifdef CONFIG_MINI_RANKS
/**
- * TODO Instead of a true/false state, increase and decrease requests
- * using a particular buffer. Otherwise, there may be a problem when
- * a buffer is re-used several times for many requests.
+ *
*/
-void MessagesHandler::checkDirtyBuffer(RingAllocator*outboxBufferAllocator,int index){
-
- #ifdef ASSERT
- assert(m_numberOfDirtyBuffers>0);
- #endif
-
- if(m_dirtyBuffers[index].m_buffer==NULL)// this entry is empty...
- return;
+void MessagesHandler::sendAndReceiveMessagesForRankProcess(ComputeCore**cores,int miniRanksPerRank,bool*communicate){
- // check the buffer and free it if it is finished.
- MPI_Status status;
- MPI_Request*request=&(m_dirtyBuffers[index].m_messageRequest);
+ int deadMiniRanks=0;
- int flag=0;
+ for(int i=0;i<miniRanksPerRank;i++){
- MPI_Test(request,&flag,&status);
+ ComputeCore*core=cores[i];
+ MessageQueue*outbox=core->getBufferedOutbox();
- if(!flag)// this slot is not ready
- return;
-
- #ifdef ASSERT
- assert( flag );
- #endif /* ASSERT */
+#ifdef CONFIG_USE_LOCKING
+ outbox->lock();
+#endif /* CONFIG_USE_LOCKING */
- void*buffer=m_dirtyBuffers[index].m_buffer;
- outboxBufferAllocator->salvageBuffer(buffer);
- m_numberOfDirtyBuffers--;
+ if(outbox->isDead())
+ deadMiniRanks++;
- #ifdef COMMUNICATION_IS_VERBOSE
- cout<<"From checkDirtyBuffer flag= "<<flag<<endl;
- #endif /* COMMUNICATION_IS_VERBOSE */
-
- #ifdef ASSERT
- assert(*request == MPI_REQUEST_NULL);
- #endif /* ASSERT */
+ #ifdef CONFIG_DEBUG_MPI_RANK
+ cout<<"[MessagesHandler] mini-rank # "<<i<<" has outbox messages"<<endl;
+ #endif
- m_dirtyBuffers[index].m_buffer=NULL;
-}
+/*
+ * The RingAllocator outbox allocator is not thread safe -- the MessagesHandler should
+ * have its own outbox allocator. Also, it is unclear at this point how the mini-rank should manage
+ * its buffers. Perhaps it should also use the RingAllocator, but without anything related to dirty
+ * buffers, just a rotating staircase round robin strategy.
+ *
+ * Messages are sent using the buffer system of the mini-rank.
+ */
+ if(outbox->hasContent()){
+ sendMessagesForMiniRank(outbox,core->getBufferedOutboxAllocator(),miniRanksPerRank);
+ }
-void MessagesHandler::cleanDirtyBuffers(RingAllocator*outboxBufferAllocator){
+#ifdef CONFIG_USE_LOCKING
+ outbox->unlock();
+#endif /* CONFIG_USE_LOCKING */
-/**
- * don't do any linear sweep if we still have plenty of free cells
+/*
+ * The message reception is interleaved
+ * with the send operations.
+ *
+ * This is the secret sauce of "mini-ranks" implementation in
+ * RayPlatform.
+ *
+ * The obvious model would be to do:
+ *
+ * while(1){
+ * receive();
+ * send();
+ * }
+ *
+ * However, receive() is O(1) because it receives at most 1 message
+ * during 1 single call.
+ *
+ * A call to send() requires a loop over all the mini-ranks. Therefore,
+ * send() is O(<number of cores>).
+ *
+ * The trick is therefore to do this instead:
+ *
+ * while(1){
+ * for each mini-rank)owever, receive() is O(1) because it receives at most 1 message
+ * during 1 single call.
+ *
+ * A call to send() requires a loop over all the mini-ranks. Therefore,
+ * send() is O(<number of cores>).
+ *
+ * The trick is therefore to do this instead:
+ *
+ * while(1){
+ * for(each mini-rank X){
+ * send message of mini-rank X
+ * receive()
+ * }
+ * }
+ *
*/
- if(m_numberOfDirtyBuffers<m_minimumNumberOfDirtyBuffersForSweep)
- return;
+ receiveMessagesForMiniRanks(cores,miniRanksPerRank);
+ }
- #ifdef ASSERT
- assert(m_numberOfDirtyBuffers>0);
+ #ifdef CONFIG_DEBUG_MPI_RANK
+ cout<<"[RankProcess::receiveMessages]"<<endl;
#endif
- m_linearSweeps++;
-
- // update the dirty buffer list.
- for(int i=0;i<m_dirtyBufferSlots;i++){
- if(m_numberOfDirtyBuffers==0)
- return;
-
- checkDirtyBuffer(outboxBufferAllocator,i);
- }
-
- if(m_numberOfDirtyBuffers>=m_minimumNumberOfDirtyBuffersForWarning){
- cout<<"[MessagesHandler] Warning: dirty buffers are still dirty after linear sweep."<<endl;
- printDirtyBuffers();
- }
+/*
+ * Since all mini-ranks died, it is no longer necessasry to do
+ * the communication.
+ */
+ if(deadMiniRanks==miniRanksPerRank)
+ (*communicate)=false;
}
-void MessagesHandler::printDirtyBuffers(){
-
- cout<<"[MessagesHandler] Dirty buffers: "<<m_numberOfDirtyBuffers<<"/";
- cout<<m_dirtyBufferSlots<<endl;
-
- for(int i=0;i<m_dirtyBufferSlots;i++){
- cout<<"DirtyBuffer # "<<i<<" State: ";
- if(m_dirtyBuffers[i].m_buffer==NULL){
- cout<<"Available"<<endl;
- }else{
- cout<<"Dirty"<<endl;
-
- MessageTag tag=m_dirtyBuffers[i].m_messageTag;
- Rank destination=m_dirtyBuffers[i].m_destination;
- Rank routingSource=m_rank;
- Rank routingDestination=destination;
-
- bool isRoutingTagValue=false;
-
-/* TODO we don't have easy access to this information */
-#if 0
- RoutingTag routingTag=tag;
-
- if(isRoutingTag(tag)){
- tag=getMessageTagFromRoutingTag(routingTag);
- routingSource=getSourceFromRoutingTag(routingTag);
- routingDestination=getDestinationFromRoutingTag(routingTag);
-
- isRoutingTagValue=true;
- }
#endif
- uint8_t index=tag;
- cout<<" MessageTag: "<<MESSAGE_TAGS[index]<<" ("<<(int)index<<") ";
- if(index<tag){
- cout<<"[this is a routing tag]"<<endl;
- }
- cout<<" Source: "<<m_rank<<endl;
- cout<<" Destination: "<<destination<<endl;
-
- if(isRoutingTagValue){
- cout<<" RoutingSource: "<<routingSource<<endl;
- cout<<" RoutingDestination: "<<routingDestination<<endl;
- }
- }
- }
-}
-
-void MessagesHandler::initializeDirtyBuffers(RingAllocator*outboxBufferAllocator){
-
- m_dirtyBufferSlots=outboxBufferAllocator->getNumberOfBuffers();
-
- m_dirtyBuffers=(DirtyBuffer*)__Malloc(m_dirtyBufferSlots*sizeof(DirtyBuffer),
- "m_dirtyBuffers",false);
-
- for(int i=0;i<m_dirtyBufferSlots;i++){
- m_dirtyBuffers[i].m_buffer=NULL;
- }
-
- // configure the real-time sweeper.
-
- m_minimumNumberOfDirtyBuffersForSweep=m_dirtyBufferSlots/4;
- m_minimumNumberOfDirtyBuffersForWarning=m_dirtyBufferSlots/2;
-
-}
-
/*
* send messages,
*
@@ -188,20 +149,34 @@ void MessagesHandler::initializeDirtyBuffers(RingAllocator*outboxBufferAllocator
* For very large jobs, this might need to be increased
* in order to avoid the situation in which
* all the buffer for the outbox are dirty/used.
+ *
+ * TODO: fix mini-rank code.
*/
-void MessagesHandler::sendMessages(StaticVector*outbox,RingAllocator*outboxBufferAllocator){
+void MessagesHandler::sendMessagesForMiniRank(MessageQueue*outbox,RingAllocator*outboxBufferAllocator,
+ int miniRanksPerRank){
// initialize the dirty buffer counters
// this is done only once
- if(m_dirtyBuffers==NULL)
- initializeDirtyBuffers(outboxBufferAllocator);
+ if(outboxBufferAllocator->getDirtyBuffers()==NULL)
+ outboxBufferAllocator->initializeDirtyBuffers();
- cleanDirtyBuffers(outboxBufferAllocator);
+ outboxBufferAllocator->cleanDirtyBuffers();
// send messages.
- for(int i=0;i<(int)outbox->size();i++){
- Message*aMessage=((*outbox)[i]);
- Rank destination=aMessage->getDestination();
+ while(outbox->hasContent()){
+ Message message;
+ Message*aMessage=&message;
+ outbox->pop(aMessage);
+
+ int miniRankDestination=aMessage->getDestination();
+ int miniRankSource=aMessage->getSource();
+
+ Rank destination=miniRankDestination/miniRanksPerRank;
+
+ #if 0
+ Rank source=miniRankSource/miniRanksPerRank;
+ #endif
+
void*buffer=aMessage->getBuffer();
int count=aMessage->getCount();
MessageTag tag=aMessage->getTag();
@@ -218,84 +193,71 @@ void MessagesHandler::sendMessages(StaticVector*outbox,RingAllocator*outboxBuffe
//assert(count<=(int)(MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit)));
#endif /* ASSERT */
- MPI_Request dummyRequest;
-
- MPI_Request*request=&dummyRequest;
-
- int handle=-1;
-
- /* compute the handle, this is O(1) */
- if(buffer!=NULL)
- handle=outboxBufferAllocator->getBufferHandle(buffer);
-
- bool mustRegister=false;
+/*
+ * It would obviously be better if the copy could be avoid
+ * by using the original buffer directly.
+ * But it is not possible because it is not thread-safe to do so.
+ */
+ MessageUnit*theBuffer=(MessageUnit*)outboxBufferAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+ memcpy(theBuffer,buffer,count*sizeof(MessageUnit));
- // this buffer is not registered.
- if(handle >=0 && m_dirtyBuffers[handle].m_buffer==NULL)
- mustRegister=true;
+/*
+ * Store minirank information too at the end.
+ */
+ theBuffer[count++]=miniRankSource;
+ theBuffer[count++]=miniRankDestination;
- /* register the buffer for processing */
- if(mustRegister){
- #ifdef ASSERT
- assert(m_dirtyBuffers[handle].m_buffer==NULL);
- #endif
+ #ifdef CONFIG_DEBUG_MINI_RANK_COMMUNICATION
+ cout<<"SEND Source= "<<miniRankSource<<" Destination= "<<miniRankDestination<<" Tag= "<<tag<<endl;
+ #endif /* CONFIG_DEBUG_MINI_RANK_COMMUNICATION */
- m_dirtyBuffers[handle].m_buffer=buffer;
- m_dirtyBuffers[handle].m_destination=destination;
- m_dirtyBuffers[handle].m_messageTag=tag;
- #ifdef ASSERT
- assert(m_dirtyBuffers[handle].m_buffer!=NULL);
- #endif
+ MPI_Request dummyRequest;
+ MPI_Request*request=&dummyRequest;
- request=&(m_dirtyBuffers[handle].m_messageRequest);
+/*
+ * Check if the buffer is already registered.
+ */
+ int bufferHandle=outboxBufferAllocator->getBufferHandle(theBuffer);
+ bool registeredAlready=outboxBufferAllocator->isRegistered(bufferHandle);
+/*
+ * Register the buffer as being dirty.
+ */
- /* this is O(1) */
- outboxBufferAllocator->markBufferAsDirty(buffer);
+ if(!registeredAlready) {
- m_numberOfDirtyBuffers++;
+ request = this->registerMessageBuffer(buffer, m_rank, destination,
+ tag, outboxBufferAllocator);
- // update the maximum number of dirty buffers
- // observed since the beginning.
- if(m_numberOfDirtyBuffers > m_maximumDirtyBuffers)
- m_maximumDirtyBuffers=m_numberOfDirtyBuffers;
}
#ifdef ASSERT
assert(request!=NULL);
+ assert(count>=2);
#endif
// MPI_Isend
// Synchronous nonblocking.
- MPI_Isend(buffer,count,m_datatype,destination,tag,MPI_COMM_WORLD,request);
-
- // if the buffer is NULL, we free the request right now
- // because we there is no buffer to protect.
- if(!mustRegister){
-
- #ifdef COMMUNICATION_IS_VERBOSE
- cout<<" From sendMessages"<<endl;
- #endif
-
- MPI_Request_free(request);
-
- #ifdef ASSERT
- assert(*request==MPI_REQUEST_NULL);
- #endif
-
- }
+ MPI_Isend(theBuffer,count,m_datatype,destination,tag,MPI_COMM_WORLD,request);
+ #if 0
// if the message was re-routed, we don't care
// we only fetch the tag for statistics
uint8_t shortTag=tag;
/** update statistics */
m_messageStatistics[destination*RAY_MPI_TAG_DUMMY+shortTag]++;
+ #endif
+
+/*
+ * We can free the dummy request because the buffer is already registered.
+ * TODO: instead, there should be a reference count for each buffer.
+ */
+ if(registeredAlready)
+ MPI_Request_free(request);
m_sentMessages++;
}
-
- outbox->clear();
}
#ifdef CONFIG_COMM_PERSISTENT
@@ -362,39 +324,131 @@ void MessagesHandler::pumpMessageFromPersistentRing(StaticVector*inbox,RingAlloc
#endif /* CONFIG_COMM_PERSISTENT */
-void MessagesHandler::receiveMessages(StaticVector*inbox,RingAllocator*inboxAllocator){
- #if defined CONFIG_COMM_IPROBE_ROUND_ROBIN
+#ifdef CONFIG_MINI_RANKS
-// round-robin reception seems to avoid starvation
-/** round robin with Iprobe will increase the latency because there will be a lot of calls to
- MPI_Iprobe that yield no messages at all */
+void MessagesHandler::receiveMessagesForMiniRanks(ComputeCore**cores,int miniRanksPerRank){
- roundRobinReception(inbox,inboxAllocator);
+ // the code here will probe from rank source
+ // with MPI_Iprobe
- #elif defined CONFIG_COMM_PERSISTENT
+ #ifdef COMMUNICATION_IS_VERBOSE
+ cout<<"call to probeAndRead source="<<source<<""<<endl;
+ #endif /* COMMUNICATION_IS_VERBOSE */
+
+ int flag=0;
+ MPI_Status status;
+ int source=MPI_ANY_SOURCE;
+ int tag=MPI_ANY_TAG;
- // use persistent communication
- pumpMessageFromPersistentRing(inbox,inboxAllocator);
+ MPI_Iprobe(source,tag,MPI_COMM_WORLD,&flag,&status);
- #elif defined CONFIG_COMM_IPROBE_ANY_SOURCE
+ // nothing to receive...
+ if(!flag)
+ return;
- // receive any message
- // it is assumed that MPI is fair
- // otherwise there may be some starvation
+/*
+ * The code below reads at most one message.
+ */
- probeAndRead(MPI_ANY_SOURCE,MPI_ANY_TAG,inbox,inboxAllocator);
+ MPI_Datatype datatype= m_datatype;
+ int actualTag=status.MPI_TAG;
+ Rank actualSource=status.MPI_SOURCE;
+ int count=-1;
+ MPI_Get_count(&status,datatype,&count);
- #elif defined CONFIG_COMM_IRECV_TESTANY
+ #ifdef ASSERT
+ assert(count >= 0);
+ assert(count >= 2);// we need mini-rank numbers !
+ #endif
- receiveMessages_irecv_testany(inbox,inboxAllocator);
- #else
+ MPI_Recv(m_staticBuffer,count,datatype,actualSource,actualTag,MPI_COMM_WORLD,&status);
- #error "No communication model is selected."
+/*
+ * Get the mini-rank source and mini-rank destination.
+ */
+ int miniRankSource=m_staticBuffer[count-2];
+ int miniRankDestination=m_staticBuffer[count-1];
+ #ifdef CONFIG_DEBUG_MINI_RANK_COMMUNICATION
+ cout<<"RECEIVE Source= "<<miniRankSource<<" Destination= "<<miniRankDestination<<" Tag= "<<actualTag<<endl;
+ #endif /* CONFIG_DEBUG_MINI_RANK_COMMUNICATION */
+
+ count-=2;
+
+ int miniRankIndex=miniRankDestination%miniRanksPerRank;
+
+ #ifdef CONFIG_DEBUG_MPI_RANK
+ cout<<"[MessagesHandler::probeAndRead] received message from "<<miniRanksPerRank<<" to "<<miniRankDestination;
+ cout<<" tag is "<<actualTag<<endl;
+ #endif
+
+ MessageUnit*incoming=NULL;
+
+ ComputeCore*core=cores[miniRankIndex];
+
+/*
+ * We can not receive a message if the inbox is not
+ * empty.
+ * Update: this is not true because we push the message
+ * directly in a MessageQueue.
+ */
+
+ MessageQueue*inbox=core->getBufferedInbox();
+
+#ifdef CONFIG_USE_LOCKING
+/*
+ * Lock the core and distribute the message.
+ */
+
+ #ifdef CONFIG_DEBUG_MPI_RANK
+ cout<<"Rank tries to lock the inbox of minirank # "<<miniRankIndex<<endl;
+ #endif
+
+ inbox->lock();
+#endif /* CONFIG_USE_LOCKING */
+
+ if(count > 0){
+ incoming=(MessageUnit*)core->getBufferedInboxAllocator()->allocate(count*sizeof(MessageUnit));
+
+ #ifdef ASSERT
+ assert(incoming!=NULL);
+ #endif
+
+ memcpy(incoming,m_staticBuffer,count*sizeof(MessageUnit));
+ }
+
+ Message aMessage(incoming,count,miniRankDestination,actualTag,miniRankSource);
+
+/*
+ * Try to push the message. If it does not work, just try again.
+ * It should work eventually.
+ */
+ int ticks=0;
+
+ while(!inbox->push(&aMessage)){
+
+ if(ticks%CONFIG_MESSAGE_QUEUE_RETRY_WARNING ==0)
+ cout<<"[MessagesHandler] Warning: inbox message queue is full, will retry in a bit... (#"<<ticks<<")"<<endl;
+
+ ticks++;
+ }
+
+#ifdef CONFIG_USE_LOCKING
+ inbox->unlock();
+#endif /* CONFIG_USE_LOCKING */
+
+ #ifdef ASSERT
+ // this assertion is not valid for mini-ranks.
+ //assert(aMessage.getDestination() == m_rank);
#endif
+
+ m_receivedMessages++;
+
}
+#endif
+
#ifdef CONFIG_COMM_IRECV_TESTANY
/*
@@ -486,7 +540,7 @@ void MessagesHandler::receiveMessages_irecv_testany(StaticVector*inbox,RingAlloc
if(hasCompleted==0)
return;
- MPI_Datatype datatype=MPI_UNSIGNED_LONG_LONG;
+ MPI_Datatype datatype= m_datatype;
MessageTag actualTag=status.MPI_TAG;
Rank actualSource=status.MPI_SOURCE;
int count=-1;
@@ -574,7 +628,7 @@ void MessagesHandler::roundRobinReception(StaticVector*inbox,RingAllocator*inbox
* not compatible because the number of persistent requests can be a limitation.
*/
void MessagesHandler::probeAndRead(Rank source,MessageTag tag,
- StaticVector*inbox,RingAllocator*inboxAllocator){
+ ComputeCore**cores,int miniRanksPerRank){
// the code here will probe from rank source
// with MPI_Iprobe
@@ -592,7 +646,7 @@ void MessagesHandler::probeAndRead(Rank source,MessageTag tag,
return;
/* read at most one message */
- MPI_Datatype datatype=MPI_UNSIGNED_LONG_LONG;
+ MPI_Datatype datatype= m_datatype;
int actualTag=status.MPI_TAG;
Rank actualSource=status.MPI_SOURCE;
int count=-1;
@@ -602,22 +656,50 @@ void MessagesHandler::probeAndRead(Rank source,MessageTag tag,
assert(count >= 0);
#endif
+ MessageUnit staticBuffer[MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit)+2];
+
+ MPI_Recv(staticBuffer,count,datatype,actualSource,actualTag,MPI_COMM_WORLD,&status);
+
+/*
+ * Get the mini-rank source and mini-rank destination.
+ */
+ int miniRankSource=staticBuffer[count-2];
+ int miniRankDestination=staticBuffer[count-1];
+
+ #ifdef CONFIG_DEBUG_MINI_RANK_COMMUNICATION
+ cout<<"RECEIVE Source= "<<miniRankSource<<" Destination= "<<miniRankDestination<<" Tag= "<<actualTag<<endl;
+ #endif /* CONFIG_DEBUG_MINI_RANK_COMMUNICATION */
+
+ count-=2;
+
+ int miniRankIndex=miniRankDestination%miniRanksPerRank;
+
+ #ifdef CONFIG_DEBUG_MPI_RANK
+ cout<<"[MessagesHandler::probeAndRead] received message from "<<miniRanksPerRank<<" to "<<miniRankDestination;
+ cout<<" tag is "<<actualTag<<endl;
+ #endif
+
MessageUnit*incoming=NULL;
+
+ ComputeCore*core=cores[miniRankIndex];
+
if(count > 0){
- incoming=(MessageUnit*)inboxAllocator->allocate(count*sizeof(MessageUnit));
+ incoming=(MessageUnit*)core->getInboxAllocator()->allocate(count*sizeof(MessageUnit));
+
+ memcpy(incoming,staticBuffer,count);
}
- MPI_Recv(incoming,count,datatype,actualSource,actualTag,MPI_COMM_WORLD,&status);
+ Message aMessage(incoming,count,miniRankDestination,actualTag,miniRankSource);
+ aMessage.setNumberOfBytes(count);
- Message aMessage(incoming,count,m_rank,actualTag,actualSource);
- inbox->push_back(aMessage);
+ core->getInbox()->push_back(&aMessage);
#ifdef ASSERT
- assert(aMessage.getDestination() == m_rank);
+ // this assertion is not valid for mini-ranks.
+ //assert(aMessage.getDestination() == m_rank);
#endif
m_receivedMessages++;
-
}
void MessagesHandler::initialiseMembers(){
@@ -660,8 +742,10 @@ void MessagesHandler::freeLeftovers(){
#endif /* CONFIG_COMM_PERSISTENT */
+ #if 0
__Free(m_messageStatistics,"RAY_MALLOC_TYPE_MESSAGE_STATISTICS",false);
m_messageStatistics=NULL;
+ #endif
}
void MessagesHandler::constructor(int*argc,char***argv){
@@ -670,9 +754,36 @@ void MessagesHandler::constructor(int*argc,char***argv){
m_sentMessages=0;
m_receivedMessages=0;
- m_datatype=MPI_UNSIGNED_LONG_LONG;
+ //m_datatype=MPI_UNSIGNED_LONG_LONG;
+ m_datatype=MPI_BYTE;
+
+ #ifdef CONFIG_MINI_RANKS_disabled
+
+ int provided;
+ MPI_Init_thread(argc,argv, MPI_THREAD_FUNNELED, &provided);
+ bool threads_ok = provided >= MPI_THREAD_FUNNELED;
+
+ cout<<"Level provided: ";
+
+ if(provided==MPI_THREAD_SINGLE)
+ cout<<"MPI_THREAD_SINGLE";
+ else if(provided==MPI_THREAD_FUNNELED)
+ cout<<"MPI_THREAD_FUNNELED";
+ else if(provided==MPI_THREAD_SERIALIZED)
+ cout<<"MPI_THREAD_SERIALIZED";
+ else if(provided==MPI_THREAD_MULTIPLE)
+ cout<<"MPI_THREAD_MULTIPLE";
+ cout<<endl;
+
+ #ifdef ASSERT
+ assert(threads_ok);
+ #endif
+ #else
+ // same as MPI_Init_thread with MPI_THREAD_SINGLE
+ // in some MPI libraries, MPI_THREAD_SINGLE and MPI_THREAD_FUNNELED are the same.
MPI_Init(argc,argv);
+ #endif
char serverName[1000];
int len;
@@ -684,32 +795,18 @@ void MessagesHandler::constructor(int*argc,char***argv){
initialiseMembers();
m_processorName=serverName;
- m_numberOfDirtyBuffers=0;
-
- m_maximumDirtyBuffers=m_numberOfDirtyBuffers;
-
- m_dirtyBuffers=NULL;
- m_linearSweeps=0;
-
-/**
- * internally, there are N buffers for MPI_Isend. However,
- * these slots become dirty when they are used and become
- * clean again when MPI_Test says so.
- * But we don't want to do too much sweep operations. Instead,
- * we want amortized operations.
- */
-
- m_minimumNumberOfDirtyBuffersForSweep=__NOT_SET;
- m_minimumNumberOfDirtyBuffersForWarning=__NOT_SET;
#ifdef CONFIG_COMM_IRECV_TESTANY
m_requests=NULL;
#endif
+
+ createBuffers();
}
void MessagesHandler::createBuffers(){
+ #if 0
/** initialize message statistics to 0 */
m_messageStatistics=(uint64_t*)__Malloc(RAY_MPI_TAG_DUMMY*m_size*sizeof(uint64_t),"RAY_MALLOC_TYPE_MESSAGE_STATISTICS",false);
for(int rank=0;rank<m_size;rank++){
@@ -717,6 +814,7 @@ void MessagesHandler::createBuffers(){
m_messageStatistics[rank*RAY_MPI_TAG_DUMMY+tag]=0;
}
}
+ #endif
#ifdef CONFIG_COMM_IPROBE_ROUND_ROBIN
// start with the first connection
@@ -742,7 +840,6 @@ void MessagesHandler::destructor(){
}
freeLeftovers();
-
}
string*MessagesHandler::getName(){
@@ -757,15 +854,12 @@ int MessagesHandler::getSize(){
return m_size;
}
-void MessagesHandler::barrier(){
- MPI_Barrier(MPI_COMM_WORLD);
-}
-
void MessagesHandler::version(int*a,int*b){
MPI_Get_version(a,b);
}
void MessagesHandler::appendStatistics(const char*file){
+ #if 0
/** add an entry for RAY_MPI_TAG_GOOD_JOB_SEE_YOU_SOON_REPLY
* because this message is sent after writting the current file
*
@@ -795,17 +889,21 @@ void MessagesHandler::appendStatistics(const char*file){
}
fp.close();
+ #endif
+
cout<<"[MessagesHandler] Hello, this is the layered communication vessel, status follows."<<endl;
cout<<"Rank "<<m_rank<<": sent "<<m_sentMessages<<" messages, received "<<m_receivedMessages<<" messages."<<endl;
- cout<<"Rank "<<m_rank<<": the maximum number of dirty buffers was "<<m_maximumDirtyBuffers<<endl;
- cout<<"Rank "<<m_rank<<": "<<m_linearSweeps<<" linear sweep operations (threshold: ";
- cout<<m_minimumNumberOfDirtyBuffersForSweep<<" dirty buffers)"<<endl;
+
+ #if 0
+ outboxBufferAllocator->printStatus()
+
cout<<"Rank "<<m_rank<<": Active peers (including self): "<<activePeers.size()<<" list:";
for(int i=0;i<(int)activePeers.size();i++){
cout<<" "<<activePeers[i];
}
cout<<endl;
+ #endif
cout<<"[MessagesHandler] Over and out."<<endl;
}
@@ -870,21 +968,253 @@ void MessagesHandler::setConnections(vector<int>*connections){
m_peers=m_connections.size();
}
-void MessagesHandler::registerPlugin(ComputeCore*core){
- m_plugin=core->allocatePluginHandle();
+MPI_Request * MessagesHandler::registerMessageBuffer(void * buffer, Rank &source,
+ Rank & destination, MessageTag & tag,
+ RingAllocator * outboxBufferAllocator) const {
- core->setPluginName(m_plugin,"MessagesHandler");
- core->setPluginDescription(m_plugin,"MPI wrapper with round-robin policy (bundled with RayPlatform)");
- core->setPluginAuthors(m_plugin,"Sébastien Boisvert");
- core->setPluginLicense(m_plugin,"GNU Lesser General License version 3");
+ MPI_Request * request = outboxBufferAllocator->registerBuffer(buffer);
- RAY_MPI_TAG_DUMMY=core->allocateMessageTagHandle(m_plugin);
- core->setMessageTagSymbol(m_plugin,RAY_MPI_TAG_DUMMY,"RAY_MPI_TAG_DUMMY");
+ outboxBufferAllocator->setRegisteredBufferAttributes(buffer, m_rank,
+ destination, tag);
- createBuffers();
+#ifdef CONFIG_ASSERT
+ assert(request != NULL);
+#endif // CONFIG_ASSERT
+
+ return request;
}
-void MessagesHandler::resolveSymbols(ComputeCore*core){
- RAY_MPI_TAG_DUMMY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_DUMMY");
+/*
+ * send messages,
+ *
+ * Regarding m_dirtyBufferSlots:
+ *
+ * this is the maximum number of dirty buffers
+ * it should be at least the number of allocated
+ * buffer in a RayPlatform virtual machine tick.
+ *
+ * For very large jobs, this might need to be increased
+ * in order to avoid the situation in which
+ * all the buffer for the outbox are dirty/used.
+ */
+void MessagesHandler::sendMessages(StaticVector*outbox,RingAllocator*outboxBufferAllocator){
+
+ // initialize the dirty buffer counters
+ // this is done only once
+ if(outboxBufferAllocator->getDirtyBuffers()==NULL)
+ outboxBufferAllocator->initializeDirtyBuffers();
+
+ outboxBufferAllocator->cleanDirtyBuffers();
+
+ // send messages.
+ for(int i=0;i<(int)outbox->size();i++){
+ Message*aMessage=((*outbox)[i]);
+ Rank destination=aMessage->getDestination();
+ void*buffer=aMessage->getBuffer();
+ int count=aMessage->getNumberOfBytes();
+ MessageTag tag=aMessage->getTag();
+
+ #ifdef ASSERT
+ assert(destination>=0);
+ if(destination>=m_size){
+ cout<<"Tag="<<tag<<" Destination="<<destination<<endl;
+ }
+ assert(destination<m_size);
+ assert(!(buffer==NULL && count>0));
+
+ // this assertion is invalid when using checksum calculation.
+ //assert(count<=(int)(MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit)));
+ #endif /* ASSERT */
+
+ MPI_Request dummyRequest;
+
+ MPI_Request*request=&dummyRequest;
+
+ int handle=-1;
+
+ /* compute the handle, this is O(1) */
+ if(buffer!=NULL)
+ handle=outboxBufferAllocator->getBufferHandle(buffer);
+
+ bool mustRegister=false;
+
+ // this buffer is not registered.
+ if(handle >=0 && !outboxBufferAllocator->isRegistered(handle))
+ mustRegister=true;
+
+ /* register the buffer for processing */
+ if(mustRegister){
+ request = this->registerMessageBuffer(buffer, m_rank, destination,
+ tag, outboxBufferAllocator);
+ }
+
+ #ifdef ASSERT
+ assert(request!=NULL);
+ #endif
+
+#ifdef CONFIG_ASSERT
+ assert(destination >= 0);
+ assert(destination < m_size);
+ assert(count >= 0);
+#endif
+
+ // MPI_Isend
+ // Synchronous nonblocking.
+ MPI_Isend(buffer,count,m_datatype,destination,tag,MPI_COMM_WORLD,request);
+
+ // if the buffer is NULL, we free the request right now
+ // because we there is no buffer to protect.
+
+ if(!mustRegister){
+ #ifdef COMMUNICATION_IS_VERBOSE
+ cout<<" From sendMessages"<<endl;
+ #endif
+
+ MPI_Request_free(request);
+
+ #ifdef ASSERT
+ assert(*request==MPI_REQUEST_NULL);
+ #endif
+ }
+
+ #if 0
+ // if the message was re-routed, we don't care
+ // we only fetch the tag for statistics
+ uint8_t shortTag=tag;
+
+ /** update statistics */
+ m_messageStatistics[destination*RAY_MPI_TAG_DUMMY+shortTag]++;
+
+ #endif
+
+ m_sentMessages++;
+ }
+
+ outbox->clear();
+}
+
+
+void MessagesHandler::receiveMessages(StaticVector*inbox,RingAllocator*inboxAllocator){
+ // the code here will probe from rank source
+ // with MPI_Iprobe
+
+ #ifdef COMMUNICATION_IS_VERBOSE
+ cout<<"call to probeAndRead source="<<source<<""<<endl;
+ #endif /* COMMUNICATION_IS_VERBOSE */
+
+ int source=MPI_ANY_SOURCE;
+ int tag=MPI_ANY_TAG;
+
+ int flag=0;
+ MPI_Status status;
+ MPI_Iprobe(source,tag,MPI_COMM_WORLD,&flag,&status);
+
+ // nothing to receive...
+ if(!flag)
+ return;
+
+ /* read at most one message */
+ MPI_Datatype datatype= m_datatype;
+ int actualTag=status.MPI_TAG;
+ Rank actualSource=status.MPI_SOURCE;
+ int count=-1;
+ MPI_Get_count(&status,datatype,&count);
+
+ #ifdef ASSERT
+ assert(count >= 0);
+ #endif
+
+ MessageUnit*incoming=NULL;
+ if(count > 0){
+ incoming=(MessageUnit*)inboxAllocator->allocate(count*sizeof(MessageUnit));
+ }
+
+ MPI_Recv(incoming,count,datatype,actualSource,actualTag,MPI_COMM_WORLD,&status);
+
+ Message aMessage(incoming,count,m_rank,actualTag,actualSource);
+ aMessage.setNumberOfBytes(count);
+
+ inbox->push_back(&aMessage);
+
+ #ifdef ASSERT
+ assert(aMessage.getDestination() == m_rank);
+ #endif
+
+ m_receivedMessages++;
+
+}
+
+void MessagesHandler::sendMessagesForComputeCore(StaticVector*outbox,MessageQueue*bufferedOutbox){
+
+#ifdef CONFIG_USE_LOCKING
+ bufferedOutbox.lock();
+#endif /* CONFIG_USE_LOCKING */
+
+ int messages=outbox->size();
+
+/*
+ * TODO: I am not sure that it is safe to give our own buffer to
+ * the other thread. If the ring is large enough and the communication
+ * is steady, probably.
+ */
+ for(int i=0;i<messages;i++){
+ Message*message=(*outbox)[i];
+
+/*
+ * If the queue is full, we just retry endlessly, it will work eventually.
+ * If not, something is wrong.
+ */
+ int ticks=0;
+
+ while(!bufferedOutbox->push(message)){
+
+ if(ticks%CONFIG_MESSAGE_QUEUE_RETRY_WARNING ==0)
+ cout<<"[MessagesHandler] Warning: outbox message queue is full, will retry in a bit... (#"<<ticks<<")"<<endl;
+
+ ticks++;
+ }
+ }
+
+#ifdef CONFIG_USE_LOCKING
+ bufferedOutbox->unlock();
+#endif /* CONFIG_USE_LOCKING */
+
+}
+
+void MessagesHandler::receiveMessagesForComputeCore(StaticVector*inbox,RingAllocator*inboxAllocator,
+ MessageQueue*bufferedInbox){
+
+#ifdef CONFIG_USE_LOCKING
+ bufferedInbox->lock();
+#endif /* CONFIG_USE_LOCKING */
+
+/*
+ * We need to copy the buffer in our own buffer here
+ * because otherwise this is not thread safe.
+ */
+ if(bufferedInbox->hasContent()){
+ Message message;
+ bufferedInbox->pop(&message);
+
+ int count=message.getCount();
+ MessageUnit*incoming=(MessageUnit*)inboxAllocator->allocate(count*sizeof(MessageUnit));
+
+/*
+ * Copy the data, this is slow.
+ */
+ memcpy(incoming,message.getBuffer(),count*sizeof(MessageUnit));
+
+/*
+ * Update the buffer so that the code is thread-safe.
+ */
+ message.setBuffer(incoming);
+
+ inbox->push_back(&message);
+ }
+
+#ifdef CONFIG_USE_LOCKING
+ m_bufferedInbox.unlock();
+#endif /* CONFIG_USE_LOCKING */
+
}
diff --git a/RayPlatform/communication/MessagesHandler.h b/RayPlatform/RayPlatform/communication/MessagesHandler.h
similarity index 75%
rename from RayPlatform/communication/MessagesHandler.h
rename to RayPlatform/RayPlatform/communication/MessagesHandler.h
index 68ecb98..8b252b1 100644
--- a/RayPlatform/communication/MessagesHandler.h
+++ b/RayPlatform/RayPlatform/communication/MessagesHandler.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,6 +22,18 @@
#ifndef _MessagesHandler
#define _MessagesHandler
+#include "Message.h"
+
+#include <RayPlatform/memory/MyAllocator.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/structures/StaticVector.h>
+
+// this is one of the two includes for mpi.h
+#include <mpi.h>
+#include <string>
+#include <vector>
+using namespace std;
+
/* Many communication models are implemented:
* latencies are for a system with 36 cores (or 120 cores), QLogic interconnect,
* and Performance scaled messaging
@@ -74,31 +86,20 @@
*/
//#define CONFIG_COMM_IRECV_TESTANY
-#include <mpi.h> // this is the only reference to MPI
-
-#include <memory/MyAllocator.h>
-#include <communication/Message.h>
-//#include <core/common_functions.h>
-#include <memory/RingAllocator.h>
-#include <structures/StaticVector.h>
-#include <plugins/CorePlugin.h>
-
class ComputeCore;
+class MessageQueue;
-#include <string>
-#include <vector>
-using namespace std;
-
-/**
- * A data model for storing dirty buffers
+/*
+ Open-MPI eager threshold is 4k (4096), and this include Open-MPI's metadata.
+ tests show that 4096-100 bytes are sent eagerly, too.
+ divide that by eight and you get the number of 64-bit integers
+ allowed in a single eager communication
+
+ * "4096 is rendezvous. For eager, try 4000 or lower. "
+ * --Eugene Loh (Oracle)
+ * http://www.open-mpi.org/community/lists/devel/2010/11/8700.php
+ *
*/
-class DirtyBuffer{
-public:
- void*m_buffer;
- MPI_Request m_messageRequest;
- Rank m_destination;
- MessageTag m_messageTag;
-};
/**
* Software layer to handle messages.
@@ -120,26 +121,22 @@ public:
*
* \author Sébastien Boisvert
*/
-class MessagesHandler: public CorePlugin{
-
- int m_minimumNumberOfDirtyBuffersForSweep;
+class MessagesHandler{
- int m_minimumNumberOfDirtyBuffersForWarning;
+#ifdef CONFIG_MINI_RANKS
+/*
+ * We need a temporary buffer because we don't know yet
+ * for which mini-rank the message is for.
+ */
+ MessageUnit m_staticBuffer[MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit)+2];
+#endif
-/** prints dirty buffers **/
- void printDirtyBuffers();
+ bool m_hasReceivedMessage;
+ int m_lastMiniRank;
// the number of peers for communication
int m_peers;
- DirtyBuffer*m_dirtyBuffers;
-
- int m_numberOfDirtyBuffers;
- int m_maximumDirtyBuffers;
- int m_dirtyBufferSlots;
-
- MessageTag RAY_MPI_TAG_DUMMY;
-
bool m_destroyed;
vector<int> m_connections;
@@ -204,8 +201,7 @@ class MessagesHandler: public CorePlugin{
/** initialize persistent communication parameters */
void initialiseMembers();
- /** probe and read a message -- this method is not utilised */
- void probeAndRead(int source,int tag,StaticVector*inbox,RingAllocator*inboxAllocator);
+ void probeAndRead(int source,int tag,ComputeCore**cores,int miniRanksPerRank);
#ifdef CONFIG_COMM_PERSISTENT
/** pump a message from the persistent ring */
@@ -228,24 +224,34 @@ class MessagesHandler: public CorePlugin{
void createBuffers();
- void checkDirtyBuffer(RingAllocator*outboxBufferAllocator,int i);
- void cleanDirtyBuffers(RingAllocator*outboxBufferAllocator);
- uint64_t m_linearSweeps;
- void initializeDirtyBuffers(RingAllocator*outboxBufferAllocator);
+
+/**
+ * send a message or more
+ */
+ void sendMessagesForMiniRank(MessageQueue*outbox,RingAllocator*outboxBufferAllocator,int miniRanksPerRank);
+
+/**
+ * receive one or zero message.
+ * the others, if any, will be picked up in the next iteration
+ */
+ void receiveMessagesForMiniRanks(ComputeCore**cores,int miniRanksPerRank);
+
+ MPI_Request * registerMessageBuffer(void * buffer, Rank &source,
+ Rank & destination, MessageTag & tag,
+ RingAllocator * outboxBufferAllocator) const;
public:
- /** initialize the message handler
- * */
+/**
+ * initialize the message handler
+*/
void constructor(int*argc,char***argv);
- /**
+/**
* send a message or more
*/
void sendMessages(StaticVector*outbox,RingAllocator*outboxBufferAllocator);
-
-
- /**
+/**
* receive one or zero message.
* the others, if any, will be picked up in the next iteration
*/
@@ -263,9 +269,6 @@ public:
/** get the number of ranks */
int getSize();
- /** makes a barrier */
- void barrier();
-
/** returns the version of the message passing interface standard that is available */
void version(int*a,int*b);
@@ -278,8 +281,19 @@ public:
void setConnections(vector<int>*connections);
+ void sendAndReceiveMessagesForRankProcess(ComputeCore**cores,int miniRanksPerRank,bool*communicate);
+
void registerPlugin(ComputeCore*core);
void resolveSymbols(ComputeCore*core);
+
+/*
+ * Services provided to the ComputeCore.
+ */
+
+ void sendMessagesForComputeCore(StaticVector*outbox,MessageQueue*bufferedOutbox);
+
+ void receiveMessagesForComputeCore(StaticVector*inbox,RingAllocator*inboxAllocator,
+ MessageQueue*bufferedOutbox);
};
#endif /* _MessagesHandler */
diff --git a/RayPlatform/communication/VirtualCommunicator.cpp b/RayPlatform/RayPlatform/communication/VirtualCommunicator.cpp
similarity index 93%
rename from RayPlatform/communication/VirtualCommunicator.cpp
rename to RayPlatform/RayPlatform/communication/VirtualCommunicator.cpp
index 2aa2296..11c935f 100644
--- a/RayPlatform/communication/VirtualCommunicator.cpp
+++ b/RayPlatform/RayPlatform/communication/VirtualCommunicator.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,12 +18,11 @@
see <http://www.gnu.org/licenses/>
*/
-/* #define DEBUG_VIRTUAL_COMMUNICATOR */
+#include "VirtualCommunicator.h"
-#include <communication/VirtualCommunicator.h>
-#include <core/OperatingSystem.h>
-#include <core/types.h>
-#include <core/ComputeCore.h>
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/core/types.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <assert.h>
#include <iostream>
@@ -32,6 +31,8 @@
#include <stdio.h>
using namespace std;
+/* #define CONFIG_DEBUG_VIRTUAL_COMMUNICATOR */
+
/** set the number of elements per message for a given tag */
void VirtualCommunicator::setElementsPerQuery(int tag,int size){
#ifdef ASSERT
@@ -188,7 +189,7 @@ void VirtualCommunicator::pushMessage(WorkerHandle workerId,Message*message){
}
void VirtualCommunicator::flushMessage(int tag,int destination){
- #ifdef DEBUG_VIRTUAL_COMMUNICATOR
+ #ifdef CONFIG_DEBUG_VIRTUAL_COMMUNICATOR
cout<<"VirtualCommunicator:: sending multiplexed message to "<<destination<<endl;
#endif
@@ -231,7 +232,7 @@ void VirtualCommunicator::flushMessage(int tag,int destination){
}
Message aMessage(messageContent,currentSize,destination,tag,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_pendingMessages++;
}
@@ -256,10 +257,10 @@ void VirtualCommunicator::getMessageResponseElements(WorkerHandle workerId,vecto
}
void VirtualCommunicator::constructor(int rank,int size,RingAllocator*outboxAllocator,StaticVector*inbox,StaticVector*outbox){
- m_debug=false;
-
- if(m_debug)
- cout<<"Rank "<<rank<<" Initializing VirtualCommunicator"<<endl;
+
+#ifdef CONFIG_DEBUG_VIRTUAL_COMMUNICATOR
+ cout<<"Rank "<<rank<<" Initializing VirtualCommunicator"<<endl;
+#endif
resetCounters();
@@ -296,7 +297,7 @@ void VirtualCommunicator::processInbox(vector<WorkerHandle>*activeWorkers){
int queryTag=m_replyTagToQueryTag[incomingTag];
if(m_activeTag==queryTag&&m_activeDestination==source){
- #ifdef DEBUG_VIRTUAL_COMMUNICATOR
+ #ifdef CONFIG_DEBUG_VIRTUAL_COMMUNICATOR
cout<<"VirtualCommunicator: receiving multiplexed message, de-multiplexing data..."<<endl;
#endif
@@ -331,9 +332,11 @@ void VirtualCommunicator::processInbox(vector<WorkerHandle>*activeWorkers){
for(int i=0;i<(int)workers.size();i++){
WorkerHandle workerId=workers[i];
activeWorkers->push_back(workerId);
- if(m_debug){
- cout<<"Reactivating "<<workerId<<" tag="<<queryTag<<endl;
- }
+
+#ifdef CONFIG_DEBUG_VIRTUAL_COMMUNICATOR
+ cout<<"Reactivating "<<workerId<<" tag="<<queryTag<<endl;
+#endif
+
int basePosition=i*elementsPerWorker;
#ifdef ASSERT
@@ -386,14 +389,16 @@ bool VirtualCommunicator::isReady(){
// force the first encountered thing
void VirtualCommunicator::forceFlush(){
- if(m_debug){
- cout<<__func__<<endl;
- }
+
+#ifdef CONFIG_DEBUG_VIRTUAL_COMMUNICATOR
+ cout<<__func__<<endl;
+#endif
if(m_priorityQueue.size()==0){
- if(m_debug){
- cout<<"queue is empty"<<endl;
- }
+
+#ifdef CONFIG_DEBUG_VIRTUAL_COMMUNICATOR
+ cout<<"queue is empty"<<endl;
+#endif
return;
}
@@ -450,11 +455,6 @@ void VirtualCommunicator::printStatistics(){
cout<<" real messages ("<<ratio<<"%)"<<endl;
}
-/** debugging will display a lot of messages */
-void VirtualCommunicator::setDebug(){
- m_debug=true;
-}
-
uint64_t VirtualCommunicator::getMessageUniqueId(Rank destination ,int tag){
uint64_t a=tag;
a=a*MAX_NUMBER_OF_MPI_PROCESSES+destination;
diff --git a/RayPlatform/communication/VirtualCommunicator.h b/RayPlatform/RayPlatform/communication/VirtualCommunicator.h
similarity index 95%
rename from RayPlatform/communication/VirtualCommunicator.h
rename to RayPlatform/RayPlatform/communication/VirtualCommunicator.h
index 811959e..9c90c43 100644
--- a/RayPlatform/communication/VirtualCommunicator.h
+++ b/RayPlatform/RayPlatform/communication/VirtualCommunicator.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -23,10 +23,11 @@
#ifndef _VirtualCommunicator
#define _VirtualCommunicator
-#include <memory/RingAllocator.h>
-#include <structures/StaticVector.h>
-#include <communication/Message.h>
-#include <memory/MyAllocator.h>
+#include "Message.h"
+
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/memory/MyAllocator.h>
+#include <RayPlatform/structures/StaticVector.h>
#include <map>
#include <vector>
@@ -48,7 +49,6 @@ using namespace std;
* \author Sébastien Boisvert
*/
class VirtualCommunicator{
- bool m_debug;
uint64_t m_pushedMessages;
uint64_t m_flushedMessages;
@@ -210,8 +210,6 @@ public:
void printStatistics();
void resetCounters();
- void setDebug();
-
int getReplyType(int tag);
};
diff --git a/RayPlatform/communication/mpi_tags.cpp b/RayPlatform/RayPlatform/communication/mpi_tags.cpp
similarity index 84%
rename from RayPlatform/communication/mpi_tags.cpp
rename to RayPlatform/RayPlatform/communication/mpi_tags.cpp
index 07f35b4..b8a52c3 100644
--- a/RayPlatform/communication/mpi_tags.cpp
+++ b/RayPlatform/RayPlatform/communication/mpi_tags.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,7 +22,7 @@ Sébastien Boisvert has a scholarship from the Canadian Institutes of Health Res
*/
-#include <communication/mpi_tags.h>
+#include "mpi_tags.h"
char MESSAGE_TAGS[MAXIMUM_NUMBER_OF_TAG_HANDLERS][128];
diff --git a/RayPlatform/communication/mpi_tags.h b/RayPlatform/RayPlatform/communication/mpi_tags.h
similarity index 77%
copy from RayPlatform/communication/mpi_tags.h
copy to RayPlatform/RayPlatform/communication/mpi_tags.h
index 6d6df54..3b196cd 100644
--- a/RayPlatform/communication/mpi_tags.h
+++ b/RayPlatform/RayPlatform/communication/mpi_tags.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -23,10 +23,10 @@
#define _mpi_tags
// tags for MPI
-// these are the message types used by Ray
+// these are the message types used by Ray plugins
// Ray instances like to communicate a lots!
-#include <core/types.h>
+#include <RayPlatform/core/types.h>
extern char MESSAGE_TAGS[MAXIMUM_NUMBER_OF_TAG_HANDLERS][128];
diff --git a/RayPlatform/core/ComputeCore.cpp b/RayPlatform/RayPlatform/core/ComputeCore.cpp
similarity index 72%
rename from RayPlatform/core/ComputeCore.cpp
rename to RayPlatform/RayPlatform/core/ComputeCore.cpp
index 9fdb7ff..1ee947d 100644
--- a/RayPlatform/core/ComputeCore.cpp
+++ b/RayPlatform/RayPlatform/core/ComputeCore.cpp
@@ -1,37 +1,66 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ rayplatform: a message-passing development framework
+ copyright (c) 2010, 2011, 2012, 2013 sébastien boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/rayplatform
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Lesser General Public License as published by
- the Free Software Foundation, version 3 of the License.
+ this program is free software: you can redistribute it and/or modify
+ it under the terms of the gnu lesser general public license as published by
+ the free software foundation, version 3 of the license.
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Lesser General Public License for more details.
+ this program is distributed in the hope that it will be useful,
+ but without any warranty; without even the implied warranty of
+ merchantability or fitness for a particular purpose. see the
+ gnu lesser general public license for more details.
- You have received a copy of the GNU Lesser General Public License
- along with this program (lgpl-3.0.txt).
+ you have received a copy of the gnu lesser general public license
+ along with this program (lgpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include <core/ComputeCore.h>
-#include <core/OperatingSystem.h>
-#include <cryptography/crypto.h>
+#include "ComputeCore.h"
+#include "OperatingSystem.h"
+
+#include <RayPlatform/profiling/ProcessStatus.h>
+#include <RayPlatform/cryptography/crypto.h>
+#include <RayPlatform/communication/MessagesHandler.h>
+
#include <stdlib.h>
#include <string.h> /* for strcpy */
-
#ifdef ASSERT
#include <assert.h>
#endif
-
#include <iostream>
using namespace std;
+#ifdef CONFIG_SLEEPY_RAY
+#include <unistd.h>
+#endif
+
+#include <signal.h>
+
+/**
+ * Global variable are bad.
+ */
+bool globalDebugMode = false;
+
+/**
+ * \see http://stackoverflow.com/questions/6168636/how-to-trigger-sigusr1-and-sigusr2
+ * \see http://stackoverflow.com/questions/231912/what-is-the-difference-between-sigaction-and-signal
+ */
+void handleSignal(int signalNumber) {
+
+ //cout << "DEBUG Received a signal !!" << endl;
+
+ if(signalNumber == SIGUSR1) {
+
+ globalDebugMode = !globalDebugMode;
+
+ //cout << "DEBUG globalDebugMode <- " << globalDebugMode << endl;
+ }
+}
+
//#define CONFIG_DEBUG_SLAVE_SYMBOLS
//#define CONFIG_DEBUG_MASTER_SYMBOLS
//#define CONFIG_DEBUG_TAG_SYMBOLS
@@ -39,9 +68,12 @@ using namespace std;
//#define CONFIG_DEBUG_CORE
ComputeCore::ComputeCore(){
+
+ m_playground.initialize(this);
+
}
-void ComputeCore::setSlaveModeObjectHandler(PluginHandle plugin,SlaveMode mode,SlaveModeHandler object){
+void ComputeCore::setSlaveModeObjectHandler(PluginHandle plugin,SlaveMode mode,SlaveModeHandlerReference object){
if(!validationPluginAllocated(plugin))
return;
@@ -67,7 +99,7 @@ void ComputeCore::setSlaveModeObjectHandler(PluginHandle plugin,SlaveMode mode,S
m_plugins[plugin].addRegisteredSlaveModeHandler(mode);
}
-void ComputeCore::setMasterModeObjectHandler(PluginHandle plugin,MasterMode mode,MasterModeHandler object){
+void ComputeCore::setMasterModeObjectHandler(PluginHandle plugin,MasterMode mode,MasterModeHandlerReference object){
if(!validationPluginAllocated(plugin))
return;
@@ -91,7 +123,7 @@ void ComputeCore::setMasterModeObjectHandler(PluginHandle plugin,MasterMode mode
m_plugins[plugin].addRegisteredMasterModeHandler(mode);
}
-void ComputeCore::setMessageTagObjectHandler(PluginHandle plugin,MessageTag tag,MessageTagHandler object){
+void ComputeCore::setMessageTagObjectHandler(PluginHandle plugin,MessageTag tag,MessageTagHandlerReference object){
if(!validationPluginAllocated(plugin))
return;
@@ -122,11 +154,17 @@ void ComputeCore::setMessageTagObjectHandler(PluginHandle plugin,MessageTag tag,
*/
void ComputeCore::run(){
+ if(m_doChecksum){
+ cout<<"[RayPlatform] Rank "<<m_rank<<" will compute a CRC32 checksum for any non-empty message." << endl;
+ }
+
// ask the router if it is enabled
// the virtual router will disable itself if there were
// problems during configuration
m_routerIsEnabled=m_router.isEnabled();
+ configureEngine();
+
if(!m_resolvedSymbols)
resolveSymbols();
@@ -140,33 +178,84 @@ void ComputeCore::run(){
if(m_routerIsEnabled)
m_router.getGraph()->start(m_rank);
+
+ /*
+ * Set up signal handler
+ * \see http://pubs.opengroup.org/onlinepubs/7908799/xsh/sigaction.html
+ * \see http://beej.us/guide/bgipc/output/html/multipage/signals.html
+ */
+
+ struct sigaction newAction;
+ newAction.sa_handler = handleSignal;
+ sigemptyset(&newAction.sa_mask);
+ newAction.sa_flags = 0;
+
+ sigaction(SIGUSR1, &newAction, NULL);
+
+ m_runProfiler = globalDebugMode;
+
+ runWithProfiler();
+
+#if 0
if(m_runProfiler){
runWithProfiler();
}else{
runVanilla();
}
+#endif
if(m_routerIsEnabled)
m_router.getGraph()->printStatus();
}
+bool ComputeCore::debugModeIsEnabled() {
+ return m_runProfiler;
+}
+
/**
- * the while loop is *the* main loop of Ray for each
+ * the while loop is *the* main loop of Ray for each
* processor.
* it is similar to the main loop of a video game, actually, but without a display.
*/
void ComputeCore::runVanilla(){
+ /*
+ * This is not used anymore !
+ */
+ return;
+
+ #ifdef CONFIG_DEBUG_CORE
+ cout<<"Locking inbox for the loop."<<endl;
+ #endif
+
#ifdef CONFIG_DEBUG_CORE
cout<<"m_alive= "<<m_alive<<endl;
#endif
+/*
+ * The inbox is only unlocked before the receiveMessages() and
+ * after sendMessages().
+ *
+ * The outbox is only unlocked during sendMessages().
+ *
+ * The inbox is cleared at the end of processData()
+ * because some plugins picks up messages in the inbox
+ * instead of using the modular plugin architecture with
+ * the handlers for message tags.
+ *
+ * The outbox is cleared by the communication layer when it
+ * picks up the messages.
+ */
while(m_alive || (m_routerIsEnabled && !m_router.hasCompletedRelayEvents())){
-
+
+#ifdef CONFIG_SLEEPY_RAY
+ usleep(10);
+#endif
+
// 1. receive the message (0 or 1 message is received)
// blazing fast, receives 0 or 1 message, never more, never less, other messages will wait for the next iteration !
- receiveMessages();
+ receiveMessages();
// 2. process the received message, if any
// consume the one message received, if any, also very fast because it is done with an array mapping tags to function pointers
@@ -184,6 +273,18 @@ void ComputeCore::runVanilla(){
#ifdef CONFIG_DEBUG_CORE
cout<<"m_alive= "<<m_alive<<endl;
#endif
+
+ if(m_miniRanksAreEnabled){
+#ifdef CONFIG_USE_LOCKING
+ m_bufferedOutbox.lock();
+#endif /* CONFIG_USE_LOCKING */
+
+ m_bufferedOutbox.sendKillSignal();
+
+#ifdef CONFIG_USE_LOCKING
+ m_bufferedOutbox.unlock();
+#endif /* CONFIG_USE_LOCKING */
+ }
}
/*
@@ -194,6 +295,8 @@ void ComputeCore::runWithProfiler(){
// define some variables that hold life statistics of this
// MPI rank
int ticks=0;
+ uint64_t globalTicks = 0;
+
int sentMessages=0;
int sentMessagesInProcessMessages=0;
int sentMessagesInProcessData=0;
@@ -201,9 +304,8 @@ void ComputeCore::runWithProfiler(){
map<int,int> receivedTags;
map<int,int> sentTagsInProcessMessages;
map<int,int> sentTagsInProcessData;
-
- int resolution=100;// milliseconds
- int parts=1000/resolution;
+
+ int resolution=1000;// milliseconds
uint64_t startingTime=getMilliSeconds();
@@ -217,59 +319,108 @@ void ComputeCore::runWithProfiler(){
vector<int> distancesForProcessMessages;
vector<int> distancesForProcessData;
- bool profilerVerbose=m_profilerVerbose;
+ bool profilerVerbose = true;
+
+ ProcessStatus status;
+
+ uint64_t t=getMilliSeconds();
+
+ //m_lastTerminalProbeOperation = time(NULL);
+ bool profileGranularity = false;
+
+ m_playground.bootActors();
+
+ while(isRankAlive()
+ || (m_routerIsEnabled && !m_router.hasCompletedRelayEvents())){
+
+ // update the debug mode.
+ // this is cheap -- it only needs to copy a boolean (usually a char)
+ //
+ m_runProfiler = globalDebugMode;
+
+#if 0
+ if(debugModeIsEnabled())
+ cout << "DEBUG Online" << endl;
+#endif
+
+ //runRayPlatformTerminal();
+
+ if(debugModeIsEnabled())
+ t=getMilliSeconds();
- while(m_alive || (m_routerIsEnabled && !m_router.hasCompletedRelayEvents())){
- uint64_t t=getMilliSeconds();
- if(t>=(lastTime+resolution)/parts*parts){
+
+ /*
+ * Show debug window for first tick, then at each 1000 ms.
+ */
+ if(debugModeIsEnabled() && (globalTicks == 0 || (t >= (lastTime+resolution)))){
double seconds=(t-startingTime)/1000.0;
int balance=sentMessages-receivedMessages;
- if(profilerVerbose){
+ if(debugModeIsEnabled()){
+
+ cout << "[/dev/actor/rank/" << getRank() << "] ";
+ cout << "[RayPlatform] epoch ends at ";
+ cout << seconds * 1000 << " ms ! (tick # " << globalTicks;
+ cout << "), length is ";
+ cout << resolution << " ms" << endl;
+
+ status.getProcessStatus();
+ status.printMemoryMetrics();
+ cout << endl;
+
printf("Rank %i: %s Time= %.2f s Speed= %i Sent= %i (processMessages: %i, processData: %i) Received= %i Balance= %i\n",
m_rank,SLAVE_MODES[m_switchMan.getSlaveMode()],
seconds,ticks,sentMessages,sentMessagesInProcessMessages,sentMessagesInProcessData,
receivedMessages,balance);
fflush(stdout);
- m_profiler.printGranularities(m_rank);
+ if(receivedMessages == 0) {
+ cout << "Warning: no messages were received !" << endl;
+ }
+
+ if(sentMessagesInProcessData == 0) {
+ cout << "Warning: no message were sent !" <<endl;
+ }
+
+ if(profileGranularity)
+ m_profiler.printGranularities(m_rank);
}
m_profiler.clearGranularities();
- if(receivedTags.size() > 0 && profilerVerbose){
- cout<<"Rank "<<m_messagesHandler.getRank()<<" received in receiveMessages:"<<endl;
+ if(debugModeIsEnabled() && receivedTags.size() > 0 && profilerVerbose){
+ cout<<"Rank "<<m_rank<<" received in receiveMessages:"<<endl;
for(map<int,int>::iterator i=receivedTags.begin();i!=receivedTags.end();i++){
int tag=i->first;
int count=i->second;
- cout<<"Rank "<<m_messagesHandler.getRank()<<" "<<MESSAGE_TAGS[tag]<<" "<<count<<endl;
+ cout<<"Rank "<<m_rank<<" "<<MESSAGE_TAGS[tag]<<" "<<count<<endl;
}
}
- if(sentTagsInProcessMessages.size() > 0 && profilerVerbose){
- cout<<"Rank "<<m_messagesHandler.getRank()<<" sent in processMessages:"<<endl;
+ if(debugModeIsEnabled() && sentTagsInProcessMessages.size() > 0 && profilerVerbose){
+ cout<<"Rank "<<m_rank<<" sent in processMessages:"<<endl;
for(map<int,int>::iterator i=sentTagsInProcessMessages.begin();i!=sentTagsInProcessMessages.end();i++){
int tag=i->first;
int count=i->second;
- cout<<"Rank "<<m_messagesHandler.getRank()<<" "<<MESSAGE_TAGS[tag]<<" "<<count<<endl;
+ cout<<"Rank "<<m_rank<<" "<<MESSAGE_TAGS[tag]<<" "<<count<<endl;
}
/*
int average1=getAverage(&distancesForProcessMessages);
int deviation1=getStandardDeviation(&distancesForProcessMessages);
-
- cout<<"Rank "<<m_messagesHandler.getRank()<<" distance between processMessages messages: average= "<<average1<<", stddev= "<<deviation1<<
+
+ cout<<"Rank "<<m_rank<<" distance between processMessages messages: average= "<<average1<<", stddev= "<<deviation1<<
", n= "<<distancesForProcessMessages.size()<<endl;
-
+
*/
#ifdef FULL_DISTRIBUTION
map<int,int> distribution1;
for(int i=0;i<(int)distancesForProcessMessages.size();i++){
distribution1[distancesForProcessMessages[i]]++;
}
- cout<<"Rank "<<m_messagesHandler.getRank()<<" distribution: "<<endl;
+ cout<<"Rank "<<m_rank<<" distribution: "<<endl;
for(map<int,int>::iterator i=distribution1.begin();i!=distribution1.end();i++){
cout<<i->first<<" "<<i->second<<endl;
}
@@ -279,27 +430,27 @@ void ComputeCore::runWithProfiler(){
distancesForProcessMessages.clear();
- if(sentTagsInProcessData.size() > 0 && profilerVerbose){
- cout<<"Rank "<<m_messagesHandler.getRank()<<" sent in processData:"<<endl;
+ if(debugModeIsEnabled () && sentTagsInProcessData.size() > 0 && profilerVerbose){
+ cout<<"Rank "<<m_rank<<" sent in processData:"<<endl;
for(map<int,int>::iterator i=sentTagsInProcessData.begin();i!=sentTagsInProcessData.end();i++){
int tag=i->first;
int count=i->second;
- cout<<"Rank "<<m_messagesHandler.getRank()<<" "<<MESSAGE_TAGS[tag]<<" "<<count<<endl;
+ cout<<"Rank "<<m_rank<<" "<<MESSAGE_TAGS[tag]<<" "<<count<<endl;
}
/*
int average2=getAverage(&distancesForProcessData);
int deviation2=getStandardDeviation(&distancesForProcessData);
-
- cout<<"Rank "<<m_messagesHandler.getRank()<<" distance between processData messages: average= "<<average2<<", stddev= "<<deviation2<<
+
+ cout<<"Rank "<<m_rank<<" distance between processData messages: average= "<<average2<<", stddev= "<<deviation2<<
", n= "<<distancesForProcessData.size()<<endl;
-
+
*/
#ifdef FULL_DISTRIBUTION
map<int,int> distribution2;
for(int i=0;i<(int)distancesForProcessData.size();i++){
distribution2[distancesForProcessData[i]]++;
}
- cout<<"Rank "<<m_messagesHandler.getRank()<<" distribution: "<<endl;
+ cout<<"Rank "<<m_rank<<" distribution: "<<endl;
for(map<int,int>::iterator i=distribution2.begin();i!=distribution2.end();i++){
cout<<i->first<<" "<<i->second<<endl;
}
@@ -324,9 +475,9 @@ void ComputeCore::runWithProfiler(){
/* collect some statistics for the profiler */
// 1. receive the message (0 or 1 message is received)
- receiveMessages();
+ receiveMessages();
receivedMessages+=m_inbox.size();
-
+
for(int i=0;i<(int)m_inbox.size();i++){
// stript routing information, if any
uint8_t tag=m_inbox[i]->getTag();
@@ -357,31 +508,37 @@ void ComputeCore::runWithProfiler(){
uint64_t endingTime = getThreadMicroseconds();
int difference = endingTime - startingTime;
-
- m_profiler.addGranularity(currentSlaveMode,difference);
+
+ if(profileGranularity)
+ m_profiler.addGranularity(currentSlaveMode,difference);
/* threshold to say something is taking too long */
/* in microseconds */
int tooLong=m_profiler.getThreshold();
- if(difference >= tooLong){
+ if(debugModeIsEnabled() && profileGranularity && difference >= tooLong){
cout<<"Warning, SlaveMode= "<<SLAVE_MODES[currentSlaveMode]<<" GranularityInMicroseconds= "<<difference<<""<<endl;
m_profiler.printStack();
}
- m_profiler.resetStack();
+ if(profileGranularity)
+ m_profiler.resetStack();
int messagesSentInProcessData = m_outbox.size() - messagesSentInProcessMessages;
sentMessagesInProcessData += messagesSentInProcessData;
sentMessages += messagesSentInProcessData;
for(int i=0;i<messagesSentInProcessMessages;i++){
+ if(!debugModeIsEnabled())
+ break;
// stript routing information, if any
uint8_t tag=m_outbox[i]->getTag();
sentTagsInProcessMessages[tag]++;
}
for(int i=messagesSentInProcessMessages;i<(int)m_outbox.size();i++){
+ if(!debugModeIsEnabled())
+ break;
// stript routing information, if any
uint8_t tag=m_outbox[i]->getTag();
sentTagsInProcessData[tag]++;
@@ -392,9 +549,11 @@ void ComputeCore::runWithProfiler(){
/* increment ticks */
ticks++;
+ globalTicks ++;
}
- m_profiler.printAllGranularities();
+ if(debugModeIsEnabled() && profileGranularity)
+ m_profiler.printAllGranularities();
}
void ComputeCore::processMessages(){
@@ -413,8 +572,8 @@ void ComputeCore::processMessages(){
// if the message has routing tag, we don't need to process it...
/*
- * If a message is routed, the original message needs to be destroyed
- * with fire. Otherwise, a slave mode or a master mode will pick it up,
+ * If a message is routed, the original message needs to be destroyed
+ * with fire. Otherwise, a slave mode or a master mode will pick it up,
* which will cause silly behaviors.
*/
m_inbox.clear();
@@ -423,10 +582,36 @@ void ComputeCore::processMessages(){
}
}
+ // the message is for us...
+ // load actor metadata
+ for(int i = 0 ; i < (int) m_inbox.size() ; ++i) {
+ Message * message = m_inbox.at(i);
+
+ // save actor metadata
+ message->loadActorMetaData();
+
+ // dispatch the message
+ if(message->isActorModelMessage()) {
+ m_playground.receiveActorMessage(message);
+ }
+ }
+
+ if(m_inbox.size() > 0 && m_showCommunicationEvents){
+ uint64_t theTime=getMicroseconds();
+ uint64_t microseconds=theTime - m_startingTimeMicroseconds;
+ for(int i=0;i<(int)m_inbox.size();i++){
+ cout<<"[Communication] "<<microseconds<<" microseconds, RECEIVE ";
+ m_inbox[i]->print();
+ cout<<endl;
+ }
+ }
+
+
+
Message*message=m_inbox[0];
MessageTag messageTag=message->getTag();
- #ifdef ASSERT
+ #ifdef ASSERT2
assert(messageTag!=INVALID_HANDLE);
assert(m_allocatedMessageTags.count(messageTag)>0);
string symbol=MESSAGE_TAGS[messageTag];
@@ -441,6 +626,42 @@ void ComputeCore::processMessages(){
}
void ComputeCore::sendMessages(){
+
+ // register actor meta-data
+
+ for(int i = 0 ; i < (int) m_outbox.size() ; ++i) {
+ Message * message = m_outbox.at(i);
+
+ // we need a buffer to store the actor metadata
+ if(message->getBuffer() == NULL) {
+ void * buffer = m_outboxAllocator.allocate(0);
+ message->setBuffer((MessageUnit*)buffer);
+ }
+
+ /*
+ cout << "DEBUG Before saving ";
+ message->printActorMetaData();
+ cout << endl;
+ */
+
+ // save actor metadata
+ // routed message already have this
+ if(!m_router.isRoutingTag(message->getTag()))
+ message->saveActorMetaData();
+
+ /*
+ cout << "DEBUG After saving ";
+ message->printActorMetaData();
+ cout << endl;
+ */
+ }
+
+/*
+ * What follows is all the crap for checking some assertions and
+ * profiling events.
+ * The code that actually sends something is at the end
+ * of this method.
+ */
// assert that we did not overflow the ring
#ifdef ASSERT
if(m_outboxAllocator.getCount() > m_maximumAllocatedOutboxBuffers){
@@ -451,6 +672,7 @@ void ComputeCore::sendMessages(){
}
assert(m_outboxAllocator.getCount()<=m_maximumAllocatedOutboxBuffers);
+
m_outboxAllocator.resetCount();
int messagesToSend=m_outbox.size();
if(messagesToSend>m_maximumNumberOfOutboxMessages){
@@ -471,6 +693,16 @@ void ComputeCore::sendMessages(){
#endif
+/*
+ * Don't do anything when there are no messages
+ * at all.
+ * This statement needs to be after the ifdef ASSERT above
+ * otherwise the no resetCount call is performed on the
+ * allocator, which will produce a run-time error.
+ */
+ if(m_outbox.size()==0)
+ return;
+
// route messages if the router is enabled
if(m_routerIsEnabled){
// if message routing is enabled,
@@ -478,7 +710,7 @@ void ComputeCore::sendMessages(){
m_router.routeOutcomingMessages();
}
- // parameters.showCommunicationEvents()
+ // parameters.showCommunicationEvents()
if( m_showCommunicationEvents && m_outbox.size() > 0){
uint64_t microseconds=getMicroseconds() - m_startingTimeMicroseconds;
for(int i=0;i<(int)m_outbox.size();i++){
@@ -498,7 +730,18 @@ void ComputeCore::sendMessages(){
}
// finally, send the messages
- m_messagesHandler.sendMessages(&m_outbox,&m_outboxAllocator);
+
+ if(!m_miniRanksAreEnabled){
+/*
+ * Implement the old communication mode too.
+ */
+ m_messagesHandler->sendMessages(&m_outbox,&m_outboxAllocator);
+ }else{
+
+ m_messagesHandler->sendMessagesForComputeCore(&m_outbox,&m_bufferedOutbox);
+ }
+
+ m_outbox.clear();
}
void ComputeCore::addMessageChecksums(){
@@ -590,8 +833,22 @@ void ComputeCore::verifyMessageChecksums(){
* next ComputeCore cycle.
*/
void ComputeCore::receiveMessages(){
+
+/*
+ * clear the inbox for the next iteration
+ */
m_inbox.clear();
- m_messagesHandler.receiveMessages(&m_inbox,&m_inboxAllocator);
+
+ if(!m_miniRanksAreEnabled){
+/*
+ * Implement the old communication model.
+ */
+ m_messagesHandler->receiveMessages(&m_inbox,&m_inboxAllocator);
+ }else{
+
+ m_messagesHandler->receiveMessagesForComputeCore(&m_inbox,&m_inboxAllocator,&m_bufferedInbox);
+
+ }
// verify checksums and remove them
if(m_doChecksum){
@@ -604,18 +861,13 @@ void ComputeCore::receiveMessages(){
#ifdef ASSERT
int receivedMessages=m_inbox.size();
+
+ if(receivedMessages>m_maximumNumberOfInboxMessages)
+ cout<<"[ComputeCore::receiveMessages] inbox has "<<receivedMessages<<" but maximum is "<<m_maximumNumberOfInboxMessages<<endl;
+
assert(receivedMessages<=m_maximumNumberOfInboxMessages);
#endif
- if(m_inbox.size() > 0 && m_showCommunicationEvents){
- uint64_t theTime=getMicroseconds();
- uint64_t microseconds=theTime - m_startingTimeMicroseconds;
- for(int i=0;i<(int)m_inbox.size();i++){
- cout<<"[Communication] "<<microseconds<<" microseconds, RECEIVE ";
- m_inbox[i]->print();
- cout<<endl;
- }
- }
}
/** process data my calling current slave and master methods */
@@ -656,33 +908,47 @@ void ComputeCore::processData(){
m_slaveModeExecutor.callHandler(slave);
m_tickLogger.logSlaveTick(slave);
+
+/*
+ * Clear the inbox for the next iteration.
+ * This is only useful if the RayPlatform runs with mini-ranks.
+ */
+ m_inbox.clear();
}
-void ComputeCore::constructor(int*argc,char***argv){
+void ComputeCore::constructor(int argc,char**argv,int miniRankNumber,int numberOfMiniRanks,int miniRanksPerRank,
+ MessagesHandler*messagesHandler){
- m_doChecksum=false;
+ #ifdef ASSERT
+ assert(miniRankNumber<numberOfMiniRanks);
+ assert(miniRanksPerRank<=numberOfMiniRanks);
+ assert(numberOfMiniRanks>=1);
+ assert(miniRanksPerRank>=1);
+ assert(miniRankNumber>=0);
+ #endif
- // checksum calculation is only tested
- // for cases with sizeof(MessageUnit)>=4 bytes
+ bool useMiniRanks=miniRanksPerRank>1;
- const char verifyMessages[]="-verify-message-integrity";
- const char router[]="-route-messages";
+ m_numberOfMiniRanksPerRank=miniRanksPerRank;
+ m_messagesHandler=messagesHandler;
- m_routerIsEnabled=false;
+ m_destroyed=false;
- int match=0;
+ m_doChecksum=false;
- for(int i=0;i<(*argc);i++){
- if(strcmp( ((*argv)[i]), verifyMessages) == match){
- m_doChecksum=true;
- }else if(strcmp( ((*argv)[i]),router) == match){
- m_routerIsEnabled=true;
- }
- }
+ m_miniRanksAreEnabled=useMiniRanks;
- m_argumentCount=*argc;
- m_argumentValues=*argv;
+ #ifdef CONFIG_DEBUG_CORE
+ cout<<"[ComputeCore::constructor] m_miniRanksAreEnabled= "<<m_miniRanksAreEnabled<<endl;
+ #endif
+ // checksum calculation is only tested
+ // for cases with sizeof(MessageUnit)>=4 bytes
+
+ m_routerIsEnabled=false;
+
+ m_argumentCount=argc;
+ m_argumentValues=argv;
m_resolvedSymbols=false;
@@ -696,37 +962,83 @@ void ComputeCore::constructor(int*argc,char***argv){
m_alive=true;
- m_messagesHandler.constructor(argc,argv);
-
m_runProfiler=false;
m_showCommunicationEvents=false;
m_profilerVerbose=false;
- m_rank=m_messagesHandler.getRank();
- m_size=m_messagesHandler.getSize();
+ m_rank=miniRankNumber;
+ m_size=numberOfMiniRanks;
- if(m_doChecksum){
- cout<<"[RayPlatform] Rank "<<m_rank<<" will compute a CRC32 checksum for any non-empty message."<<" ("<<verifyMessages<<")"<<endl;
+ for(int i=0;i<MAXIMUM_NUMBER_OF_MASTER_HANDLERS;i++){
+ strcpy(MASTER_MODES[i],"UnnamedMasterMode");
+ }
+ for(int i=0;i<MAXIMUM_NUMBER_OF_SLAVE_HANDLERS;i++){
+ strcpy(SLAVE_MODES[i],"UnnamedSlaveMode");
+ }
+ for(int i=0;i<MAXIMUM_NUMBER_OF_TAG_HANDLERS;i++){
+ strcpy(MESSAGE_TAGS[i],"UnnamedMessageTag");
}
+ m_currentSlaveModeToAllocate=0;
+ m_currentMasterModeToAllocate=0;
+ m_currentMessageTagToAllocate=0;
+}
+
+void ComputeCore::configureEngine() {
+
// set the number of buffers to use
int minimumNumberOfBuffers=128;
int availableBuffers=minimumNumberOfBuffers;
+ /**
+ * Instead of limiting the buffers, just use a sane amount already.
+ *
+ * The VirtualCommunicator layer will use one group per tag-rank tuple anyway.
+ * So using more buffers does not change anything.
+ *
+ * At 4096 ranks, this will use 32768000 bytes, or roughly 32 MiB.
+ *
+ * ***************************
+ *
+ * On the IBM Blue Gene/Q in Toronto, using 1024 nodes provides 16 TiB of memory.
+ * At the moment, the highly symmetric identity mapping in the virtual
+ * memory layer on BGQ prohibits Ray at low memory ranges.
+ *
+ * This is because most of Ray processes uses < 800 MiB, but a few rogue
+ * processes sit sometimes at 2.5 GiB. Rank 0 is one of them since it is the
+ * master of the tribe. If 0 dies, so does everyone.
+ *
+ * Access to bgq-fen0 is provided by SciNet. The hardware is owned by
+ * SOSCIP | Southern Ontario Smart Computing Innovation Platform
+ *
+ * \see https://support.scinet.utoronto.ca/wiki/index.php/BGQ
+ * \see http://soscip.org/
+ *
+ * ***************************
+ *
+ * On the Cray XE6 at Cray, Inc. (collaboration with Cray, Inc. with Carlos Sosa), there was
+ * a segmentation fault on 2048 ranks.
+ *
+ * This was most likely due to this lack of buffers. Because of this depletion,
+ * buffers were reused.
+ */
+ bool useMoreBuffers = true;
+
// even a message with a NULL buffer requires a buffer for routing
- if(m_routerIsEnabled)
+ if(useMoreBuffers || m_routerIsEnabled || m_miniRanksAreEnabled)
availableBuffers=m_size*2;
// this will occur when using the virtual router with a few processes
if(availableBuffers<minimumNumberOfBuffers)
availableBuffers=minimumNumberOfBuffers;
-
m_switchMan.constructor(m_rank,m_size);
m_virtualCommunicator.constructor(m_rank,m_size,&m_outboxAllocator,&m_inbox,&m_outbox);
+ m_keyValueStore.initialize(m_rank, m_size, &m_outboxAllocator, &m_inbox, &m_outbox);
+
/***********************************************************************************/
/** initialize the VirtualProcessor */
m_virtualProcessor.constructor(&m_outbox,&m_inbox,&m_outboxAllocator,
@@ -734,7 +1046,7 @@ void ComputeCore::constructor(int*argc,char***argv){
m_maximumAllocatedInboxBuffers=1;
m_maximumAllocatedOutboxBuffers=availableBuffers;
-
+
m_maximumNumberOfInboxMessages=1;
m_maximumNumberOfOutboxMessages=availableBuffers;
@@ -745,29 +1057,33 @@ void ComputeCore::constructor(int*argc,char***argv){
}
getInbox()->constructor(m_maximumNumberOfInboxMessages,"RAY_MALLOC_TYPE_INBOX_VECTOR",false);
-
- #if 0
- cout<<"[ComputeCore] the inbox capacity is "<<m_maximumNumberOfInboxMessages<<" message"<<endl;
- #endif
-
getOutbox()->constructor(m_maximumNumberOfOutboxMessages,"RAY_MALLOC_TYPE_OUTBOX_VECTOR",false);
- #if 0
+ if(m_miniRanksAreEnabled){
+
+ getBufferedInbox()->constructor(m_maximumNumberOfOutboxMessages);
+ getBufferedOutbox()->constructor(m_maximumNumberOfOutboxMessages);
+ }
+
+ #ifdef CONFIG_DEBUG_CORE
+ cout<<"[ComputeCore] the inbox capacity is "<<m_maximumNumberOfInboxMessages<<" message"<<endl;
cout<<"[ComputeCore] the outbox capacity is "<<m_maximumNumberOfOutboxMessages<<" message"<<endl;
#endif
int maximumMessageSizeInByte=MAXIMUM_MESSAGE_SIZE_IN_BYTES;
+ bool useActorModel = true;
+
// add a message unit to store the checksum or the routing information
// with 64-bit integers as MessageUnit, this is 4008 bytes or 501 MessageUnit maximum
- // TODO: RayPlatform can not do both routing and checksums
- if(m_doChecksum || m_routerIsEnabled){
+ // also add 8 bytes for actor metadata
+ if(m_miniRanksAreEnabled || m_doChecksum || m_routerIsEnabled || useActorModel){
if(sizeof(MessageUnit)>=4){
- maximumMessageSizeInByte+=sizeof(MessageUnit);
+ maximumMessageSizeInByte+=(2 + 2)*sizeof(MessageUnit);
}else if(sizeof(MessageUnit)>=2){
- maximumMessageSizeInByte+=2*sizeof(MessageUnit);
+ maximumMessageSizeInByte+=(2 + 2) *2*sizeof(MessageUnit);
}else{
- maximumMessageSizeInByte+=4*sizeof(MessageUnit);
+ maximumMessageSizeInByte+=(2 + 2) *4*sizeof(MessageUnit);
}
}
@@ -775,32 +1091,41 @@ void ComputeCore::constructor(int*argc,char***argv){
maximumMessageSizeInByte,
"RAY_MALLOC_TYPE_INBOX_ALLOCATOR",false);
- #if 0
- cout<<"[ComputeCore] allocated "<<m_maximumAllocatedInboxBuffers<<" buffers of size "<<maximumMessageSizeInByte<<" for inbox messages"<<endl;
- #endif
-
m_outboxAllocator.constructor(m_maximumAllocatedOutboxBuffers,
maximumMessageSizeInByte,
"RAY_MALLOC_TYPE_OUTBOX_ALLOCATOR",false);
- #if 0
+ if(m_miniRanksAreEnabled){
+
+ #if 0
+ cout<<"Building buffered inbox allocator."<<endl;
+ #endif
+
+ m_bufferedInboxAllocator.constructor(m_maximumAllocatedOutboxBuffers,
+ maximumMessageSizeInByte,
+ "RAY_MALLOC_TYPE_INBOX_ALLOCATOR/Buffered",false);
+
+ m_bufferedOutboxAllocator.constructor(m_maximumAllocatedOutboxBuffers,
+ maximumMessageSizeInByte,
+ "RAY_MALLOC_TYPE_OUTBOX_ALLOCATOR/Buffered",false);
+
+ }
+
+ #ifdef CONFIG_DEBUG_CORE
+ cout<<"[ComputeCore] allocated "<<m_maximumAllocatedInboxBuffers<<" buffers of size "<<maximumMessageSizeInByte<<" for inbox messages"<<endl;
cout<<"[ComputeCore] allocated "<<m_maximumAllocatedOutboxBuffers<<" buffers of size "<<maximumMessageSizeInByte<<" for outbox messages"<<endl;
#endif
- for(int i=0;i<MAXIMUM_NUMBER_OF_MASTER_HANDLERS;i++){
- strcpy(MASTER_MODES[i],"UnnamedMasterMode");
- }
- for(int i=0;i<MAXIMUM_NUMBER_OF_SLAVE_HANDLERS;i++){
- strcpy(SLAVE_MODES[i],"UnnamedSlaveMode");
- }
- for(int i=0;i<MAXIMUM_NUMBER_OF_TAG_HANDLERS;i++){
- strcpy(MESSAGE_TAGS[i],"UnnamedMessageTag");
- }
+}
- m_currentSlaveModeToAllocate=0;
- m_currentMasterModeToAllocate=0;
- m_currentMessageTagToAllocate=0;
+void ComputeCore::enableCheckSums(){
+ m_doChecksum=true;
+}
+void ComputeCore::setMiniRank(int miniRank,int numberOfMiniRanks){
+
+ m_rank=miniRank;
+ m_size=numberOfMiniRanks;
}
void ComputeCore::enableProfiler(){
@@ -815,10 +1140,6 @@ void ComputeCore::enableProfilerVerbosity(){
m_profilerVerbose=true;
}
-MessagesHandler*ComputeCore::getMessagesHandler(){
- return &m_messagesHandler;
-}
-
Profiler*ComputeCore::getProfiler(){
return &m_profiler;
}
@@ -864,7 +1185,7 @@ VirtualCommunicator*ComputeCore::getVirtualCommunicator(){
}
void ComputeCore::registerPlugin(CorePlugin*plugin){
-
+
if(m_firstRegistration){
m_firstRegistration=false;
@@ -872,6 +1193,7 @@ void ComputeCore::registerPlugin(CorePlugin*plugin){
// register some built-in plugins
registerPlugin(&m_switchMan);
+ registerPlugin(&m_keyValueStore);
}
@@ -884,7 +1206,7 @@ void ComputeCore::resolveSymbols(){
if(m_resolvedSymbols)
return;
- registerPlugin(&m_messagesHandler); // must be the last registered
+ registerDummyPlugin();
for(int i=0;i<(int)m_listOfPlugins.size();i++){
CorePlugin*plugin=m_listOfPlugins[i];
@@ -894,8 +1216,53 @@ void ComputeCore::resolveSymbols(){
m_resolvedSymbols=true;
}
+void ComputeCore::registerDummyPlugin(){
+ ComputeCore*core=this;
+ PluginHandle plugin=core->allocatePluginHandle();
+
+ core->setPluginName(plugin,"DummySun");
+ core->setPluginDescription(plugin,"Dummy plugin, does nothing.");
+ core->setPluginAuthors(plugin,"Sébastien Boisvert");
+ core->setPluginLicense(plugin,"GNU Lesser General License version 3");
+
+ RAY_MPI_TAG_DUMMY=core->allocateMessageTagHandle(plugin);
+ core->setMessageTagSymbol(plugin,RAY_MPI_TAG_DUMMY,"RAY_MPI_TAG_DUMMY");
+
+}
+
void ComputeCore::destructor(){
- getMessagesHandler()->destructor();
+
+ #if 0
+ cout<<"ComputeCore::destructor"<<endl;
+ #endif
+
+ if(m_destroyed)
+ return;
+
+ m_destroyed=true;
+
+ #ifdef CONFIG_DEBUG_CORE
+ cout<<"Rank "<<m_rank<<" is cleaning the inbox allocator."<<endl;
+ #endif
+
+ m_inboxAllocator.clear();
+
+ if(m_miniRanksAreEnabled)
+ m_bufferedInboxAllocator.clear();
+
+ #ifdef CONFIG_DEBUG_CORE
+ cout<<"Rank "<<m_rank<<" is cleaning the outbox allocator."<<endl;
+ #endif
+
+ m_outboxAllocator.clear();
+
+ if(m_miniRanksAreEnabled)
+ m_bufferedOutboxAllocator.clear();
+
+ if(m_miniRanksAreEnabled){
+ m_bufferedInbox.destructor();
+ m_bufferedOutbox.destructor();
+ }
}
void ComputeCore::stop(){
@@ -905,7 +1272,7 @@ void ComputeCore::stop(){
void ComputeCore::setSlaveModeSymbol(PluginHandle plugin,SlaveMode mode,const char*symbol){
if(!validationPluginAllocated(plugin))
return;
-
+
if(!validationSlaveModeRange(plugin,mode))
return;
@@ -948,7 +1315,7 @@ void ComputeCore::setSlaveModeSymbol(PluginHandle plugin,SlaveMode mode,const ch
}
void ComputeCore::setMasterModeSymbol(PluginHandle plugin,MasterMode mode,const char*symbol){
-
+
if(!validationPluginAllocated(plugin))
return;
@@ -1028,7 +1395,7 @@ void ComputeCore::setMessageTagSymbol(PluginHandle plugin,MessageTag tag,const c
PluginHandle ComputeCore::allocatePluginHandle(){
PluginHandle handle=generatePluginHandle();
-
+
while(m_plugins.count(handle)>0){
handle=generatePluginHandle();
}
@@ -1074,7 +1441,7 @@ SlaveMode ComputeCore::allocateSlaveModeHandle(PluginHandle plugin){
assert(m_allocatedSlaveModes.count(handle)>0);
assert(m_plugins[plugin].hasSlaveMode(handle));
#endif
-
+
return handle;
}
@@ -1112,7 +1479,7 @@ MasterMode ComputeCore::allocateMasterModeHandle(PluginHandle plugin){
assert(m_plugins[plugin].hasMasterMode(handle));
#endif
- return handle;
+ return handle;
}
MessageTag ComputeCore::allocateMessageTagHandle(PluginHandle plugin){
@@ -1220,7 +1587,7 @@ void ComputeCore::printPlugins(string directory){
for(map<PluginHandle,RegisteredPlugin>::iterator i=m_plugins.begin();
i!=m_plugins.end();i++){
-
+
f1<<i->first<<" plugin_"<<i->second.getPluginName()<<endl;
}
@@ -1251,7 +1618,7 @@ SlaveMode ComputeCore::getSlaveModeFromSymbol(PluginHandle plugin,const char*sym
if(m_slaveModeSymbols.count(key)>0){
SlaveMode handle=m_slaveModeSymbols[key];
-
+
m_plugins[plugin].addResolvedSlaveMode(handle);
#ifdef CONFIG_DEBUG_SLAVE_SYMBOLS
@@ -1537,7 +1904,11 @@ void ComputeCore::setMasterModeNextMasterMode(PluginHandle plugin,MasterMode cur
m_plugins[plugin].addRegisteredMasterModeNextMasterMode(current);
}
-void ComputeCore::setFirstMasterMode(PluginHandle plugin,MasterMode mode){
+bool ComputeCore::isPublicMasterMode(PluginHandle plugin,MasterMode mode){
+ return m_publicMasterModes.count(mode)>0;
+}
+
+void ComputeCore::setMasterModePublicAccess(PluginHandle plugin,MasterMode mode){
if(!validationPluginAllocated(plugin))
return;
@@ -1547,6 +1918,19 @@ void ComputeCore::setFirstMasterMode(PluginHandle plugin,MasterMode mode){
if(!validationMasterModeOwnership(plugin,mode))
return;
+ m_publicMasterModes.insert(mode);
+}
+
+void ComputeCore::setFirstMasterMode(PluginHandle plugin,MasterMode mode){
+ if(!validationPluginAllocated(plugin))
+ return;
+
+ if(!validationMasterModeRange(plugin,mode))
+ return;
+
+ if(!isPublicMasterMode(plugin,mode) && !validationMasterModeOwnership(plugin,mode))
+ return;
+
if(m_hasFirstMode)
cout<<"Error, already has a first master mode."<<endl;
@@ -1679,3 +2063,90 @@ int ComputeCore::getNumberOfArguments(){
char**ComputeCore::getArgumentValues(){
return m_argumentValues;
}
+
+bool ComputeCore::hasFinished(){
+ bool alive=*(getLife());
+
+ return !alive;
+}
+
+int ComputeCore::getRank() const{
+ return m_rank;
+}
+
+int ComputeCore::getSize() const{
+ return m_size;
+}
+
+MessageQueue*ComputeCore::getBufferedInbox(){
+ return &m_bufferedInbox;
+}
+
+MessageQueue*ComputeCore::getBufferedOutbox(){
+ return &m_bufferedOutbox;
+}
+
+RingAllocator*ComputeCore::getBufferedOutboxAllocator(){
+ return &m_bufferedOutboxAllocator;
+}
+
+RingAllocator*ComputeCore::getBufferedInboxAllocator(){
+ return &m_bufferedInboxAllocator;
+}
+
+int ComputeCore::getMiniRanksPerRank(){
+ return m_numberOfMiniRanksPerRank;
+}
+
+MessagesHandler*ComputeCore::getMessagesHandler(){
+ return m_messagesHandler;
+}
+
+void ComputeCore::closeSlaveModeLocally() {
+
+ getSwitchMan()->closeSlaveModeLocally(getOutbox(), getRank());
+}
+
+KeyValueStore & ComputeCore::getKeyValueStore() {
+
+ return m_keyValueStore;
+}
+
+#if 0
+void ComputeCore::runRayPlatformTerminal() {
+
+ time_t seconds = time(NULL);
+
+ int delay = seconds - m_lastTerminalProbeOperation;
+ int minimumDelay = 60;
+
+ if(delay > minimumDelay) {
+
+ }
+}
+#endif
+
+Playground * ComputeCore::getPlayground() {
+
+ return & m_playground;
+}
+
+void ComputeCore::spawnActor(Actor * actor) {
+
+ getPlayground()->spawnActor(actor);
+}
+
+void ComputeCore::send(Message * message) {
+
+ m_outbox.push_back(message);
+}
+
+bool ComputeCore::isRankAlive() const {
+
+ if(m_playground.hasAliveActors())
+ return true;
+
+ return m_alive;
+}
+
+
diff --git a/RayPlatform/core/ComputeCore.h b/RayPlatform/RayPlatform/core/ComputeCore.h
similarity index 74%
rename from RayPlatform/core/ComputeCore.h
rename to RayPlatform/RayPlatform/core/ComputeCore.h
index 905806c..95ef58d 100644
--- a/RayPlatform/core/ComputeCore.h
+++ b/RayPlatform/RayPlatform/core/ComputeCore.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform: a message-passing development framework
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,32 +22,48 @@
#ifndef _ComputeCore_h
#define _ComputeCore_h
-#include <handlers/MessageTagHandler.h>
-#include <handlers/MessageTagExecutor.h>
-#include <handlers/SlaveModeHandler.h>
-#include <handlers/MasterModeHandler.h>
-#include <handlers/MasterModeExecutor.h>
-#include <handlers/SlaveModeExecutor.h>
-#include <communication/MessagesHandler.h>
-#include <profiling/Profiler.h>
-#include <structures/StaticVector.h>
-#include <communication/Message.h>
-#include <communication/MessageRouter.h>
-#include <profiling/TickLogger.h>
-#include <scheduling/SwitchMan.h>
-#include <memory/RingAllocator.h>
-#include <scheduling/VirtualProcessor.h>
-#include <communication/VirtualCommunicator.h>
-#include <plugins/CorePlugin.h>
-#include <plugins/RegisteredPlugin.h>
-#include <core/OperatingSystem.h>
+/*
+ * For the runtime configuration parameters.
+ */
+#include "types.h"
+#include "OperatingSystem.h"
+
+#include <RayPlatform/handlers/MessageTagHandler.h>
+#include <RayPlatform/handlers/MessageTagExecutor.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/handlers/MasterModeHandler.h>
+#include <RayPlatform/handlers/MasterModeExecutor.h>
+#include <RayPlatform/handlers/SlaveModeExecutor.h>
+
+#include <RayPlatform/profiling/Profiler.h>
+#include <RayPlatform/profiling/TickLogger.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/scheduling/SwitchMan.h>
+#include <RayPlatform/scheduling/VirtualProcessor.h>
+
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/communication/MessageRouter.h>
+#include <RayPlatform/communication/MessagesHandler.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/communication/MessageQueue.h>
+
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/plugins/CorePlugin.h>
+#include <RayPlatform/plugins/RegisteredPlugin.h>
+
+#include <RayPlatform/store/KeyValueStore.h>
+
+#include <RayPlatform/actors/Playground.h>
#include <iostream>
+using namespace std;
+
#include <stdlib.h>
#include <stdint.h>
-using namespace std;
+#include <time.h>
-/** this class is a compute core
+/**
+ * this class is a compute core
* to use it, you must set the handlers of your program
* using setMessageTagObjectHandler(), setSlaveModeObjectHandler(),
* and setMasterModeObjectHandler().
@@ -58,6 +74,40 @@ using namespace std;
*/
class ComputeCore{
+ //time_t m_lastTerminalProbeOperation;
+
+ Playground m_playground;
+
+ KeyValueStore m_keyValueStore;
+
+/*
+ * This is the middleware communication layer.
+ * All messages go through it.
+ */
+ MessagesHandler*m_messagesHandler;
+
+/*
+ * In the legacy mode, RayPlatform will not
+ * spawn a thread for MPI and every MPI rank will do
+ * its own communication.
+ *
+ * If the legacy mode is disabled, each rank runs in
+ * a thread (the mini-ranks) and one thread is used
+ * for MPI.
+ */
+ bool m_miniRanksAreEnabled;
+
+ int m_numberOfMiniRanksPerRank;
+
+ bool m_destroyed;
+
+ MessageQueue m_bufferedInbox;
+ MessageQueue m_bufferedOutbox;
+ RingAllocator m_bufferedOutboxAllocator;
+ RingAllocator m_bufferedInboxAllocator;
+
+ bool m_outboxIsFull;
+
int m_argumentCount;
char**m_argumentValues;
@@ -68,6 +118,7 @@ class ComputeCore{
bool m_firstRegistration;
vector<CorePlugin*> m_listOfPlugins;
+ set<MasterMode> m_publicMasterModes;
vector<void*> m_objects;
@@ -128,8 +179,7 @@ class ComputeCore{
StaticVector m_outbox;
StaticVector m_inbox;
- /** middleware to handle messages */
- MessagesHandler m_messagesHandler;
+ MessageTag RAY_MPI_TAG_DUMMY;
/** this object handles messages */
MessageTagExecutor m_messageTagExecutor;
@@ -192,14 +242,19 @@ class ComputeCore{
void verifyMessageChecksums();
void addMessageChecksums();
+
+ void registerDummyPlugin();
+ bool isPublicMasterMode(PluginHandle plugin,MasterMode mode);
+ void configureEngine();
+
+ bool isRankAlive() const;
+
public:
/** this is the main method */
void run();
- /** get the middleware object */
- MessagesHandler*getMessagesHandler();
-
- void constructor(int*argc,char***argv);
+ void constructor(int argc,char**argv,int miniRankNumber,int numberOfMiniRanks,int miniRanksPerRank,
+ MessagesHandler*messagesHandler);
void enableProfiler();
void showCommunicationEvents();
@@ -217,6 +272,8 @@ public:
RingAllocator*getOutboxAllocator();
RingAllocator*getInboxAllocator();
+ RingAllocator*getBufferedOutboxAllocator();
+ RingAllocator*getBufferedInboxAllocator();
bool*getLife();
@@ -264,7 +321,7 @@ public:
MasterMode getMasterModeFromSymbol(PluginHandle plugin,const char*symbol);
/** add a master mode handler */
- void setMasterModeObjectHandler(PluginHandle plugin,MasterMode mode,MasterModeHandler object);
+ void setMasterModeObjectHandler(PluginHandle plugin,MasterMode mode,MasterModeHandlerReference object);
/** sets the master mode switch for a master mode
* this tells the core which message tag is to be automaticalled broadcasted for
@@ -292,7 +349,7 @@ Not all master modes have yet been ported to that list.
void setSlaveModeSymbol(PluginHandle plugin,SlaveMode mode,const char*symbol);
/** add a slave mode handler */
- void setSlaveModeObjectHandler(PluginHandle plugin,SlaveMode mode,SlaveModeHandler object);
+ void setSlaveModeObjectHandler(PluginHandle plugin,SlaveMode mode,SlaveModeHandlerReference object);
/** get a slave mode from its symbol **/
SlaveMode getSlaveModeFromSymbol(PluginHandle plugin,const char*symbol);
@@ -309,7 +366,7 @@ Not all master modes have yet been ported to that list.
MessageTag getMessageTagFromSymbol(PluginHandle plugin,const char*symbol);
/** add a message tag handler */
- void setMessageTagObjectHandler(PluginHandle plugin,MessageTag tag,MessageTagHandler object);
+ void setMessageTagObjectHandler(PluginHandle plugin,MessageTag tag,MessageTagHandlerReference object);
/** sets the slave switch for a slave mode
* this tells the core which slave mode to switch to when receiving a particular message tag **/
@@ -334,6 +391,31 @@ Not all master modes have yet been ported to that list.
int getNumberOfArguments();
char**getArgumentValues();
+
+ void setMiniRank(int miniRank,int numberOfMiniRanks);
+
+ bool hasFinished();
+
+ MessageQueue*getBufferedInbox();
+ MessageQueue*getBufferedOutbox();
+
+ int getRank() const;
+ int getSize() const;
+
+ int getMiniRanksPerRank();
+
+ MessagesHandler*getMessagesHandler();
+ void setMasterModePublicAccess(PluginHandle plugin,MasterMode mode);
+ void enableCheckSums();
+
+ void closeSlaveModeLocally();
+
+ KeyValueStore & getKeyValueStore();
+ bool debugModeIsEnabled();
+
+ Playground * getPlayground();
+ void spawnActor(Actor * actor);
+ void send(Message * message);
};
#endif
diff --git a/RayPlatform/core/slave_modes.h b/RayPlatform/RayPlatform/core/MiniRank.cpp
similarity index 70%
copy from RayPlatform/core/slave_modes.h
copy to RayPlatform/RayPlatform/core/MiniRank.cpp
index 4c85d67..9497d57 100644
--- a/RayPlatform/core/slave_modes.h
+++ b/RayPlatform/RayPlatform/core/MiniRank.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform: a message-passing development framework
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -16,13 +16,13 @@
You have received a copy of the GNU Lesser General Public License
along with this program (lgpl-3.0.txt).
see <http://www.gnu.org/licenses/>
+
*/
-#ifndef _slave_modes
-#define _slave_modes
+#include "MiniRank.h"
-#include <core/types.h>
+ComputeCore*MiniRank::getCore(){
+ return &m_computeCore;
+}
-extern char SLAVE_MODES[MAXIMUM_NUMBER_OF_SLAVE_HANDLERS][128];
-#endif
diff --git a/RayPlatform/scheduling/Worker.h b/RayPlatform/RayPlatform/core/MiniRank.h
similarity index 55%
copy from RayPlatform/scheduling/Worker.h
copy to RayPlatform/RayPlatform/core/MiniRank.h
index 322586c..6bfaad1 100644
--- a/RayPlatform/scheduling/Worker.h
+++ b/RayPlatform/RayPlatform/core/MiniRank.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform: a message-passing development framework
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -16,34 +16,31 @@
You have received a copy of the GNU Lesser General Public License
along with this program (lgpl-3.0.txt).
see <http://www.gnu.org/licenses/>
+
*/
-#ifndef _Worker_h
-#define _Worker_h
+#ifndef _MiniRank_h
+#define _MiniRank_h
-#include <stdint.h>
-#include <memory/RingAllocator.h>
-#include <communication/VirtualCommunicator.h>
+#include "ComputeCore.h"
-/** a general worker class
+/**
+ * The class for a minirank.
+ * This is the only place in RayPlatform where there is
+ * any lock. Here, the design uses spinlocks.
+ *
* \author Sébastien Boisvert
+ *
+ * \see https://computing.llnl.gov/tutorials/pthreads/
*/
-class Worker{
+class MiniRank{
+protected:
+ ComputeCore m_computeCore;
public:
- /** work a little bit
- * the class Worker provides no implementation for that
- */
- virtual void work() = 0;
-
- /** is the worker done doing its things */
- virtual bool isDone() = 0;
-
- /** get the worker number */
- virtual WorkerHandle getWorkerIdentifier() = 0 ;
-
- virtual ~Worker(){}
+ virtual void run()=0;
+ ComputeCore*getCore();
};
#endif
diff --git a/RayPlatform/core/OperatingSystem.cpp b/RayPlatform/RayPlatform/core/OperatingSystem.cpp
similarity index 73%
rename from RayPlatform/core/OperatingSystem.cpp
rename to RayPlatform/RayPlatform/core/OperatingSystem.cpp
index 2fe05ba..b94215f 100644
--- a/RayPlatform/core/OperatingSystem.cpp
+++ b/RayPlatform/RayPlatform/core/OperatingSystem.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -14,19 +14,37 @@
GNU Lesser General Public License for more details.
You have received a copy of the GNU Lesser General Public License
- along with this program (lgpl-3.0.txt).
+ along with this program (lgpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include <core/OperatingSystem.h>
+#include "OperatingSystem.h"
+
#include <iostream>
#include <string>
#include <fstream>
+#include <map>
#include <assert.h>
using namespace std;
/*
+ * I am not sure that dirent.h is available on Microsoft(R) Windows(R).
+ * Anyway, at this point, you are better off installing Cygwin in Windows(R)
+ * to acquire a POSIX compatibility layer on this otherwise not-POSIX
+ * system
+ */
+#include <dirent.h> /* for opendir, readdir and closedir */
+
+
+/*
+ * Use System Programming Interface on the IBM Blue Gene/Q to get memory usage.
+ */
+#ifdef __bgq__
+#include <spi/include/kernel/memory.h>
+#endif
+
+/*
* Detect the operating system
*/
@@ -67,7 +85,7 @@ using namespace std;
#define OS_WIN
/* this will never be picked up because WIN32 or WIN64 will be picked up */
-#elif defined(__BORLANDC__)
+#elif defined(__BORLANDC__)
#define OS_WIN
/* SGI IRIX I guess */
@@ -89,7 +107,7 @@ using namespace std;
#include <unistd.h> /* getpid */
#include <time.h> /* possibly clock_gettime */
-#include <sys/time.h> /* gettimeofday*/
+#include <sys/time.h> /* gettimeofday*/
#include <sys/stat.h> /* mkdir */
#include <sys/types.h> /* mode_t */
@@ -133,11 +151,36 @@ string getOperatingSystem(){
return "Unknown";
}
-/** only ported to Linux */
+/**
+ * only ported to Linux and IBM Blue Gene/Q
+ */
uint64_t getMemoryUsageInKiBytes(){
+
+/*
+ * For the IBM Blue Gene/Q
+ *
+ * The code is based on the one from Argonne National Laboratory wiki.
+ *
+ * \see https://wiki.alcf.anl.gov/parts/index.php/Blue_Gene/Q#Malloc_Information_and_Tuning
+ * \see https://svn.mcs.anl.gov/repos/ZeptoOS/branches/UPC-lib-hack/arch/include/spi/kernel_interface.h
+ * \see https://support.scinet.utoronto.ca/wiki/index.php/BGQ
+ * \see http://www.vlsci.org.au/AvocaPreview
+ */
+ #if defined(__bgq__)
+
+ uint64_t heapSize=0;
+ Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAP,&heapSize);
+
+ return heapSize/1024;// return KiB units
+
+/*
+ * Get the memory usage with a Linux kernel.
+ * It is (wrongly) assumed that /proc is mounted.
+ */
+ #elif defined(__linux__)
+
uint64_t count=0;
- #ifdef __linux__
ifstream f("/proc/self/status");
while(!f.eof()){
string key;
@@ -148,9 +191,15 @@ uint64_t getMemoryUsageInKiBytes(){
}
}
f.close();
- #endif
return count;
+
+ #endif
+
+/*
+ * Any other system will report 0 KiB !
+ */
+ return 0;
}
/** real-time only ported to real-time POSIX systems */
@@ -170,7 +219,7 @@ uint64_t getMicroseconds(){
return seconds*1000*1000+microSeconds;
#elif defined (OS_WIN)
-
+
/* TODO: get microseconds is not implemented on Windows */
// could start with this: http://www.decompile.com/cpp/faq/windows_timer_api.htm
@@ -216,12 +265,12 @@ uint64_t getThreadMicroseconds(){
void createDirectory(const char*directory){
#ifdef OS_POSIX
- /*
+ /*
* S_IRWXU
- read, write, execute/search by owner
+ read, write, execute/search by owner
*
* S_IRWXG
- * read, write, execute/search by group
+ * read, write, execute/search by group
* */
mode_t mode=S_IRWXU | S_IRWXG;
@@ -238,7 +287,7 @@ void createDirectory(const char*directory){
assert(status==0);
#endif
-
+
#elif defined(OS_WIN)
LPSECURITY_ATTRIBUTES attr=NULL;
@@ -253,17 +302,17 @@ bool fileExists(const char*file){
#ifdef OS_POSIX
struct stat st;
int returnValue=stat(file,&st);
-
+
bool theFileExists=(returnValue == 0);
return theFileExists;
-
+
#elif defined(OS_WIN)
/* Return TRUE if file 'fileName' exists */
DWORD fileAttr = GetFileAttributes(fileName);
if(0xFFFFFFFF == fileAttr)
return false;
return true;
-
+
#else
/* not implemented */
#endif
@@ -310,3 +359,33 @@ void printTheSeconds(int difference,ostream*stream){
(*stream)<<seconds<<" seconds";
}
+
+void getDirectoryFiles(string & path, vector<string> & files) {
+ struct dirent *ent;
+ DIR*dir = opendir (path.c_str());
+
+ if(dir != NULL){
+
+ while((ent = readdir (dir)) != NULL){
+ string fileName=ent->d_name;
+
+ files.push_back(fileName);
+ }
+ closedir (dir);
+ }
+}
+
+bool isDirectory(const string & file) {
+
+#ifdef OS_POSIX
+ const char * target = file.c_str();
+
+ struct stat statbuf;
+ stat(target, &statbuf);
+
+ return S_ISDIR(statbuf.st_mode);
+#else
+ // TODO: not implemented...
+ return false;
+#endif
+}
diff --git a/RayPlatform/core/OperatingSystem.h b/RayPlatform/RayPlatform/core/OperatingSystem.h
similarity index 70%
rename from RayPlatform/core/OperatingSystem.h
rename to RayPlatform/RayPlatform/core/OperatingSystem.h
index 715a929..cbc750c 100644
--- a/RayPlatform/core/OperatingSystem.h
+++ b/RayPlatform/RayPlatform/core/OperatingSystem.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -26,25 +26,14 @@
#define EXIT_NEEDS_ARGUMENTS 5
#define EXIT_NO_MORE_MEMORY 42
-/*
- Open-MPI eager threshold is 4k (4096), and this include Open-MPI's metadata.
- tests show that 4096-100 bytes are sent eagerly, too.
- divide that by eight and you get the number of 64-bit integers
- allowed in a single eager communication
-
- * "4096 is rendezvous. For eager, try 4000 or lower. "
- * --Eugene Loh (Oracle)
- * http://www.open-mpi.org/community/lists/devel/2010/11/8700.php
- *
- */
-
/** only this file knows the operating system */
#include <string>
-//#include <core/constants.h>
-#include <communication/MessagesHandler.h>
+#include <vector>
using namespace std;
+#include <stdint.h>
+
/** show memory usage */
uint64_t getMemoryUsageInKiBytes();
@@ -72,6 +61,8 @@ bool fileExists(const char*file);
void printTheSeconds(int seconds,ostream*stream);
+void getDirectoryFiles(string & file, vector<string> & files);
+bool isDirectory(const string & file);
#endif
diff --git a/RayPlatform/RayPlatform/core/RankProcess.h b/RayPlatform/RayPlatform/core/RankProcess.h
new file mode 100644
index 0000000..210acf8
--- /dev/null
+++ b/RayPlatform/RayPlatform/core/RankProcess.h
@@ -0,0 +1,323 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2012 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform: a message-passing development framework
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#ifndef _RankProcess_h
+#define _RankProcess_h
+
+#include "MiniRank.h"
+
+#include <RayPlatform/communication/MessagesHandler.h>
+#include <RayPlatform/core/types.h> /* for CONFIG_MINI_RANKS */
+
+#if defined(CONFIG_DEBUG_MPI_RANK) || defined(ASSERT)
+#include <iostream>
+using namespace std;
+#endif
+
+#ifdef CONFIG_DEBUG_MPI_RANK
+#endif /* CONFIG_DEBUG_MPI_RANK */
+
+#ifdef ASSERT
+#include <assert.h>
+#endif /* ASSERT */
+
+#ifdef CONFIG_MINI_RANKS
+
+#define MAXIMUM_NUMBER_OF_MINIRANKS_PER_RANK 64
+#include <pthread.h>
+
+#else
+
+#define MAXIMUM_NUMBER_OF_MINIRANKS_PER_RANK 1
+
+#endif
+
+
+/*
+ * The Rank is part of the layered minirank
+ * architecture.
+ *
+ * \author Sébastien Boisvert
+ */
+template<class Application>
+class RankProcess{
+
+ int m_argc;
+ char**m_argv;
+
+ bool m_communicate;
+ bool m_destroyed;
+
+/*
+ * Middleware to handle messages.
+ */
+ MessagesHandler m_messagesHandler;
+
+ MiniRank*m_miniRanks[MAXIMUM_NUMBER_OF_MINIRANKS_PER_RANK];
+
+#ifdef CONFIG_MINI_RANKS
+ pthread_t m_threads[MAXIMUM_NUMBER_OF_MINIRANKS_PER_RANK];
+#endif
+
+ ComputeCore*m_cores[MAXIMUM_NUMBER_OF_MINIRANKS_PER_RANK];
+
+/**
+ * The number of this rank.
+ */
+ int m_rank;
+
+/**
+ * The number of ranks.
+ */
+ int m_numberOfRanks;
+
+ int m_numberOfMiniRanksPerRank;
+ int m_numberOfInstalledMiniRanks;
+
+/*
+ * The total number of mini-ranks in ranks.
+ */
+ int m_numberOfMiniRanks;
+
+#ifdef CONFIG_MINI_RANKS
+ void startMiniRanks();
+#endif
+ void startMiniRank();
+
+ void spawnApplicationObjects(int argc,char**argv);
+
+ void addMiniRank(MiniRank*miniRank);
+
+/*
+ * Get the middleware object
+ */
+ MessagesHandler*getMessagesHandler();
+
+public:
+ void constructor(int*argc,char***argv);
+ void destructor();
+ void run();
+};
+
+
+/*
+ * Below are the implementation of these template methods.
+ */
+
+//#define CONFIG_DEBUG_MPI_RANK
+
+template<class Application>
+void RankProcess<Application>::spawnApplicationObjects(int argc,char**argv){
+
+ int miniRanksPerRank=1;
+
+/*
+ * Get the number of mini-ranks per rank.
+ */
+ for(int i=0;i<argc;i++){
+ if(strcmp(argv[i],"-mini-ranks-per-rank")==0 && i+1<argc)
+ miniRanksPerRank=atoi(argv[i+1]);
+ }
+
+ if(miniRanksPerRank<1)
+ miniRanksPerRank=1;
+
+ m_numberOfMiniRanksPerRank=miniRanksPerRank;
+
+/*
+ * Add the mini-ranks.
+ */
+ for(int i=0;i<m_numberOfMiniRanksPerRank;i++){
+
+ MiniRank*miniRank=new Application(argc,argv);
+ addMiniRank(miniRank);
+ }
+}
+
+template<class Application>
+void RankProcess<Application>::constructor(int*argc,char***argv){
+
+ m_numberOfInstalledMiniRanks=0;
+
+ m_messagesHandler.constructor(argc,argv);
+
+ m_destroyed=false;
+
+ spawnApplicationObjects(*argc,*argv);
+
+ #ifdef CONFIG_DEBUG_MPI_RANK
+ cout<<"[RankProcess] m_numberOfMiniRanksPerRank= "<<m_numberOfMiniRanksPerRank<<endl;
+ #endif
+
+ m_numberOfMiniRanks=m_numberOfRanks*m_numberOfMiniRanksPerRank;
+
+ m_argc=*argc;
+ m_argv=*argv;
+
+ m_rank=m_messagesHandler.getRank();
+ m_numberOfRanks=m_messagesHandler.getSize();
+
+ #ifdef CONFIG_DEBUG_MPI_RANK
+ cout<<"[RankProcess] rank= "<<m_rank<<" size= "<<m_numberOfRanks<<endl;
+ #endif
+}
+
+template<class Application>
+void RankProcess<Application>::addMiniRank(MiniRank*miniRank){
+
+ #ifdef CONFIG_DEBUG_MPI_RANK
+ cout<<"Adding mini-rank"<<endl;
+ #endif
+
+ #ifdef ASSERT
+ assert(m_numberOfInstalledMiniRanks<m_numberOfMiniRanksPerRank);
+ #endif
+
+ m_miniRanks[m_numberOfInstalledMiniRanks]=miniRank;
+ m_cores[m_numberOfInstalledMiniRanks]=miniRank->getCore();
+
+ m_numberOfInstalledMiniRanks++;
+
+ #ifdef ASSERT
+ assert(m_numberOfInstalledMiniRanks<=m_numberOfMiniRanksPerRank);
+ #endif
+
+}
+
+void*Rank_startMiniRank(void*object){
+
+
+ ((MiniRank*)object)->run();
+
+ return object;
+}
+
+template<class Application>
+void RankProcess<Application>::run(){
+
+ #ifdef ASSERT
+ if(m_numberOfMiniRanksPerRank!=m_numberOfInstalledMiniRanks){
+ cout<<"Error: "<<m_numberOfInstalledMiniRanks<<" installed mini-ranks, but need "<<m_numberOfMiniRanksPerRank<<endl;
+ }
+
+ assert(m_numberOfMiniRanksPerRank==m_numberOfInstalledMiniRanks);
+ #endif
+
+ bool useMiniRanks=m_numberOfMiniRanksPerRank>1;
+
+#ifndef CONFIG_MINI_RANKS
+ useMiniRanks = false;
+#endif
+
+ if(useMiniRanks){
+#ifdef CONFIG_MINI_RANKS
+ startMiniRanks();
+#endif
+ }else{
+ startMiniRank();
+ }
+
+ #ifdef CONFIG_DEBUG_MPI_RANK
+ cout<<"All mini-ranks are dead."<<endl;
+ #endif
+
+ for(int i=0;i<m_numberOfInstalledMiniRanks;i++)
+ m_cores[i]->destructor();
+
+ destructor();
+}
+
+#ifdef CONFIG_MINI_RANKS
+
+template<class Application>
+void RankProcess<Application>::startMiniRanks(){
+
+ int miniRankNumber=m_messagesHandler.getRank()*m_numberOfMiniRanksPerRank;
+ int numberOfMiniRanks=m_messagesHandler.getSize()*m_numberOfMiniRanksPerRank;
+
+ for(int i=0;i<m_numberOfInstalledMiniRanks;i++){
+ m_cores[i]->constructor(m_argc,m_argv,miniRankNumber+i,numberOfMiniRanks,
+ m_numberOfMiniRanksPerRank,&m_messagesHandler);
+ }
+
+ for(int i=0;i<m_numberOfInstalledMiniRanks;i++){
+
+ #ifdef CONFIG_DEBUG_MPI_RANK
+ cout<<"[RankProcess::run] starting mini-rank in parallel # "<<i<<""<<endl;
+ #endif
+
+/*
+ * TODO: possibly change the stack size -> pthread_attr_setstacksize
+ *
+ * pthread_attr_t attributes;
+ * int stackSize=8*1024*1024;
+ * int errorCode=pthread_attr_setstacksize(&attributes,stackSize);
+ * // manage error
+ *
+ * pthread_create(m_threads+i,&attributes,Rank_startMiniRank,m_miniRanks[i]);
+ *
+ * on Colosse (CentOS 5.7), the default is 10240 KiB
+ * on Fedora 16 and 17, it is 8192 KiB
+ *
+ */
+ pthread_create(m_threads+i,NULL,Rank_startMiniRank,m_miniRanks[i]);
+ }
+
+ m_communicate=true;
+
+ while(m_communicate){
+
+ m_messagesHandler.sendAndReceiveMessagesForRankProcess(m_cores,m_numberOfInstalledMiniRanks,&m_communicate);
+ }
+
+}
+#endif /* CONFIG_MINI_RANKS */
+
+template<class Application>
+void RankProcess<Application>::destructor(){
+
+ if(m_destroyed)
+ return;
+
+ getMessagesHandler()->destructor();
+
+ m_destroyed=true;
+}
+
+template<class Application>
+MessagesHandler*RankProcess<Application>::getMessagesHandler(){
+ return &m_messagesHandler;
+}
+
+template<class Application>
+void RankProcess<Application>::startMiniRank(){
+
+ int miniRankIndex=0;
+
+ m_cores[miniRankIndex]->constructor(m_argc,m_argv,m_rank,m_numberOfRanks,
+ m_numberOfMiniRanksPerRank,&m_messagesHandler);
+
+ m_miniRanks[miniRankIndex]->run();
+}
+
+
+#endif /* _Rank_h */
+
diff --git a/RayPlatform/core/master_modes.cpp b/RayPlatform/RayPlatform/core/master_modes.cpp
similarity index 81%
rename from RayPlatform/core/master_modes.cpp
rename to RayPlatform/RayPlatform/core/master_modes.cpp
index 97d3409..4e753b3 100644
--- a/RayPlatform/core/master_modes.cpp
+++ b/RayPlatform/RayPlatform/core/master_modes.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,7 +19,7 @@
*/
-#include <core/master_modes.h>
+#include "master_modes.h"
char MASTER_MODES[MAXIMUM_NUMBER_OF_MASTER_HANDLERS][128];
diff --git a/RayPlatform/core/master_modes.h b/RayPlatform/RayPlatform/core/master_modes.h
similarity index 81%
rename from RayPlatform/core/master_modes.h
rename to RayPlatform/RayPlatform/core/master_modes.h
index ffcda50..081c9e3 100644
--- a/RayPlatform/core/master_modes.h
+++ b/RayPlatform/RayPlatform/core/master_modes.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,7 +21,7 @@
#ifndef _master_modes
#define _master_modes
-#include <core/types.h>
+#include "types.h"
extern char MASTER_MODES[MAXIMUM_NUMBER_OF_MASTER_HANDLERS][128];
diff --git a/RayPlatform/core/slave_modes.cpp b/RayPlatform/RayPlatform/core/slave_modes.cpp
similarity index 80%
rename from RayPlatform/core/slave_modes.cpp
rename to RayPlatform/RayPlatform/core/slave_modes.cpp
index 7975b96..d711624 100644
--- a/RayPlatform/core/slave_modes.cpp
+++ b/RayPlatform/RayPlatform/core/slave_modes.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,6 +19,7 @@
*/
-#include <core/slave_modes.h>
+#include "slave_modes.h"
char SLAVE_MODES[MAXIMUM_NUMBER_OF_SLAVE_HANDLERS][128];
+
diff --git a/RayPlatform/core/slave_modes.h b/RayPlatform/RayPlatform/core/slave_modes.h
similarity index 81%
rename from RayPlatform/core/slave_modes.h
rename to RayPlatform/RayPlatform/core/slave_modes.h
index 4c85d67..263e50d 100644
--- a/RayPlatform/core/slave_modes.h
+++ b/RayPlatform/RayPlatform/core/slave_modes.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,7 +21,7 @@
#ifndef _slave_modes
#define _slave_modes
-#include <core/types.h>
+#include "types.h"
extern char SLAVE_MODES[MAXIMUM_NUMBER_OF_SLAVE_HANDLERS][128];
diff --git a/RayPlatform/core/statistics.cpp b/RayPlatform/RayPlatform/core/statistics.cpp
similarity index 92%
rename from RayPlatform/core/statistics.cpp
rename to RayPlatform/RayPlatform/core/statistics.cpp
index c5d5436..cd453b1 100644
--- a/RayPlatform/core/statistics.cpp
+++ b/RayPlatform/RayPlatform/core/statistics.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,7 +18,8 @@
see <http://www.gnu.org/licenses/>
*/
-#include <core/statistics.h>
+#include "statistics.h"
+
#include <math.h> /* for sqrt */
#include <stdint.h>
#include <map>
diff --git a/RayPlatform/core/statistics.h b/RayPlatform/RayPlatform/core/statistics.h
similarity index 87%
rename from RayPlatform/core/statistics.h
rename to RayPlatform/RayPlatform/core/statistics.h
index ab7f24d..84d8925 100644
--- a/RayPlatform/core/statistics.h
+++ b/RayPlatform/RayPlatform/core/statistics.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
diff --git a/RayPlatform/core/types.h b/RayPlatform/RayPlatform/core/types.h
similarity index 78%
rename from RayPlatform/core/types.h
rename to RayPlatform/RayPlatform/core/types.h
index c29229f..4a529e1 100644
--- a/RayPlatform/core/types.h
+++ b/RayPlatform/RayPlatform/core/types.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -63,4 +63,17 @@ typedef uint64_t MessageUnit;
#define MAX_NUMBER_OF_MPI_PROCESSES 1000000
#define INVALID_RANK MAX_NUMBER_OF_MPI_PROCESSES
+/**
+ * Enable the mini-ranks technology.
+ *
+ * This is disabled by default.
+ */
+#define CONFIG_MINI_RANKS
+
+/**
+ * With this option, RayPlatform will perform aggressive caching
+ * of operation codes. This reduces the CPU cache footprint of RayPlatform.
+ */
+#define CONFIG_CACHE_OPERATION_CODES
+
#endif
diff --git a/RayPlatform/cryptography/crypto.cpp b/RayPlatform/RayPlatform/cryptography/crypto.cpp
similarity index 89%
rename from RayPlatform/cryptography/crypto.cpp
rename to RayPlatform/RayPlatform/cryptography/crypto.cpp
index daf5e7f..54ec5d8 100644
--- a/RayPlatform/cryptography/crypto.cpp
+++ b/RayPlatform/RayPlatform/cryptography/crypto.cpp
@@ -1,27 +1,29 @@
/*
- RayPlatform
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform: a message-passing development framework
This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
+ it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, version 3 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
+ GNU Lesser General Public License for more details.
- You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include <cryptography/crypto.h>
-#include <string.h>
+
+#include "crypto.h"
+
+#include <string.h>
#ifdef ASSERT
#include <assert.h>
#endif
@@ -34,11 +36,11 @@
*/
uint64_t uniform_hashing_function_1_64_64(uint64_t key){
// some magic here and there.
- key = (~key) + (key << 21);
+ key = (~key) + (key << 21);
key = key ^ (key >> 24);
- key = (key + (key << 3)) + (key << 8);
+ key = (key + (key << 3)) + (key << 8);
key = key ^ (key >> 14);
- key = (key + (key << 2)) + (key << 4);
+ key = (key + (key << 2)) + (key << 4);
key = key ^ (key >> 28);
key = key + (key << 31);
return key;
@@ -48,11 +50,11 @@ uint64_t uniform_hashing_function_1_64_64(uint64_t key){
* based on uniform_hashing_function_1_64_64, but with different values
*/
uint64_t uniform_hashing_function_2_64_64(uint64_t key){
- key = (~key) + (key << 31);
+ key = (~key) + (key << 31);
key = key ^ (key >> 14);
- key = (key + (key << 7)) + (key << 11);
+ key = (key + (key << 7)) + (key << 11);
key = key ^ (key >> 13);
- key = (key + (key << 4)) + (key << 8);
+ key = (key + (key << 4)) + (key << 8);
key = key ^ (key >> 44);
key = key + (key << 6);
return key;
diff --git a/RayPlatform/communication/mpi_tags.h b/RayPlatform/RayPlatform/cryptography/crypto.h
similarity index 55%
rename from RayPlatform/communication/mpi_tags.h
rename to RayPlatform/RayPlatform/cryptography/crypto.h
index 6d6df54..31884f3 100644
--- a/RayPlatform/communication/mpi_tags.h
+++ b/RayPlatform/RayPlatform/cryptography/crypto.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform: a message-passing development framework
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -14,20 +14,22 @@
GNU Lesser General Public License for more details.
You have received a copy of the GNU Lesser General Public License
- along with this program (lgpl-3.0.txt).
+ along with this program (lgpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#ifndef _mpi_tags
-#define _mpi_tags
-// tags for MPI
-// these are the message types used by Ray
-// Ray instances like to communicate a lots!
-#include <core/types.h>
-extern char MESSAGE_TAGS[MAXIMUM_NUMBER_OF_TAG_HANDLERS][128];
+#ifndef _crypto
+#define _crypto
+
+#include <stdint.h>
+
+uint64_t uniform_hashing_function_1_64_64(uint64_t key);
+uint64_t uniform_hashing_function_2_64_64(uint64_t key);
+
+uint32_t computeCyclicRedundancyCode32(uint8_t * bytes, uint32_t numberOfBytes);
#endif
diff --git a/RayPlatform/RayPlatform/files/FileReader.cpp b/RayPlatform/RayPlatform/files/FileReader.cpp
new file mode 100644
index 0000000..055189f
--- /dev/null
+++ b/RayPlatform/RayPlatform/files/FileReader.cpp
@@ -0,0 +1,179 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of RayPlatform.
+
+ RayPlatform is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ RayPlatform is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with RayPlatform. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "FileReader.h"
+
+#include <iostream>
+using namespace std;
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+
+FileReader::FileReader() {
+
+ //m_bufferSize = 2097152;
+ m_bufferSize = 524288;
+
+ m_buffer = NULL;
+ m_startInBuffer = 0;
+ m_availableBytes = 0;
+
+ m_active = false;
+}
+
+FileReader::~FileReader() {
+ if(m_active == true) {
+ close();
+ }
+}
+
+void FileReader::open(const char * file) {
+
+ if(m_active == true)
+ return;
+
+ m_reader.open(file);
+
+#if 0
+ cout << "DEBUG open " << file << endl;
+#endif
+
+ m_active = true;
+}
+
+int FileReader::getAvailableBytes() {
+
+ return m_availableBytes;
+}
+
+char * FileReader::getAvailableBuffer() {
+ return m_buffer + m_startInBuffer;
+}
+
+void FileReader::getline(char * buffer, int size) {
+
+#if 0
+ cout << "DEBUG getline" << endl;
+#endif
+
+ if(m_active == false)
+ return;
+
+ if(size > getAvailableBytes())
+ this->read();
+
+ char * availableBuffer = getAvailableBuffer();
+
+ int availableBytes = getAvailableBytes();
+
+ int i = 0;
+ while(i < availableBytes && i < size
+ && availableBuffer[i] != '\n') {
+
+ i++;
+ }
+
+ if(availableBuffer[i] == '\n')
+ i++;
+
+ memcpy(buffer, availableBuffer, i);
+
+ buffer[i] = '\0';
+
+#if 0
+ cout << "DEBUG m_startInBuffer " << m_startInBuffer;
+ cout << " m_availableBytes " << m_availableBytes;
+ cout << endl;
+#endif
+
+#if 0
+ cout << "DEBUG getline result: " << i << " bytes <result>" << buffer << "</result>" << endl;
+#endif
+
+ m_startInBuffer += i;
+ m_availableBytes -= i;
+
+ //m_reader.getline(buffer, size);
+}
+
+void FileReader::read() {
+
+ if(m_buffer == NULL) {
+ m_buffer = (char * ) malloc(m_bufferSize);
+
+ }
+
+ int source = m_startInBuffer;
+ int destination = 0;
+
+ while(source < m_bufferSize) {
+
+ m_buffer[destination++] = m_buffer[source++];
+ }
+
+ m_startInBuffer = 0;
+
+ int bytes = m_bufferSize - getAvailableBytes();
+
+ char * destinationBuffer = m_buffer + getAvailableBytes();
+
+#if 0
+ cout << "DEBUG read " << bytes << " bytes" << endl;
+#endif
+
+ m_reader.read(destinationBuffer, bytes);
+
+ int bytesRead = m_reader.gcount();
+
+ m_availableBytes += bytesRead;
+
+ //cout << "DEBUG got " << bytesRead << " bytes" << endl;
+}
+
+void FileReader::close() {
+
+ if(!m_active)
+ return;
+
+ m_reader.close();
+
+ if(m_buffer != NULL) {
+ free(m_buffer);
+ m_buffer = NULL;
+ }
+
+ m_active = false;
+}
+
+bool FileReader::eof() {
+
+ if(!m_active)
+ return true;
+
+ if(m_availableBytes == 0 && m_reader.eof())
+ return true;
+
+ return false;
+
+ //return m_reader.eof();
+}
+
diff --git a/RayPlatform/RayPlatform/files/FileReader.h b/RayPlatform/RayPlatform/files/FileReader.h
new file mode 100644
index 0000000..1a668cc
--- /dev/null
+++ b/RayPlatform/RayPlatform/files/FileReader.h
@@ -0,0 +1,60 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of RayPlatform.
+
+ RayPlatform is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ RayPlatform is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with RayPlatform. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef FileReaderHeader
+#define FileReaderHeader
+
+#include <string>
+#include <fstream>
+using namespace std;
+
+/**
+ * This is a wrapper around ifstream.
+ * However, the supported operations are currently limited to
+ * reading lines.
+ *
+ * \author Sébastien Boisvert
+ */
+class FileReader {
+
+private:
+ int m_startInBuffer;
+ int m_availableBytes;
+
+ int m_bufferSize;
+ char * m_buffer;
+ bool m_active;
+ ifstream m_reader;
+
+ int getAvailableBytes();
+ char * getAvailableBuffer();
+ void read();
+public:
+
+ FileReader();
+ ~FileReader();
+
+ void open(const char * file);
+ void getline(char * buffer, int size);
+ void close();
+ bool eof();
+};
+
+#endif
diff --git a/RayPlatform/handlers/MasterModeExecutor.cpp b/RayPlatform/RayPlatform/handlers/MasterModeExecutor.cpp
similarity index 64%
rename from RayPlatform/handlers/MasterModeExecutor.cpp
rename to RayPlatform/RayPlatform/handlers/MasterModeExecutor.cpp
index d0ec30a..c7db0a9 100644
--- a/RayPlatform/handlers/MasterModeExecutor.cpp
+++ b/RayPlatform/RayPlatform/handlers/MasterModeExecutor.cpp
@@ -1,8 +1,8 @@
/*
- Ray
+ RayPlatform: a message-passing development framework
Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,18 +18,27 @@
see <http://www.gnu.org/licenses/>
*/
+#include "MasterModeExecutor.h"
+#include <RayPlatform/core/types.h>
-#include <handlers/MasterModeExecutor.h>
-#include <core/types.h>
#include <stdlib.h> /* for NULL */
-
#ifdef ASSERT
#include <assert.h>
#endif
void MasterModeExecutor::callHandler(MasterMode mode){
- MasterModeHandler object=m_objects[mode];
+
+#ifdef CONFIG_CACHE_OPERATION_CODES
+ MasterModeHandlerReference object=m_cachedOperationHandler;
+/*
+ * We don't want to look up too much.
+ */
+ if(mode!=m_cachedOperationCode)
+ object=m_objects[mode];
+#else
+ MasterModeHandlerReference object=m_objects[mode];
+#endif /* CONFIG_CACHE_OPERATION_CODES */
// don't do it if it is NULL because it does nothing
if(object==NULL)
@@ -37,21 +46,30 @@ void MasterModeExecutor::callHandler(MasterMode mode){
/** otherwise, fetch the method and call it*/
+#ifdef CONFIG_MINI_RANKS
+ object->call();
+#else
object();
+#endif
}
MasterModeExecutor::MasterModeExecutor(){
for(int i=0;i<MAXIMUM_NUMBER_OF_MASTER_HANDLERS;i++){
m_objects[i]=NULL;
}
+
+#ifdef CONFIG_CACHE_OPERATION_CODES
+ m_cachedOperationCode=INVALID_HANDLE;
+ m_cachedOperationHandler=NULL;
+#endif
}
-void MasterModeExecutor::setObjectHandler(MasterMode mode,MasterModeHandler object){
+void MasterModeExecutor::setObjectHandler(MasterMode mode,MasterModeHandlerReference object){
- #ifdef ASSERT
+#ifdef ASSERT
assert(mode>=0);
assert(mode < MAXIMUM_NUMBER_OF_MASTER_HANDLERS);
- #endif
+#endif
m_objects[mode]=object;
}
diff --git a/RayPlatform/handlers/MasterModeExecutor.h b/RayPlatform/RayPlatform/handlers/MasterModeExecutor.h
similarity index 60%
rename from RayPlatform/handlers/MasterModeExecutor.h
rename to RayPlatform/RayPlatform/handlers/MasterModeExecutor.h
index 67d0fab..759f68c 100644
--- a/RayPlatform/handlers/MasterModeExecutor.h
+++ b/RayPlatform/RayPlatform/handlers/MasterModeExecutor.h
@@ -1,8 +1,8 @@
/*
- Ray
+ RayPlatform: a message-passing development framework
Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,25 +21,37 @@
#ifndef _MasterModeExecutor_h
#define _MasterModeExecutor_h
-#include <core/types.h>
-#include <core/master_modes.h>
-#include <handlers/MasterModeHandler.h>
+#include "MasterModeHandler.h"
+#include <RayPlatform/core/types.h>
+#include <RayPlatform/core/master_modes.h>
+
+/**
+ * This class is responsible to handling event
+ * for master modes.
+ * The technology uses aggressive caching.
+ *
+ * \author Sébastien Boisvert
+ */
class MasterModeExecutor{
+#ifdef CONFIG_CACHE_OPERATION_CODES
+ MasterMode m_cachedOperationCode;
+ MasterModeHandlerReference m_cachedOperationHandler;
+#endif /* CONFIG_CACHE_OPERATION_CODES */
+
/** a list of objects to use for calling methods */
- MasterModeHandler m_objects[MAXIMUM_NUMBER_OF_MASTER_HANDLERS];
+ MasterModeHandlerReference m_objects[MAXIMUM_NUMBER_OF_MASTER_HANDLERS];
public:
/** call the handler */
void callHandler(MasterMode mode);
/** set the correct object to call for a given master mode */
- void setObjectHandler(MasterMode mode, MasterModeHandler object);
+ void setObjectHandler(MasterMode mode, MasterModeHandlerReference object);
/** initialise default object and method handlers */
MasterModeExecutor();
-
};
#endif
diff --git a/RayPlatform/RayPlatform/handlers/MasterModeHandler.h b/RayPlatform/RayPlatform/handlers/MasterModeHandler.h
new file mode 100644
index 0000000..4691e63
--- /dev/null
+++ b/RayPlatform/RayPlatform/handlers/MasterModeHandler.h
@@ -0,0 +1,106 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+*/
+
+#ifndef _MasterModeHandler_h
+#define _MasterModeHandler_h
+
+#include <RayPlatform/core/types.h>
+#include <RayPlatform/core/master_modes.h>
+
+#ifdef CONFIG_MINI_RANKS
+
+#define MasterModeHandlerReference MasterModeHandler*
+
+#define __CreateMasterModeAdapter ____CreateMasterModeAdapterImplementation
+#define __DeclareMasterModeAdapter ____CreateMasterModeAdapterDeclaration
+
+/* this is a macro to create the header code for an adapter */
+#define ____CreateMasterModeAdapterDeclaration(corePlugin,handle) \
+class Adapter_ ## handle : public MasterModeHandler{ \
+ corePlugin *m_object; \
+public: \
+ void setObject(corePlugin *object); \
+ void call(); \
+};
+
+/* this is a macro to create the cpp code for an adapter */
+#define ____CreateMasterModeAdapterImplementation(corePlugin,handle)\
+void Adapter_ ## handle ::setObject( corePlugin *object){ \
+ m_object=object; \
+} \
+ \
+void Adapter_ ## handle ::call(){ \
+ m_object->call_ ## handle(); \
+}
+
+/**
+ * base class for handling master modes
+ *
+ * \author Sébastien Boisvert
+ *
+ * with help from Élénie Godzaridis for the design
+ *
+ * \date 2012-08-08 replaced this with function pointers
+ */
+class MasterModeHandler{
+
+public:
+
+ virtual void call() = 0;
+
+ virtual ~MasterModeHandler(){}
+};
+
+#else
+
+/*
+ * Without mini-ranks.
+ */
+
+#define __DeclareMasterModeAdapter(corePlugin, handle)
+
+/* this is a macro to create the cpp code for an adapter */
+#define __CreateMasterModeAdapter( corePlugin,handle )\
+void __GetAdapter( corePlugin, handle ) () { \
+ __GetPlugin( corePlugin ) -> __GetMethod( handle ) () ; \
+}
+
+/**
+ * base class for handling master modes
+ * \author Sébastien Boisvert
+ * with help from Élénie Godzaridis for the design
+ * \date 2012-08-08 replaced this with function pointers
+ */
+typedef void (*MasterModeHandler) () /* */ ;
+
+#define MasterModeHandlerReference MasterModeHandler
+
+#endif
+
+#define __ConfigureMasterModeHandler(pluginName,handlerHandle) \
+ handlerHandle= m_core->allocateMasterModeHandle(m_plugin); \
+ m_core->setMasterModeObjectHandler(m_plugin,handlerHandle, \
+ __GetAdapter(pluginName,handlerHandle)); \
+ m_core->setMasterModeSymbol(m_plugin, handlerHandle, # handlerHandle); \
+ __BindAdapter(pluginName, handlerHandle);
+
+
+
+#endif
diff --git a/RayPlatform/handlers/MessageTagExecutor.cpp b/RayPlatform/RayPlatform/handlers/MessageTagExecutor.cpp
similarity index 64%
rename from RayPlatform/handlers/MessageTagExecutor.cpp
rename to RayPlatform/RayPlatform/handlers/MessageTagExecutor.cpp
index 927cfd5..4dcd441 100644
--- a/RayPlatform/handlers/MessageTagExecutor.cpp
+++ b/RayPlatform/RayPlatform/handlers/MessageTagExecutor.cpp
@@ -1,8 +1,8 @@
/*
- Ray
+ RayPlatform: a message-passing development framework
Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,33 +18,51 @@
see <http://www.gnu.org/licenses/>
*/
-#include <handlers/MessageTagHandler.h>
-#include <handlers/MessageTagExecutor.h>
-#include <communication/mpi_tags.h>
+#include "MessageTagHandler.h"
+#include "MessageTagExecutor.h"
+
+#include <RayPlatform/communication/mpi_tags.h>
+
#ifdef ASSERT
#include <assert.h>
#endif
#include <stdlib.h> /* for NULL */
-
void MessageTagExecutor::callHandler(MessageTag messageTag,Message*message){
- MessageTagHandler object=m_objects[messageTag];
+
+#ifdef CONFIG_CACHE_OPERATION_CODES
+ MessageTagHandlerReference object=m_cachedOperationHandler;
+
+ if(messageTag!=m_cachedOperationCode)
+ object=m_objects[messageTag];
+#else
+ MessageTagHandlerReference object=m_objects[messageTag];
+#endif /* CONFIG_CACHE_OPERATION_CODES */
// it is useless to call base implementations
// because they are empty
if(object==NULL)
return;
+#ifdef CONFIG_MINI_RANKS
+ object->call(message);
+#else
object(message);
+#endif
}
MessageTagExecutor::MessageTagExecutor(){
for(int i=0;i<MAXIMUM_NUMBER_OF_TAG_HANDLERS;i++){
m_objects[i]=NULL;
}
+
+#ifdef CONFIG_CACHE_OPERATION_CODES
+ m_cachedOperationCode=INVALID_HANDLE;
+ m_cachedOperationHandler=NULL;
+#endif
}
-void MessageTagExecutor::setObjectHandler(MessageTag messageTag,MessageTagHandler object){
+void MessageTagExecutor::setObjectHandler(MessageTag messageTag,MessageTagHandlerReference object){
#ifdef ASSERT
assert(messageTag>=0);
diff --git a/RayPlatform/handlers/MessageTagExecutor.h b/RayPlatform/RayPlatform/handlers/MessageTagExecutor.h
similarity index 59%
rename from RayPlatform/handlers/MessageTagExecutor.h
rename to RayPlatform/RayPlatform/handlers/MessageTagExecutor.h
index b226424..4c4ed6f 100644
--- a/RayPlatform/handlers/MessageTagExecutor.h
+++ b/RayPlatform/RayPlatform/handlers/MessageTagExecutor.h
@@ -1,8 +1,8 @@
/*
- Ray
+ RayPlatform: a message-passing development framework
Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,22 +18,31 @@
see <http://www.gnu.org/licenses/>
*/
-
-
#ifndef _MessageTagExecutor_h
#define _MessageTagExecutor_h
-#include <handlers/MessageTagHandler.h>
-#include <communication/Message.h>
-#include <core/types.h>
-#include <communication/mpi_tags.h>
+#include "MessageTagHandler.h"
+#include <RayPlatform/core/types.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/communication/mpi_tags.h>
+/**
+ * This class is responsible to handling event
+ * for master modes.
+ * The technology uses aggressive caching.
+ *
+ * \author Sébastien Boisvert
+ */
class MessageTagExecutor{
+#ifdef CONFIG_CACHE_OPERATION_CODES
+ MessageTag m_cachedOperationCode;
+ MessageTagHandlerReference m_cachedOperationHandler;
+#endif /* CONFIG_CACHE_OPERATION_CODES */
/** table of object handlers */
- MessageTagHandler m_objects[MAXIMUM_NUMBER_OF_TAG_HANDLERS];
+ MessageTagHandlerReference m_objects[MAXIMUM_NUMBER_OF_TAG_HANDLERS];
public:
@@ -41,12 +50,10 @@ public:
void callHandler(MessageTag messageTag,Message*message);
/** set the object to call for a given tag */
- void setObjectHandler(MessageTag messageTag,MessageTagHandler object);
+ void setObjectHandler(MessageTag messageTag,MessageTagHandlerReference object);
/** set default object and method handlers */
MessageTagExecutor();
-
-
};
#endif
diff --git a/RayPlatform/RayPlatform/handlers/MessageTagHandler.h b/RayPlatform/RayPlatform/handlers/MessageTagHandler.h
new file mode 100644
index 0000000..998a51d
--- /dev/null
+++ b/RayPlatform/RayPlatform/handlers/MessageTagHandler.h
@@ -0,0 +1,101 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+*/
+
+#ifndef _MessageTagHandler_h
+#define _MessageTagHandler_h
+
+#include <RayPlatform/core/types.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/communication/mpi_tags.h>
+
+#ifdef CONFIG_MINI_RANKS
+
+#define MessageTagHandlerReference MessageTagHandler*
+
+#define __CreateMessageTagAdapter ____CreateMessageTagAdapterImplementation
+#define __DeclareMessageTagAdapter ____CreateMessageTagAdapterDeclaration
+
+/* this is a macro to create the header code for an adapter */
+#define ____CreateMessageTagAdapterDeclaration(corePlugin,handle) \
+class Adapter_ ## handle : public MessageTagHandler{ \
+ corePlugin *m_object; \
+public: \
+ void setObject(corePlugin *object); \
+ void call(Message*message); \
+};
+
+/* this is a macro to create the cpp code for an adapter */
+#define ____CreateMessageTagAdapterImplementation(corePlugin,handle)\
+void Adapter_ ## handle ::setObject( corePlugin *object){ \
+ m_object=object; \
+} \
+ \
+void Adapter_ ## handle ::call(Message*message){ \
+ m_object->call_ ## handle(message); \
+}
+
+/**
+ * base class for handling message tags
+ * \author Sébastien Boisvert
+ * with help from Élénie Godzaridis for the design
+ */
+class MessageTagHandler{
+public:
+
+ virtual void call(Message*message) = 0;
+
+ virtual ~MessageTagHandler(){}
+};
+
+#else
+
+/*
+ * Without mini-ranks.
+ */
+
+#define __DeclareMessageTagAdapter(plugin, handle)
+
+/* this is a macro to create the cpp code for an adapter */
+#define __CreateMessageTagAdapter( corePlugin, handle ) \
+void __GetAdapter( corePlugin, handle) ( Message*message ) { \
+ __GetPlugin( corePlugin ) -> __GetMethod( handle ) ( message ) ; \
+}
+
+/**
+ * base class for handling message tags
+ * \author Sébastien Boisvert
+ * with help from Élénie Godzaridis for the design
+ * \date 2012-08-08 replaced this with function pointers
+ */
+typedef void (*MessageTagHandler) (Message*message) /* */ ;
+
+#define MessageTagHandlerReference MessageTagHandler
+
+#endif /* CONFIG_MINI_RANKS */
+
+#define __ConfigureMessageTagHandler(pluginName, handlerHandle) \
+ handlerHandle= m_core->allocateMessageTagHandle(m_plugin); \
+ m_core->setMessageTagObjectHandler(m_plugin, handlerHandle, \
+ __GetAdapter(pluginName,handlerHandle)); \
+ m_core->setMessageTagSymbol(m_plugin, handlerHandle, # handlerHandle); \
+ __BindAdapter(pluginName, handlerHandle);
+
+
+#endif
diff --git a/RayPlatform/handlers/SlaveModeExecutor.cpp b/RayPlatform/RayPlatform/handlers/SlaveModeExecutor.cpp
similarity index 66%
rename from RayPlatform/handlers/SlaveModeExecutor.cpp
rename to RayPlatform/RayPlatform/handlers/SlaveModeExecutor.cpp
index d291398..b1ca151 100644
--- a/RayPlatform/handlers/SlaveModeExecutor.cpp
+++ b/RayPlatform/RayPlatform/handlers/SlaveModeExecutor.cpp
@@ -1,8 +1,8 @@
/*
- Ray
+ RayPlatform: a message-passing development framework
Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,7 +18,7 @@
see <http://www.gnu.org/licenses/>
*/
-#include <handlers/SlaveModeExecutor.h>
+#include "SlaveModeExecutor.h"
#ifdef ASSERT
#include <assert.h>
@@ -26,31 +26,46 @@
#include <stdlib.h> /* for NULL */
void SlaveModeExecutor::callHandler(SlaveMode mode){
- SlaveModeHandler object=m_objects[mode];
+
+#ifdef CONFIG_CACHE_OPERATION_CODES
+ SlaveModeHandlerReference object=m_cachedOperationHandler;
+
+ if(mode!=m_cachedOperationCode)
+ object=m_objects[mode];
+#else
+ SlaveModeHandlerReference object=m_objects[mode];
+#endif /* CONFIG_CACHE_OPERATION_CODES */
// don't call it if it is NULL
if(object==NULL)
return;
// call it
+#ifdef CONFIG_MINI_RANKS
+ object->call();
+#else
object();
+#endif
}
SlaveModeExecutor::SlaveModeExecutor(){
for(int i=0;i<MAXIMUM_NUMBER_OF_SLAVE_HANDLERS;i++){
m_objects[i]=NULL;
}
+
+#ifdef CONFIG_CACHE_OPERATION_CODES
+ m_cachedOperationCode=INVALID_HANDLE;
+ m_cachedOperationHandler=NULL;
+#endif
}
-void SlaveModeExecutor::setObjectHandler(SlaveMode mode,SlaveModeHandler object){
+void SlaveModeExecutor::setObjectHandler(SlaveMode mode,SlaveModeHandlerReference object){
- #ifdef ASSERT
+#ifdef ASSERT
assert(mode>=0);
assert(mode<MAXIMUM_NUMBER_OF_SLAVE_HANDLERS);
- #endif
+#endif
m_objects[mode]=object;
}
-
-
diff --git a/RayPlatform/handlers/SlaveModeExecutor.h b/RayPlatform/RayPlatform/handlers/SlaveModeExecutor.h
similarity index 60%
rename from RayPlatform/handlers/SlaveModeExecutor.h
rename to RayPlatform/RayPlatform/handlers/SlaveModeExecutor.h
index 242f10a..7a47e8c 100644
--- a/RayPlatform/handlers/SlaveModeExecutor.h
+++ b/RayPlatform/RayPlatform/handlers/SlaveModeExecutor.h
@@ -1,8 +1,8 @@
/*
- Ray
+ RayPlatform: a message-passing development framework
Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,20 +18,30 @@
see <http://www.gnu.org/licenses/>
*/
-
-
-
#ifndef _SlaveModeExecutor_h
#define _SlaveModeExecutor_h
-#include <handlers/SlaveModeHandler.h>
-#include <core/slave_modes.h>
-#include <core/types.h>
+#include "SlaveModeHandler.h"
+#include <RayPlatform/core/slave_modes.h>
+#include <RayPlatform/core/types.h>
+
+/**
+ * This class is responsible to handling event
+ * for master modes.
+ * The technology uses aggressive caching.
+ *
+ * \author Sébastien Boisvert
+ */
class SlaveModeExecutor{
+#ifdef CONFIG_CACHE_OPERATION_CODES
+ SlaveMode m_cachedOperationCode;
+ SlaveModeHandlerReference m_cachedOperationHandler;
+#endif /* CONFIG_CACHE_OPERATION_CODES */
+
/** table of slave objects */
- SlaveModeHandler m_objects[MAXIMUM_NUMBER_OF_SLAVE_HANDLERS];
+ SlaveModeHandlerReference m_objects[MAXIMUM_NUMBER_OF_SLAVE_HANDLERS];
public:
@@ -42,7 +52,7 @@ public:
SlaveModeExecutor();
/** set the object to call for a slave mode */
- void setObjectHandler(SlaveMode mode,SlaveModeHandler object);
+ void setObjectHandler(SlaveMode mode,SlaveModeHandlerReference object);
};
diff --git a/RayPlatform/RayPlatform/handlers/SlaveModeHandler.h b/RayPlatform/RayPlatform/handlers/SlaveModeHandler.h
new file mode 100644
index 0000000..535c92f
--- /dev/null
+++ b/RayPlatform/RayPlatform/handlers/SlaveModeHandler.h
@@ -0,0 +1,102 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+*/
+
+#ifndef _SlaveModeHandler_h
+#define _SlaveModeHandler_h
+
+#include <RayPlatform/core/slave_modes.h>
+#include <RayPlatform/core/types.h>
+
+#ifdef CONFIG_MINI_RANKS
+
+#define SlaveModeHandlerReference SlaveModeHandler*
+
+#define __CreateSlaveModeAdapter ____CreateSlaveModeAdapterImplementation
+#define __DeclareSlaveModeAdapter ____CreateSlaveModeAdapterDeclaration
+
+/* this is a macro to create the header code for an adapter */
+#define ____CreateSlaveModeAdapterDeclaration(corePlugin,handle) \
+class Adapter_ ## handle : public SlaveModeHandler{ \
+ corePlugin *m_object; \
+public: \
+ void setObject(corePlugin *object); \
+ void call(); \
+};
+
+/* this is a macro to create the cpp code for an adapter */
+#define ____CreateSlaveModeAdapterImplementation(corePlugin,handle)\
+void Adapter_ ## handle ::setObject( corePlugin *object){ \
+ m_object=object; \
+} \
+ \
+void Adapter_ ## handle ::call(){ \
+ m_object->call_ ## handle(); \
+}
+
+/**
+ * base class for handling slave modes
+ * \author Sébastien Boisvert
+ * with help from Élénie Godzaridis for the design
+ */
+class SlaveModeHandler{
+
+public:
+
+ virtual void call() = 0;
+
+ virtual ~SlaveModeHandler(){}
+};
+
+#else
+
+/*
+ * Without mini-ranks.
+ */
+
+#define __DeclareSlaveModeAdapter(corePlugin, handle)
+
+/* this is a macro to create the cpp code for an adapter */
+#define __CreateSlaveModeAdapter( corePlugin,handle ) \
+void __GetAdapter( corePlugin, handle ) () { \
+ __GetPlugin( corePlugin ) -> __GetMethod( handle ) () ; \
+}
+
+/**
+ * base class for handling slave modes
+ * \author Sébastien Boisvert
+ * with help from Élénie Godzaridis for the design
+ * \date 2012-08-08 replaced this with function pointers
+ */
+typedef void (*SlaveModeHandler) () /* */ ;
+
+#define SlaveModeHandlerReference SlaveModeHandler
+
+#endif
+
+#define __ConfigureSlaveModeHandler(pluginName, handlerHandle) \
+ handlerHandle= m_core->allocateSlaveModeHandle(m_plugin); \
+ m_core->setSlaveModeObjectHandler(m_plugin, handlerHandle, \
+ __GetAdapter(pluginName,handlerHandle)); \
+ m_core->setSlaveModeSymbol(m_plugin, handlerHandle, # handlerHandle); \
+ __BindAdapter(pluginName, handlerHandle);
+
+
+
+#endif
diff --git a/RayPlatform/memory/ChunkAllocatorWithDefragmentation.cpp b/RayPlatform/RayPlatform/memory/ChunkAllocatorWithDefragmentation.cpp
similarity index 97%
rename from RayPlatform/memory/ChunkAllocatorWithDefragmentation.cpp
rename to RayPlatform/RayPlatform/memory/ChunkAllocatorWithDefragmentation.cpp
index 2fde554..94a97c5 100644
--- a/RayPlatform/memory/ChunkAllocatorWithDefragmentation.cpp
+++ b/RayPlatform/RayPlatform/memory/ChunkAllocatorWithDefragmentation.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,13 +18,12 @@
see <http://www.gnu.org/licenses/>
*/
-#include <memory/ChunkAllocatorWithDefragmentation.h>
-#include <memory/allocator.h>
+#include "ChunkAllocatorWithDefragmentation.h"
+#include "allocator.h"
#ifdef ASSERT
#include <assert.h>
#endif
-
#include <iostream>
using namespace std;
diff --git a/RayPlatform/memory/ChunkAllocatorWithDefragmentation.h b/RayPlatform/RayPlatform/memory/ChunkAllocatorWithDefragmentation.h
similarity index 92%
rename from RayPlatform/memory/ChunkAllocatorWithDefragmentation.h
rename to RayPlatform/RayPlatform/memory/ChunkAllocatorWithDefragmentation.h
index 6aa337e..c2f3170 100644
--- a/RayPlatform/memory/ChunkAllocatorWithDefragmentation.h
+++ b/RayPlatform/RayPlatform/memory/ChunkAllocatorWithDefragmentation.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,8 +21,9 @@
#ifndef _ChunkAllocatorWithDefragmentation_H
#define _ChunkAllocatorWithDefragmentation_H
-#include <memory/DefragmentationGroup.h>
-#include <memory/DefragmentationLane.h>
+#include "DefragmentationGroup.h"
+#include "DefragmentationLane.h"
+
#include <stdlib.h>
/**
diff --git a/RayPlatform/memory/DefragmentationGroup.cpp b/RayPlatform/RayPlatform/memory/DefragmentationGroup.cpp
similarity index 98%
rename from RayPlatform/memory/DefragmentationGroup.cpp
rename to RayPlatform/RayPlatform/memory/DefragmentationGroup.cpp
index 996870b..9bb6fe2 100644
--- a/RayPlatform/memory/DefragmentationGroup.cpp
+++ b/RayPlatform/RayPlatform/memory/DefragmentationGroup.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -27,8 +27,9 @@
/* #define LOW_LEVEL_ASSERT */
-#include <memory/DefragmentationGroup.h>
-#include <memory/allocator.h>
+#include "DefragmentationGroup.h"
+#include "allocator.h"
+
#include <stdlib.h>
#include <string.h>
#include <iostream>
diff --git a/RayPlatform/memory/DefragmentationGroup.h b/RayPlatform/RayPlatform/memory/DefragmentationGroup.h
similarity index 96%
rename from RayPlatform/memory/DefragmentationGroup.h
rename to RayPlatform/RayPlatform/memory/DefragmentationGroup.h
index f557e8b..fb88446 100644
--- a/RayPlatform/memory/DefragmentationGroup.h
+++ b/RayPlatform/RayPlatform/memory/DefragmentationGroup.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,6 +21,8 @@
#ifndef _DefragmentationGroup_H
#define _DefragmentationGroup_H
+#include <stdint.h>
+
/*
* How many elements per group ?
* this is the number of values of uint16_t */
@@ -32,7 +34,6 @@
*/
#define FAST_POINTERS 256
-#include <stdint.h>
/**
* A SmallSmartPointer is a smart pointer than only a DefragmentationGroup
diff --git a/RayPlatform/memory/DefragmentationLane.cpp b/RayPlatform/RayPlatform/memory/DefragmentationLane.cpp
similarity index 95%
rename from RayPlatform/memory/DefragmentationLane.cpp
rename to RayPlatform/RayPlatform/memory/DefragmentationLane.cpp
index 89578a3..7384201 100644
--- a/RayPlatform/memory/DefragmentationLane.cpp
+++ b/RayPlatform/RayPlatform/memory/DefragmentationLane.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,7 +18,8 @@
see <http://www.gnu.org/licenses/>
*/
-#include <memory/DefragmentationLane.h>
+#include "DefragmentationLane.h"
+
#include <stdlib.h>
#include <assert.h>
#include <iostream>
diff --git a/RayPlatform/memory/DefragmentationLane.h b/RayPlatform/RayPlatform/memory/DefragmentationLane.h
similarity index 93%
rename from RayPlatform/memory/DefragmentationLane.h
rename to RayPlatform/RayPlatform/memory/DefragmentationLane.h
index dcf3d1c..cbcc511 100644
--- a/RayPlatform/memory/DefragmentationLane.h
+++ b/RayPlatform/RayPlatform/memory/DefragmentationLane.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,7 +21,7 @@
#ifndef _DefragmentationLane_H
#define _DefragmentationLane_H
-#include <memory/DefragmentationGroup.h>
+#include "DefragmentationGroup.h"
/** value for a invalid DefragmentationGroup */
#define INVALID_GROUP 123467
diff --git a/RayPlatform/RayPlatform/memory/DirtyBuffer.cpp b/RayPlatform/RayPlatform/memory/DirtyBuffer.cpp
new file mode 100644
index 0000000..c7dc1d4
--- /dev/null
+++ b/RayPlatform/RayPlatform/memory/DirtyBuffer.cpp
@@ -0,0 +1,72 @@
+/*
+ * RayPlatform: a message-passing development framework
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://github.com/sebhtml/RayPlatform
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You have received a copy of the GNU Lesser General Public License
+ * along with this program (lgpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ *
+ * HeaderVersion_ 2013-08-19
+ * /sebseb
+ */
+
+
+
+#include "DirtyBuffer.h"
+
+void DirtyBuffer::setBuffer(void * buffer) {
+
+ m_buffer = buffer;
+}
+
+void DirtyBuffer::setSource(Rank source) {
+
+ m_source = source;
+}
+
+void DirtyBuffer::setDestination(Rank destination) {
+
+ m_destination = destination;
+}
+
+void DirtyBuffer::setTag(MessageTag tag) {
+
+ m_messageTag = tag;
+}
+
+Rank DirtyBuffer::getSource() {
+
+ return m_source;
+}
+
+Rank DirtyBuffer::getDestination() {
+
+ return m_destination;
+}
+
+MessageTag DirtyBuffer::getTag() {
+
+ return m_messageTag;
+}
+
+MPI_Request * DirtyBuffer::getRequest() {
+
+ return & m_messageRequest;
+}
+
+void * DirtyBuffer::getBuffer() {
+
+ return m_buffer;
+}
+
diff --git a/RayPlatform/RayPlatform/memory/DirtyBuffer.h b/RayPlatform/RayPlatform/memory/DirtyBuffer.h
new file mode 100644
index 0000000..8bdc15e
--- /dev/null
+++ b/RayPlatform/RayPlatform/memory/DirtyBuffer.h
@@ -0,0 +1,56 @@
+/*
+ * RayPlatform: a message-passing development framework
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://github.com/sebhtml/RayPlatform
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You have received a copy of the GNU Lesser General Public License
+ * along with this program (lgpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ *
+ * HeaderVersion_ 2013-08-19
+ * /sebseb
+ */
+
+#ifndef DirtyBufferHeader
+#define DirtyBufferHeader
+
+#include <mpi.h>
+
+#include <RayPlatform/core/types.h>
+
+/**
+ * A data model for storing dirty buffers
+ */
+class DirtyBuffer{
+
+ void * m_buffer;
+ MPI_Request m_messageRequest;
+ Rank m_destination;
+ MessageTag m_messageTag;
+ Rank m_source;
+
+public:
+
+ void setBuffer(void * buffer);
+ void setSource(Rank source);
+ void setDestination(Rank destination);
+ void setTag(MessageTag tag);
+
+ Rank getSource();
+ Rank getDestination();
+ MessageTag getTag();
+ MPI_Request * getRequest();
+ void * getBuffer();
+};
+
+#endif // DirtyBufferHeader
diff --git a/RayPlatform/memory/MyAllocator.cpp b/RayPlatform/RayPlatform/memory/MyAllocator.cpp
similarity index 89%
rename from RayPlatform/memory/MyAllocator.cpp
rename to RayPlatform/RayPlatform/memory/MyAllocator.cpp
index 7643ef8..49334db 100644
--- a/RayPlatform/memory/MyAllocator.cpp
+++ b/RayPlatform/RayPlatform/memory/MyAllocator.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -14,15 +14,17 @@
GNU Lesser General Public License for more details.
You have received a copy of the GNU Lesser General Public License
- along with this program (lgpl-3.0.txt).
+ along with this program (lgpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include <memory/allocator.h>
-#include <memory/MyAllocator.h>
+#include "allocator.h"
+#include "MyAllocator.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
+
#include <stdlib.h>
-#include <core/OperatingSystem.h>
#include <iostream>
#include <stdio.h>
#include <assert.h>
@@ -41,7 +43,7 @@ void MyAllocator::reset(){
/** the constructor does not allocate a chunk */
void MyAllocator::constructor(int chunkSize,const char*type,bool show){
m_show=show;
- m_CHUNK_SIZE=chunkSize;
+ m_CHUNK_SIZE=chunkSize;
strcpy(m_type,type);
}
@@ -56,7 +58,7 @@ void MyAllocator::addChunk(){
m_currentPosition=0;
}
-/** this is a chunk allocator
+/** this is a chunk allocator
* when a chunk is consumed totally,
* a new one is created
*/
@@ -140,7 +142,8 @@ void*MyAllocator::allocate(int s){
return r;
}
-MyAllocator::~MyAllocator(){}
+MyAllocator::~MyAllocator(){
+}
void MyAllocator::clear(){
m_store.reset();
@@ -169,8 +172,8 @@ int MyAllocator::getNumberOfChunks(){
return m_chunks.size();
}
-void MyAllocator::free(void*a,int b){
- m_store.addAddressToReuse(a,b);
+void MyAllocator::free(void * address, int bytes){
+ m_store.addAddressToReuse(address, bytes);
}
MyAllocator::MyAllocator(){
diff --git a/RayPlatform/memory/MyAllocator.h b/RayPlatform/RayPlatform/memory/MyAllocator.h
similarity index 84%
rename from RayPlatform/memory/MyAllocator.h
rename to RayPlatform/RayPlatform/memory/MyAllocator.h
index b13bbbc..82f2582 100644
--- a/RayPlatform/memory/MyAllocator.h
+++ b/RayPlatform/RayPlatform/memory/MyAllocator.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,8 +22,9 @@
#ifndef _MyAllocator
#define _MyAllocator
+#include "ReusableMemoryStore.h"
+
#include <vector>
-#include <memory/ReusableMemoryStore.h>
using namespace std;
/**
@@ -59,12 +60,12 @@ public:
/**
* assign a size to the allocator.
*/
- void constructor(int a,const char*type,bool show);
+ void constructor(int chunkSize, const char*type, bool show);
/**
* allocate memory
*/
- void*allocate(int s);
+ void*allocate(int numberOfBytes);
~MyAllocator();
int getChunkSize();
@@ -79,7 +80,7 @@ public:
void reset();
/** free memory */
- void free(void*a,int b);
+ void free(void * address, int numberOfBytes);
};
int roundNumber(int number,int alignment);
diff --git a/RayPlatform/memory/ReusableMemoryStore.cpp b/RayPlatform/RayPlatform/memory/ReusableMemoryStore.cpp
similarity index 93%
rename from RayPlatform/memory/ReusableMemoryStore.cpp
rename to RayPlatform/RayPlatform/memory/ReusableMemoryStore.cpp
index eba1dcf..9ab0d0e 100644
--- a/RayPlatform/memory/ReusableMemoryStore.cpp
+++ b/RayPlatform/RayPlatform/memory/ReusableMemoryStore.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,7 +19,8 @@
*/
-#include <memory/ReusableMemoryStore.h>
+#include "ReusableMemoryStore.h"
+
#include <iostream>
#include <assert.h>
using namespace std;
diff --git a/RayPlatform/memory/ReusableMemoryStore.h b/RayPlatform/RayPlatform/memory/ReusableMemoryStore.h
similarity index 89%
rename from RayPlatform/memory/ReusableMemoryStore.h
rename to RayPlatform/RayPlatform/memory/ReusableMemoryStore.h
index c49304c..eb2d73c 100644
--- a/RayPlatform/memory/ReusableMemoryStore.h
+++ b/RayPlatform/RayPlatform/memory/ReusableMemoryStore.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
diff --git a/RayPlatform/RayPlatform/memory/RingAllocator.cpp b/RayPlatform/RayPlatform/memory/RingAllocator.cpp
new file mode 100644
index 0000000..3c0702c
--- /dev/null
+++ b/RayPlatform/RayPlatform/memory/RingAllocator.cpp
@@ -0,0 +1,536 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#include "RingAllocator.h"
+#include "allocator.h"
+
+#include <RayPlatform/communication/mpi_tags.h>
+
+#include <string.h>
+#include <assert.h>
+#include <iostream>
+using namespace std;
+
+//#define CONFIG_RING_VERBOSE
+
+#define BUFFER_STATE_AVAILABLE 0x0
+#define BUFFER_STATE_DIRTY 0x1
+
+#define __NOT_SET -1
+
+
+void RingAllocator::constructor(int chunks,int size,const char*type,bool show){
+
+ #ifdef CONFIG_RING_VERBOSE
+ cout<<"[RingAllocator::constructor] "<<type<<" this "<<hex<<this<<dec<<endl;
+ #endif
+
+ resetCount();
+
+ /* m_max should never be 0 */
+ #ifdef ASSERT
+ assert(size>0);
+ assert(chunks>0);
+ #endif /* ASSERT */
+
+ m_chunks=chunks;// number of buffers
+
+ m_max=size;// maximum buffer size in bytes
+
+ #ifdef ASSERT
+ assert(m_chunks==chunks);
+ assert(m_max==size);
+ #endif /* ASSERT */
+
+ strcpy(m_type,type);
+
+ m_numberOfBytes=m_chunks*m_max;
+
+ #ifdef ASSERT
+ assert(m_numberOfBytes>0);
+ #endif /* ASSERT */
+
+ #ifdef ASSERT
+ assert(m_memory==NULL);
+ #endif
+
+ m_memory=(uint8_t*)__Malloc(sizeof(uint8_t)*m_numberOfBytes,m_type,show);
+
+ #ifdef ASSERT
+ assert(m_memory!=NULL);
+ #endif
+
+ m_bufferStates=(uint8_t*)__Malloc(m_chunks*sizeof(uint8_t),m_type,show);
+
+ #ifdef CONFIG_RING_VERBOSE
+ cout<<"[RingAllocator::constructor] memory= "<<(void*)m_memory<<endl;
+ #endif /* CONFIG_RING_VERBOSE */
+
+ for(int i=0;i<m_chunks;i++)
+ m_bufferStates[i]= BUFFER_STATE_AVAILABLE;
+
+ m_current=0;// the current head for allocation operations
+
+ m_show=show;
+
+ // in the beginning, all buffers are available
+ m_availableBuffers=m_chunks;
+
+ #if 0
+ cout<<"[RingAllocator] m_chunks: "<<m_chunks<<" m_max: "<<m_max<<endl;
+ #endif
+
+ #ifdef ASSERT
+ assert(size>0);
+ assert(chunks>0);
+
+ if(m_chunks==0)
+ cout<<"Error: chunks: "<<chunks<<" m_chunks: "<<m_chunks<<endl;
+
+ assert(m_chunks>0);
+ assert(m_max>0);
+ #endif /* ASSERT */
+
+ m_linearSweeps=0;
+
+/**
+ * internally, there are N buffers for MPI_Isend. However,
+ * these slots become dirty when they are used and become
+ * clean again when MPI_Test says so.
+ * But we don't want to do too much sweep operations. Instead,
+ * we want amortized operations.
+ */
+
+ m_minimumNumberOfDirtyBuffersForSweep=__NOT_SET;
+ m_minimumNumberOfDirtyBuffersForWarning=__NOT_SET;
+
+ m_numberOfDirtyBuffers=0;
+
+ m_maximumDirtyBuffers=m_numberOfDirtyBuffers;
+
+ m_dirtyBuffers=NULL;
+
+}
+
+RingAllocator::RingAllocator(){
+
+ #if 0
+ cout<<"[RingAllocator::RingAllocator] "<<hex<<this<<dec<<endl;
+ #endif
+
+ m_memory=NULL;
+}
+
+/*
+ * allocate a chunk of m_max bytes in constant time
+ */
+void*RingAllocator::allocate(int a){
+
+ m_count++;
+
+ #ifdef ASSERT
+ assert(m_chunks>0);
+
+ if(m_memory==NULL)
+ cout<<"Error: you must call constructor() before calling allocate(), type="<<m_type<<" this "<<hex<<this<<dec<<endl;
+
+ assert(m_memory!=NULL);
+
+ /*
+ if(a>m_max){
+ cout<<"Request "<<a<<" but maximum is "<<m_max<<endl;
+ }
+ assert(a<=m_max);
+ */
+ #endif
+
+ int origin=m_current;
+
+ // first half of the circle
+ // from origin to N-1
+ while(m_bufferStates[m_current] == BUFFER_STATE_DIRTY
+ && m_current < m_chunks){
+
+ m_current++;
+
+ }
+
+ // start from 0 if we completed.
+ // set i to 0
+ if(m_current==m_chunks){
+ m_current=0;
+ }
+
+ // from 0 to origin-1
+ while(m_bufferStates[m_current] == BUFFER_STATE_DIRTY
+ && m_current < origin){
+
+ m_current++;
+ }
+
+ // if all buffers are dirty, we throw a runtime error
+ #ifdef ASSERT
+ if(m_current==origin && m_bufferStates[m_current]==BUFFER_STATE_DIRTY){
+ cout<<"Error: all buffers are dirty !, chunks: "<<m_chunks<<endl;
+ assert(m_current!=origin);
+ }
+ #endif
+
+ void*address=(void*)(m_memory+m_current*m_max);
+
+
+ m_current++;
+
+ // depending on the architecture
+ // branching (if) can be faster than integer division/modulo
+
+ if(m_current==m_chunks){
+ m_current=0;
+ }
+
+ #ifdef ASSERT
+ assert(address!=NULL);
+ #endif
+
+ return address;
+}
+
+void RingAllocator::salvageBuffer(void*buffer){
+ int bufferNumber=getBufferHandle(buffer);
+
+ m_bufferStates[bufferNumber]= BUFFER_STATE_AVAILABLE;
+
+ #ifdef CONFIG_RING_VERBOSE
+ cout<<"[RingAllocator::salvageBuffer] "<<bufferNumber<<" -> BUFFER_STATE_AVAILABLE"<<endl;
+ #endif
+
+ m_availableBuffers++;
+
+ #ifdef ASSERT
+ assert(m_availableBuffers>=1);
+ #endif
+
+}
+
+void RingAllocator::markBufferAsDirty(void*buffer){
+ int bufferNumber=getBufferHandle(buffer);
+
+ m_bufferStates[bufferNumber]= BUFFER_STATE_DIRTY;
+
+ #ifdef CONFIG_RING_VERBOSE
+ cout<<"[RingAllocator::markBufferAsDirty] "<<bufferNumber<<" -> BUFFER_STATE_DIRTY"<<endl;
+ #endif
+
+ #ifdef ASSERT
+ assert(m_availableBuffers>=1);
+ #endif
+
+ m_availableBuffers--;
+
+ #ifdef ASSERT
+ assert(m_availableBuffers>=0);
+ #endif
+}
+
+int RingAllocator::getBufferHandle(void*buffer){
+
+ void*origin=m_memory;
+
+ uint64_t originValue=(uint64_t)origin;
+ uint64_t bufferValue=(uint64_t)buffer;
+
+ uint64_t difference=bufferValue-originValue;
+
+ int index=difference/(m_max*sizeof(uint8_t));
+
+ return index;
+}
+
+int RingAllocator::getSize(){
+ return m_max;
+}
+
+void RingAllocator::clear(){
+
+ #ifdef CONFIG_RING_VERBOSE
+ cout<<"[RingAllocator::clear] "<<m_type<<" memory is "<<hex<<(void*)m_memory<<dec<<endl;
+ #endif
+
+ if(m_memory == NULL)
+ return;
+
+ __Free(m_memory,m_type,m_show);
+ m_memory=NULL;
+}
+
+void RingAllocator::resetCount(){
+ m_count=0;
+}
+
+int RingAllocator::getCount(){
+ return m_count;
+}
+
+int RingAllocator::getNumberOfBuffers(){
+ return m_chunks;
+}
+
+void RingAllocator::initializeDirtyBuffers(){
+
+ m_dirtyBufferSlots=getNumberOfBuffers();
+
+ m_dirtyBuffers=(DirtyBuffer*)__Malloc(m_dirtyBufferSlots*sizeof(DirtyBuffer),
+ "m_dirtyBuffers",false);
+
+ for(int i=0;i<m_dirtyBufferSlots;i++){
+ m_dirtyBuffers[i].setBuffer(NULL);
+ }
+
+ // configure the real-time sweeper.
+
+ m_minimumNumberOfDirtyBuffersForSweep=m_dirtyBufferSlots/4;
+ m_minimumNumberOfDirtyBuffersForWarning=m_dirtyBufferSlots/2;
+
+}
+
+/*
+ * Take the handle of a buffer and check
+ * if the buffer is registered already.
+ */
+bool RingAllocator::isRegistered(int handle){
+
+ return m_dirtyBuffers[handle].getBuffer() != NULL;
+}
+
+DirtyBuffer*RingAllocator::getDirtyBuffers(){
+ return m_dirtyBuffers;
+}
+
+/**
+ * TODO Instead of a true/false state, increase and decrease requests
+ * using a particular buffer. Otherwise, there may be a problem when
+ * a buffer is re-used several times for many requests.
+ */
+void RingAllocator::checkDirtyBuffer(int index){
+
+ #ifdef ASSERT
+ assert(m_numberOfDirtyBuffers>0);
+ #endif
+
+ if(m_dirtyBuffers[index].getBuffer() == NULL)// this entry is empty...
+ return;
+
+ // check the buffer and free it if it is finished.
+ MPI_Status status;
+ MPI_Request*request = (m_dirtyBuffers[index].getRequest());
+
+ int flag=0;
+
+ MPI_Test(request,&flag,&status);
+
+ if(!flag)// this slot is not ready
+ return;
+
+ #ifdef ASSERT
+ assert( flag );
+ #endif /* ASSERT */
+
+ void*buffer=m_dirtyBuffers[index].getBuffer();
+ salvageBuffer(buffer);
+ m_numberOfDirtyBuffers--;
+
+ #ifdef COMMUNICATION_IS_VERBOSE
+ cout<<"From checkDirtyBuffer flag= "<<flag<<endl;
+ #endif /* COMMUNICATION_IS_VERBOSE */
+
+ #ifdef ASSERT
+ assert(*request == MPI_REQUEST_NULL);
+ #endif /* ASSERT */
+
+ m_dirtyBuffers[index].setBuffer(NULL);
+}
+
+void RingAllocator::cleanDirtyBuffers(){
+
+/**
+ * don't do any linear sweep if we still have plenty of free cells
+ */
+ if(m_numberOfDirtyBuffers<m_minimumNumberOfDirtyBuffersForSweep)
+ return;
+
+ #ifdef ASSERT
+ assert(m_numberOfDirtyBuffers>0);
+ #endif
+
+ m_linearSweeps++;
+
+ // update the dirty buffer list.
+ for(int i=0;i<m_dirtyBufferSlots;i++){
+
+/*
+ * All buffer are clean now, it is useless to look
+ * for more dirtiness.
+ */
+ if(m_numberOfDirtyBuffers==0)
+ return;
+
+ checkDirtyBuffer(i);
+ }
+
+ if(m_numberOfDirtyBuffers>=m_minimumNumberOfDirtyBuffersForWarning){
+ cout<<"[MessagesHandler] Warning: dirty buffers are still dirty after linear sweep."<<endl;
+ printDirtyBuffers();
+ }
+}
+
+#define CONFIG_DIRTY_MESSAGE_SUPPORT
+
+void RingAllocator::printDirtyBuffers(){
+
+#ifdef CONFIG_DIRTY_MESSAGE_SUPPORT
+
+ cout<<"[MessagesHandler] Dirty buffers: "<<m_numberOfDirtyBuffers<<"/";
+ cout<<m_dirtyBufferSlots<<endl;
+
+ int rank = -1;
+
+ for(int i=0;i<m_dirtyBufferSlots;i++){
+ cout<<"DirtyBuffer # "<<i<<" State: ";
+ if(m_dirtyBuffers[i].getBuffer() == NULL){
+ cout<<"Available"<<endl;
+ }else{
+ cout<<"Dirty"<<endl;
+
+ MessageTag tag=m_dirtyBuffers[i].getTag();
+ Rank destination=m_dirtyBuffers[i].getDestination();
+ Rank routingSource=rank;
+ Rank routingDestination=destination;
+
+ bool isRoutingTagValue=false;
+
+/* TODO we don't have easy access to this information */
+/* in early version of RayPlatform, routing information
+ * was stored inside the tag. Now, it is stored in the
+ * buffer.
+ */
+#if 0
+ RoutingTag routingTag=tag;
+
+ if(isRoutingTag(tag)){
+ tag=getMessageTagFromRoutingTag(routingTag);
+ routingSource=getSourceFromRoutingTag(routingTag);
+ routingDestination=getDestinationFromRoutingTag(routingTag);
+
+ isRoutingTagValue=true;
+ }
+#endif
+
+ uint8_t index=tag;
+ cout<<" MessageTag: "<< MESSAGE_TAGS[index]<<" ("<<(int)index<<") ";
+ if(index<tag){
+ cout<<"[this is a routing tag]"<<endl;
+ }
+ cout<<" Source: "<<rank<<endl;
+ cout<<" Destination: "<<destination<<endl;
+
+ if(isRoutingTagValue){
+ cout<<" RoutingSource: "<<routingSource<<endl;
+ cout<<" RoutingDestination: "<<routingDestination<<endl;
+ }
+ }
+ }
+
+#endif
+}
+
+void RingAllocator::setRegisteredBufferAttributes(void * buffer,
+ Rank source,
+ Rank destination, MessageTag tag) {
+
+ int handle=this->getBufferHandle(buffer);
+
+ DirtyBuffer & dirtyBuffer = m_dirtyBuffers[handle];
+
+ dirtyBuffer.setSource(source);
+ dirtyBuffer.setDestination(destination);
+ dirtyBuffer.setTag(tag);
+
+ m_rank = source;
+}
+
+MPI_Request * RingAllocator::registerBuffer(void*buffer){
+
+ int handle=this->getBufferHandle(buffer);
+ bool mustRegister=false;
+
+ MPI_Request*request=NULL;
+
+ // this buffer is not registered.
+ if(handle >=0 && m_dirtyBuffers[handle].getBuffer() == NULL)
+ mustRegister=true;
+
+ #ifdef ASSERT
+ assert(m_dirtyBuffers[handle].getBuffer() == NULL);
+ #endif
+
+ /* register the buffer for processing */
+ if(mustRegister){
+ #ifdef ASSERT
+ assert(m_dirtyBuffers[handle].getBuffer() == NULL);
+ #endif
+
+ m_dirtyBuffers[handle].setBuffer(buffer);
+
+ #if 0 // the attributes for dirty buffers are
+ // configured elsewhere by the caller
+ m_dirtyBuffers[handle].m_destination=destination;
+ m_dirtyBuffers[handle].m_messageTag=tag;
+ #endif
+
+ #ifdef ASSERT
+ assert(m_dirtyBuffers[handle].getBuffer() != NULL);
+ #endif
+
+ request = (m_dirtyBuffers[handle].getRequest());
+
+ /* this is O(1) */
+ this->markBufferAsDirty(buffer);
+
+ m_numberOfDirtyBuffers++;
+
+ // update the maximum number of dirty buffers
+ // observed since the beginning.
+ if(m_numberOfDirtyBuffers > m_maximumDirtyBuffers)
+ m_maximumDirtyBuffers=m_numberOfDirtyBuffers;
+ }
+
+ return request;
+}
+
+void RingAllocator::printStatus(){
+
+ #if 0
+ cout<<"Rank "<<m_rank<<": the maximum number of dirty buffers was "<<m_maximumDirtyBuffers<<endl;
+ cout<<"Rank "<<m_rank<<": "<<m_linearSweeps<<" linear sweep operations (threshold: ";
+ cout<<m_minimumNumberOfDirtyBuffersForSweep<<" dirty buffers)"<<endl;
+ #endif
+
+}
diff --git a/RayPlatform/memory/RingAllocator.h b/RayPlatform/RayPlatform/memory/RingAllocator.h
similarity index 67%
rename from RayPlatform/memory/RingAllocator.h
rename to RayPlatform/RayPlatform/memory/RingAllocator.h
index 9dee86c..263c7c5 100644
--- a/RayPlatform/memory/RingAllocator.h
+++ b/RayPlatform/RayPlatform/memory/RingAllocator.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,11 +22,16 @@
#ifndef _RingAllocator
#define _RingAllocator
-#include<set>
-#include<stdint.h>
-#include <core/types.h>
+#include "DirtyBuffer.h"
+
+#include <RayPlatform/core/types.h>
+
+#include <set>
+#include <stdint.h>
+#include <mpi.h>
using namespace std;
+
/**
* This class is a ring buffer. No !, it is an allocator. Thus, referred to as a ring allocator.
*
@@ -35,6 +40,25 @@ using namespace std;
* \author Sébastien Boisvert
*/
class RingAllocator{
+
+ int m_minimumNumberOfDirtyBuffersForSweep;
+ int m_minimumNumberOfDirtyBuffersForWarning;
+ uint64_t m_linearSweeps;
+/** prints dirty buffers **/
+ void printDirtyBuffers();
+
+ Rank m_rank;
+
+ // this contain information about dirty buffers.
+ // the state are stored in m_bufferStates
+
+ DirtyBuffer * m_dirtyBuffers;
+
+ int m_numberOfDirtyBuffers;
+ int m_maximumDirtyBuffers;
+ int m_dirtyBufferSlots;
+
+
/** the number of call to allocate() since the last hard reset */
int m_count;
@@ -62,8 +86,24 @@ class RingAllocator{
uint8_t*m_bufferStates;
+/**
+ * marks a buffer as used
+ */
+ void markBufferAsDirty(void*buffer);
+
+/**
+ * marks a buffer as available
+ */
+
+ void salvageBuffer(void*buffer);
+
public:
+
+ void setRegisteredBufferAttributes(void * buffer,
+ Rank source,
+ Rank destination, MessageTag tag);
+
RingAllocator();
void constructor(int chunks,int size,const char*type,bool show);
@@ -78,17 +118,6 @@ public:
void clear();
void resetCount();
int getCount();
-
-/**
- * marks a buffer as available
- */
- void salvageBuffer(void*buffer);
-
-/**
- * marks a buffer as used
- */
- void markBufferAsDirty(void*buffer);
-
/**
* returns the number of buffers in the ring
*/
@@ -98,6 +127,16 @@ public:
* get the handle for a buffer
*/
int getBufferHandle(void*buffer);
+
+ void checkDirtyBuffer(int i);
+ void cleanDirtyBuffers();
+ void initializeDirtyBuffers();
+ DirtyBuffer*getDirtyBuffers();
+ MPI_Request*registerBuffer(void*buffer);
+
+ void printStatus();
+
+ bool isRegistered(int handle);
};
diff --git a/RayPlatform/memory/allocator.cpp b/RayPlatform/RayPlatform/memory/allocator.cpp
similarity index 87%
rename from RayPlatform/memory/allocator.cpp
rename to RayPlatform/RayPlatform/memory/allocator.cpp
index 93f4dff..46ea333 100644
--- a/RayPlatform/memory/allocator.cpp
+++ b/RayPlatform/RayPlatform/memory/allocator.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,13 +19,14 @@
*/
-#include <memory/allocator.h>
+#include "allocator.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
+
#include <iostream>
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
-#include <core/OperatingSystem.h>
-//#include <core/constants.h>
using namespace std;
/**
diff --git a/RayPlatform/memory/allocator.h b/RayPlatform/RayPlatform/memory/allocator.h
similarity index 84%
rename from RayPlatform/memory/allocator.h
rename to RayPlatform/RayPlatform/memory/allocator.h
index e421c00..befaec6 100644
--- a/RayPlatform/memory/allocator.h
+++ b/RayPlatform/RayPlatform/memory/allocator.h
@@ -1,8 +1,8 @@
/*
-Ray
-Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+RayPlatform: a message-passing development framework
+Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
-http://DeNovoAssembler.SourceForge.Net/
+http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
diff --git a/RayPlatform/plugins/CorePlugin.cpp b/RayPlatform/RayPlatform/plugins/CorePlugin.cpp
similarity index 82%
rename from RayPlatform/plugins/CorePlugin.cpp
rename to RayPlatform/RayPlatform/plugins/CorePlugin.cpp
index 45d07a9..94dfff7 100644
--- a/RayPlatform/plugins/CorePlugin.cpp
+++ b/RayPlatform/RayPlatform/plugins/CorePlugin.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,7 +19,7 @@
*/
-#include <plugins/CorePlugin.h>
+#include "CorePlugin.h"
// empty implementation
void CorePlugin::registerPlugin(ComputeCore*computeCore){
diff --git a/RayPlatform/plugins/CorePlugin.h b/RayPlatform/RayPlatform/plugins/CorePlugin.h
similarity index 60%
rename from RayPlatform/plugins/CorePlugin.h
rename to RayPlatform/RayPlatform/plugins/CorePlugin.h
index ee4d324..6e5b003 100644
--- a/RayPlatform/plugins/CorePlugin.h
+++ b/RayPlatform/RayPlatform/plugins/CorePlugin.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,7 +22,45 @@
#ifndef _CorePlugin_h
#define _CorePlugin_h
-#include <core/types.h>
+#include <RayPlatform/core/types.h>
+
+#ifdef CONFIG_MINI_RANKS
+
+#define __GetPlugin(corePlugin)
+#define __GetMethod(handle)
+
+#define __GetAdapterObject( corePlugin, handle) \
+ m_adapter_ ## handle
+
+#define __BindAdapter(corePlugin,handle) \
+ __GetAdapterObject( corePlugin, handle) . setObject( this )
+
+/*
+ * Get the adapter
+ */
+#define __GetAdapter(corePlugin,handle) \
+ & __GetAdapterObject (corePlugin, handle)
+
+/*
+ * Add an adapter in a plugin
+ */
+#define __AddAdapter(corePlugin,handle) \
+ Adapter_ ## handle m_adapter_ ## handle
+
+#define __CreatePlugin( corePlugin )
+
+#define __BindPlugin( corePlugin )
+
+#define __DeclarePlugin( corePlugin ) \
+class corePlugin;
+
+#else
+
+/**
+ * Without mini-ranks.
+ */
+
+#define __BindAdapter(corePlugin, handle)
/** get the static name for the variable **/
#define __GetPlugin(corePlugin) \
@@ -36,6 +74,8 @@
#define __GetAdapter(corePlugin,handle) \
Adapter_ ## corePlugin ## _call_ ## handle
+#define __AddAdapter(corePlugin,handle)
+
/* create the static core pluging thing. */
#define __CreatePlugin( corePlugin ) \
static corePlugin * __GetPlugin( corePlugin ) ;
@@ -43,9 +83,18 @@
#define __BindPlugin( corePlugin ) \
__GetPlugin( corePlugin ) = this;
+#define __DeclarePlugin( corePlugin ) \
+
+#endif /* CONFIG_MINI_RANKS */
+
+
class ComputeCore;
/**
+ * This is part of the
+ *
+ * RayPlatform Plugin API
+ *
* In the Ray Platform, plugins are registered onto the
* core (ComputeCore).
* The only method of this interface is to register the plugin.
@@ -55,6 +104,7 @@ class CorePlugin{
protected:
PluginHandle m_plugin;
+ ComputeCore*m_core;
public:
diff --git a/RayPlatform/plugins/RegisteredPlugin.cpp b/RayPlatform/RayPlatform/plugins/RegisteredPlugin.cpp
similarity index 95%
rename from RayPlatform/plugins/RegisteredPlugin.cpp
rename to RayPlatform/RayPlatform/plugins/RegisteredPlugin.cpp
index d137eff..39978b7 100644
--- a/RayPlatform/plugins/RegisteredPlugin.cpp
+++ b/RayPlatform/RayPlatform/plugins/RegisteredPlugin.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,10 +19,11 @@
*/
-#include <plugins/RegisteredPlugin.h>
-#include <core/slave_modes.h>
-#include <core/master_modes.h>
-#include <communication/mpi_tags.h>
+#include "RegisteredPlugin.h"
+
+#include <RayPlatform/core/slave_modes.h>
+#include <RayPlatform/core/master_modes.h>
+#include <RayPlatform/communication/mpi_tags.h>
#include <iostream>
using namespace std;
diff --git a/RayPlatform/plugins/RegisteredPlugin.h b/RayPlatform/RayPlatform/plugins/RegisteredPlugin.h
similarity index 94%
rename from RayPlatform/plugins/RegisteredPlugin.h
rename to RayPlatform/RayPlatform/plugins/RegisteredPlugin.h
index d108fdb..4af71ff 100644
--- a/RayPlatform/plugins/RegisteredPlugin.h
+++ b/RayPlatform/RayPlatform/plugins/RegisteredPlugin.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,7 +22,8 @@
#ifndef _RegisteredPlugin_h
#define _RegisteredPlugin_h
-#include <core/types.h>
+#include <RayPlatform/core/types.h>
+
#include <set>
#include <string>
#include <iostream>
diff --git a/RayPlatform/profiling/Derivative.cpp b/RayPlatform/RayPlatform/profiling/Derivative.cpp
similarity index 86%
rename from RayPlatform/profiling/Derivative.cpp
rename to RayPlatform/RayPlatform/profiling/Derivative.cpp
index 877c06c..0225288 100644
--- a/RayPlatform/profiling/Derivative.cpp
+++ b/RayPlatform/RayPlatform/profiling/Derivative.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,10 +19,11 @@
*/
-#include <profiling/Derivative.h>
-#include <core/statistics.h>
-#include <core/slave_modes.h>
-#include <core/OperatingSystem.h>
+#include "Derivative.h"
+
+#include <RayPlatform/core/statistics.h>
+#include <RayPlatform/core/slave_modes.h>
+#include <RayPlatform/core/OperatingSystem.h>
#include <iostream>
using namespace std;
diff --git a/RayPlatform/profiling/Derivative.h b/RayPlatform/RayPlatform/profiling/Derivative.h
similarity index 86%
rename from RayPlatform/profiling/Derivative.h
rename to RayPlatform/RayPlatform/profiling/Derivative.h
index 20f5361..df0703b 100644
--- a/RayPlatform/profiling/Derivative.h
+++ b/RayPlatform/RayPlatform/profiling/Derivative.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,9 +22,10 @@
#ifndef _Derivative_h
#define _Derivative_h
+#include <RayPlatform/core/OperatingSystem.h>
+
#include <vector>
#include <fstream>
-#include <core/OperatingSystem.h>
#include <map>
#include <stdint.h>
using namespace std;
diff --git a/RayPlatform/RayPlatform/profiling/ProcessStatus.cpp b/RayPlatform/RayPlatform/profiling/ProcessStatus.cpp
new file mode 100644
index 0000000..7e6b769
--- /dev/null
+++ b/RayPlatform/RayPlatform/profiling/ProcessStatus.cpp
@@ -0,0 +1,102 @@
+
+#include "ProcessStatus.h"
+
+#include <fstream>
+#include <sstream>
+#include <vector>
+using namespace std;
+
+void ProcessStatus::getProcessStatus() {
+
+ m_values.clear();
+
+#ifdef __linux__
+
+ ifstream file("/proc/self/status");
+
+ if(!file) {
+ file.close();
+ return;
+ }
+
+ while(!file.eof()) {
+
+ char line[1024];
+ file.getline(line, 1024);
+
+ istringstream lineStream(line);
+
+ string key = "";
+ lineStream >> key;
+
+ string value = lineStream.str();
+ int position = lineStream.tellg();
+ while(position < (int)value.length()) {
+ char symbol = value[position];
+ if(symbol == ' ' || symbol == '\t' || symbol == '\n')
+ position++;
+ else
+ break;
+ }
+
+ string newValue = value.c_str() + position;
+ //cout << " newValue " << newValue << " position= " << position << endl;
+
+ m_values[key] = newValue;
+ }
+
+ file.close();
+
+#endif
+
+}
+
+void ProcessStatus::printMemoryMetrics() {
+
+ vector<string> metrics;
+
+ metrics.push_back("VmLck:");
+ metrics.push_back("VmPin:");
+ metrics.push_back("VmRSS:");
+ metrics.push_back("VmData:");
+ metrics.push_back("VmStk:");
+ metrics.push_back("VmExe:");
+ metrics.push_back("VmLib:");
+ metrics.push_back("VmPTE:");
+ metrics.push_back("VmSwap:");
+
+ for(vector<string>::iterator i = metrics.begin();
+ i != metrics.end();
+ ++i) {
+
+ string & metric = *i;
+
+ /*
+ if(!hasKey(metric))
+ continue;
+ */
+
+ cout << " " << metric << " " << getValue(metric);
+ //cout << "" << metric << "";
+ }
+}
+
+bool ProcessStatus::hasKey(string & key) {
+ return m_values.count(key) > 0;
+}
+
+string & ProcessStatus::getValue(string & metric) {
+
+ if(hasKey(metric))
+ return m_values[metric];
+
+ return m_nothing;
+}
+
+ProcessStatus::ProcessStatus(){
+
+}
+
+ProcessStatus::~ProcessStatus() {
+
+}
diff --git a/RayPlatform/RayPlatform/profiling/ProcessStatus.h b/RayPlatform/RayPlatform/profiling/ProcessStatus.h
new file mode 100644
index 0000000..188fc32
--- /dev/null
+++ b/RayPlatform/RayPlatform/profiling/ProcessStatus.h
@@ -0,0 +1,28 @@
+
+#ifndef ProcessStatusHeader
+#define ProcessStatusHeader
+
+#include <map>
+#include <iostream>
+#include <string>
+using namespace std;
+
+class ProcessStatus {
+
+private:
+ map<string,string> m_values;
+ string m_nothing;
+
+public:
+
+ ProcessStatus();
+ ~ProcessStatus();
+
+ bool hasKey(string & key);
+ void getProcessStatus();
+ void printMemoryMetrics();
+ string & getValue(string & metric);
+
+};
+
+#endif
diff --git a/RayPlatform/profiling/Profiler.cpp b/RayPlatform/RayPlatform/profiling/Profiler.cpp
similarity index 93%
rename from RayPlatform/profiling/Profiler.cpp
rename to RayPlatform/RayPlatform/profiling/Profiler.cpp
index 9d3bc89..bd79590 100644
--- a/RayPlatform/profiling/Profiler.cpp
+++ b/RayPlatform/RayPlatform/profiling/Profiler.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,9 +19,11 @@
*/
-#include <core/OperatingSystem.h>
-#include <profiling/Profiler.h>
-#include <core/slave_modes.h>
+#include "Profiler.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/core/slave_modes.h>
+
#include <assert.h>
#include <iostream>
#include <iomanip>
diff --git a/RayPlatform/profiling/Profiler.h b/RayPlatform/RayPlatform/profiling/Profiler.h
similarity index 94%
rename from RayPlatform/profiling/Profiler.h
rename to RayPlatform/RayPlatform/profiling/Profiler.h
index ed4dd9f..ca52413 100644
--- a/RayPlatform/profiling/Profiler.h
+++ b/RayPlatform/RayPlatform/profiling/Profiler.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
diff --git a/RayPlatform/profiling/TickLogger.cpp b/RayPlatform/RayPlatform/profiling/TickLogger.cpp
similarity index 97%
rename from RayPlatform/profiling/TickLogger.cpp
rename to RayPlatform/RayPlatform/profiling/TickLogger.cpp
index f8fcac3..f923490 100644
--- a/RayPlatform/profiling/TickLogger.cpp
+++ b/RayPlatform/RayPlatform/profiling/TickLogger.cpp
@@ -1,8 +1,8 @@
/*
- Ray
+ RayPlatform: a message-passing development framework
Copyright (C) 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,11 +18,13 @@
see <http://www.gnu.org/licenses/>
*/
-#include <profiling/TickLogger.h>
+#include "TickLogger.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
+
#include <assert.h>
#include <iostream>
#include <time.h>
-#include <core/OperatingSystem.h>
using namespace std;
//#define CONFIG_DISPLAY_TICKS
diff --git a/RayPlatform/profiling/TickLogger.h b/RayPlatform/RayPlatform/profiling/TickLogger.h
similarity index 91%
rename from RayPlatform/profiling/TickLogger.h
rename to RayPlatform/RayPlatform/profiling/TickLogger.h
index b5bcd4f..43075d3 100644
--- a/RayPlatform/profiling/TickLogger.h
+++ b/RayPlatform/RayPlatform/profiling/TickLogger.h
@@ -1,8 +1,8 @@
/*
- Ray
+ RayPlatform: a message-passing development framework
Copyright (C) 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,8 +21,8 @@
#ifndef _TickLogger_h
#define _TickLogger_h
-#include <core/master_modes.h>
-#include <core/slave_modes.h>
+#include <RayPlatform/core/master_modes.h>
+#include <RayPlatform/core/slave_modes.h>
#include <time.h>
#include <fstream>
diff --git a/RayPlatform/profiling/TimePrinter.cpp b/RayPlatform/RayPlatform/profiling/TimePrinter.cpp
similarity index 74%
rename from RayPlatform/profiling/TimePrinter.cpp
rename to RayPlatform/RayPlatform/profiling/TimePrinter.cpp
index b9fa14b..c784f75 100644
--- a/RayPlatform/profiling/TimePrinter.cpp
+++ b/RayPlatform/RayPlatform/profiling/TimePrinter.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -14,17 +14,19 @@
GNU Lesser General Public License for more details.
You have received a copy of the GNU Lesser General Public License
- along with this program (lgpl-3.0.txt).
+ along with this program (lgpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include <profiling/TimePrinter.h>
-#include <core/OperatingSystem.h>
+#include "TimePrinter.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
#include <stdio.h>
#include <iostream>
#include <sstream>
+#include <time.h>
using namespace std;
void TimePrinter::printElapsedTime(string description){
@@ -32,7 +34,7 @@ void TimePrinter::printElapsedTime(string description){
m_endingTime=time(NULL);
int differenceWithLast=m_endingTime-m_lastTime;
m_lastTime=m_endingTime;
- struct tm * timeinfo;
+ struct tm* timeinfo;
timeinfo=localtime(&m_endingTime);
m_descriptions.push_back(description);
@@ -41,12 +43,11 @@ void TimePrinter::printElapsedTime(string description){
printElapsedTimeInStream(&cout,description,timeinfo,differenceWithLast);
if(m_fileSet){
- printElapsedTimeInStream(&m_file,description,timeinfo,differenceWithLast);
+ printElapsedTimeInStreamWithTabulation(&m_file,description,timeinfo,differenceWithLast);
}
}
-void TimePrinter::printElapsedTimeInStream(ostream*stream,string description,struct tm*timeinfo,
-int differenceWithLast){
+void TimePrinter::printElapsedTimeInStream(ostream*stream,string description,struct tm*timeinfo, int differenceWithLast) {
(*stream)<<endl;
(*stream)<<"***"<<endl;
(*stream)<<"Step: "<<description<<endl;
@@ -60,7 +61,18 @@ int differenceWithLast){
(*stream)<<endl;
(*stream)<<"***"<<endl;
(*stream)<<endl;
- fflush(stdout);
+}
+
+void TimePrinter::printElapsedTimeInStreamWithTabulation(ostream* stream, string description, struct tm* timeinfo, int differenceWithLast) {
+ int totalSeconds = m_endingTime - m_startingTime;
+ char buffer[20];
+ strftime(buffer, sizeof(buffer), "%FT%T",timeinfo);
+
+ (*stream) << description << "\t" << buffer << "\t";
+ printDifference(differenceWithLast, stream);
+ (*stream) << "\t";
+ printDifference(totalSeconds, stream);
+ (*stream) << endl;
}
void TimePrinter::printDifferenceFromStart(int rank){
@@ -88,11 +100,12 @@ void TimePrinter::constructor(){
void TimePrinter::setFile(string prefix){
ostringstream fileName;
- fileName<<prefix<<"ElapsedTime.txt";
+ string head = "#Step\tDate\tElapsed time\tSince Beginning\n";
+ fileName<<prefix<<"ElapsedTime.txt";
m_file.open(fileName.str().c_str(),ios_base::out);
-
m_fileSet=true;
+ m_file.write(head.c_str(), head.size());
}
void TimePrinter::printDifference(int difference,ostream*stream){
diff --git a/RayPlatform/profiling/TimePrinter.h b/RayPlatform/RayPlatform/profiling/TimePrinter.h
similarity index 80%
rename from RayPlatform/profiling/TimePrinter.h
rename to RayPlatform/RayPlatform/profiling/TimePrinter.h
index 66371e4..bb36f1b 100644
--- a/RayPlatform/profiling/TimePrinter.h
+++ b/RayPlatform/RayPlatform/profiling/TimePrinter.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -24,10 +24,10 @@
// TODO: move this class in the core
-#include<time.h>
-#include<vector>
+#include <time.h>
+#include <vector>
#include <fstream>
-#include<string>
+#include <string>
using namespace std;
/**
@@ -49,7 +49,10 @@ class TimePrinter{
void printDurationsInStream(ostream*stream,struct tm*t);
void printElapsedTimeInStream(ostream*stream, string description,struct tm*timeinfo,
-int differenceWithLast);
+ int differenceWithLast);
+
+ void printElapsedTimeInStreamWithTabulation(ostream*stream, string description,struct tm*timeinfo,
+ int differenceWithLast);
public:
void printElapsedTime(string description);
void setFile(string prefix);
diff --git a/RayPlatform/routing/ConnectionGraph.cpp b/RayPlatform/RayPlatform/routing/ConnectionGraph.cpp
similarity index 88%
rename from RayPlatform/routing/ConnectionGraph.cpp
rename to RayPlatform/RayPlatform/routing/ConnectionGraph.cpp
index 1b154bb..5b2621a 100644
--- a/RayPlatform/routing/ConnectionGraph.cpp
+++ b/RayPlatform/RayPlatform/routing/ConnectionGraph.cpp
@@ -1,8 +1,8 @@
/*
- RayPlatform
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,7 +19,9 @@
*/
-#include <routing/ConnectionGraph.h>
+#include "ConnectionGraph.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
enum {
__COMPLETE,
@@ -28,7 +30,8 @@ __RANDOM,
__DEBRUIJN,
__KAUTZ,
__EXPERIMENTAL,
-__HYPERCUBE
+__POLYTOPE,
+__TORUS
};
/**
@@ -63,6 +66,10 @@ bool ConnectionGraph::isConnected(Rank source,Rank destination){
* Write files.
*/
void ConnectionGraph::writeFiles(string prefix){
+
+ if(m_rank==MASTER_RANK)
+ createDirectory(prefix.c_str());
+
// dump the connections in a file
ostringstream file;
file<<prefix<<"Connections.txt";
@@ -92,7 +99,7 @@ void ConnectionGraph::writeFiles(string prefix){
file2<<prefix<<"Routes.txt";
ofstream f2(file2.str().c_str());
- if(m_typeCode==__HYPERCUBE){
+ if(m_typeCode==__POLYTOPE || m_typeCode==__TORUS){
f2<<"Routes are dynamically determined with real-time load-balancing."<<endl;
}else{
f2<<"#Source Destination Hops Route"<<endl;
@@ -241,10 +248,13 @@ int degree){
m_size=numberOfRanks;
- /** provide the user-provided degree for those
- * requiring it */
+/**
+ * Provide the user-provided degree for those
+ * requiring it.
+ */
m_deBruijn.setDegree(degree);
- m_hypercube.setDegree(degree);
+ m_polytope.setDegree(degree);
+ m_torus.setDegree(degree);
if(type==""){
type="debruijn";
@@ -255,11 +265,19 @@ int degree){
if(type=="random"){
m_implementation=&m_random;
m_typeCode=__RANDOM;
+
}else if((type=="hypercube"||type=="polytope")
- && m_hypercube.isValid(numberOfRanks)){
+ && m_polytope.isValid(numberOfRanks)){
+
+ m_implementation=&m_polytope;
+ m_typeCode=__POLYTOPE;
+
+ }else if(type=="torus"
+ && m_torus.isValid(numberOfRanks)){
+
+ m_implementation=&m_torus;
+ m_typeCode=__TORUS;
- m_implementation=&m_hypercube;
- m_typeCode=__HYPERCUBE;
}else if(type=="group"){
m_implementation=&m_group;
m_typeCode=__GROUP;
@@ -310,14 +328,19 @@ int ConnectionGraph::getRelaysTo0(Rank rank){
void ConnectionGraph::printStatus(){
- if(m_typeCode==__HYPERCUBE)
- m_hypercube.printStatus(m_rank);
+ if(m_typeCode==__POLYTOPE)
+ m_polytope.printStatus(m_rank);
+ else if(m_typeCode==__TORUS)
+ m_torus.printStatus(m_rank);
+
}
void ConnectionGraph::start(Rank rank){
m_rank=rank;
- if(m_typeCode==__HYPERCUBE)
- m_hypercube.start();
+ if(m_typeCode==__POLYTOPE)
+ m_polytope.start();
+ else if(m_typeCode==__TORUS)
+ m_torus.start();
}
diff --git a/RayPlatform/routing/ConnectionGraph.h b/RayPlatform/RayPlatform/routing/ConnectionGraph.h
similarity index 79%
rename from RayPlatform/routing/ConnectionGraph.h
rename to RayPlatform/RayPlatform/routing/ConnectionGraph.h
index a7a3d84..2664bd9 100644
--- a/RayPlatform/routing/ConnectionGraph.h
+++ b/RayPlatform/RayPlatform/routing/ConnectionGraph.h
@@ -1,8 +1,8 @@
/*
- RayPlatform
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,23 +22,26 @@
#ifndef _ConnectionGraph
#define _ConnectionGraph
+#include "GraphImplementation.h"
+#include "GraphImplementationRandom.h"
+#include "GraphImplementationGroup.h"
+#include "GraphImplementationDeBruijn.h"
+#include "GraphImplementationKautz.h"
+#include "GraphImplementationExperimental.h"
+#include "GraphImplementationComplete.h"
+#include "Polytope.h"
+#include "Torus.h"
+
+#include <RayPlatform/core/types.h>
+#include <RayPlatform/core/statistics.h>
+
#include <vector>
-#include <set>
-#include <map>
-#include <core/statistics.h>
+#include <string>
#include <iostream>
#include <fstream>
#include <sstream>
-#include <routing/GraphImplementation.h>
-#include <routing/GraphImplementationRandom.h>
-#include <routing/GraphImplementationGroup.h>
-#include <routing/GraphImplementationDeBruijn.h>
-#include <routing/GraphImplementationKautz.h>
-#include <routing/GraphImplementationExperimental.h>
-#include <routing/GraphImplementationComplete.h>
-#include <routing/Hypercube.h>
-#include <string>
-#include <core/types.h>
+#include <set>
+#include <map>
using namespace std;
/** a graph for connections between compute cores */
@@ -57,7 +60,8 @@ class ConnectionGraph{
GraphImplementationKautz m_kautz;
GraphImplementationExperimental m_experimental;
GraphImplementationGroup m_group;
- Hypercube m_hypercube;
+ Polytope m_polytope;
+ Torus m_torus;
/** verbosity */
bool m_verbose;
diff --git a/RayPlatform/routing/GraphImplementation.cpp b/RayPlatform/RayPlatform/routing/GraphImplementation.cpp
similarity index 98%
rename from RayPlatform/routing/GraphImplementation.cpp
rename to RayPlatform/RayPlatform/routing/GraphImplementation.cpp
index 57208ed..9edde1d 100644
--- a/RayPlatform/routing/GraphImplementation.cpp
+++ b/RayPlatform/RayPlatform/routing/GraphImplementation.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,7 +19,8 @@
*/
-#include <routing/GraphImplementation.h>
+#include "GraphImplementation.h"
+
#include <iostream>
#include <stdlib.h> /* srand, rand */
#include <algorithm> /* random_shuffle */
diff --git a/RayPlatform/routing/GraphImplementation.h b/RayPlatform/RayPlatform/routing/GraphImplementation.h
similarity index 93%
rename from RayPlatform/routing/GraphImplementation.h
rename to RayPlatform/RayPlatform/routing/GraphImplementation.h
index 2829aa6..3b3009e 100644
--- a/RayPlatform/routing/GraphImplementation.h
+++ b/RayPlatform/RayPlatform/routing/GraphImplementation.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -35,7 +35,7 @@ public:
#include <vector>
#include <set>
#include <map>
-#include <core/types.h>
+#include <RayPlatform/core/types.h>
using namespace std;
/** virtual interface for graphs */
diff --git a/RayPlatform/routing/GraphImplementationComplete.cpp b/RayPlatform/RayPlatform/routing/GraphImplementationComplete.cpp
similarity index 88%
rename from RayPlatform/routing/GraphImplementationComplete.cpp
rename to RayPlatform/RayPlatform/routing/GraphImplementationComplete.cpp
index d822a78..562166b 100644
--- a/RayPlatform/routing/GraphImplementationComplete.cpp
+++ b/RayPlatform/RayPlatform/routing/GraphImplementationComplete.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,7 +19,7 @@
*/
-#include <routing/GraphImplementationComplete.h>
+#include "GraphImplementationComplete.h"
/**
* complete graph
diff --git a/RayPlatform/routing/GraphImplementationComplete.h b/RayPlatform/RayPlatform/routing/GraphImplementationComplete.h
similarity index 85%
rename from RayPlatform/routing/GraphImplementationComplete.h
rename to RayPlatform/RayPlatform/routing/GraphImplementationComplete.h
index c97ec45..bd2a747 100644
--- a/RayPlatform/routing/GraphImplementationComplete.h
+++ b/RayPlatform/RayPlatform/routing/GraphImplementationComplete.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,7 +22,7 @@
#ifndef _GraphImplementationComplete_h
#define _GraphImplementationComplete_h
-#include <routing/GraphImplementation.h>
+#include "GraphImplementation.h"
/**
* complete graph
diff --git a/RayPlatform/routing/GraphImplementationDeBruijn.cpp b/RayPlatform/RayPlatform/routing/GraphImplementationDeBruijn.cpp
similarity index 97%
rename from RayPlatform/routing/GraphImplementationDeBruijn.cpp
rename to RayPlatform/RayPlatform/routing/GraphImplementationDeBruijn.cpp
index 997b748..ec27557 100644
--- a/RayPlatform/routing/GraphImplementationDeBruijn.cpp
+++ b/RayPlatform/RayPlatform/routing/GraphImplementationDeBruijn.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,7 +21,8 @@
//#define CONFIG_ROUTING_DE_BRUIJN_COMPUTE_ROUTES
-#include <routing/GraphImplementationDeBruijn.h>
+#include "GraphImplementationDeBruijn.h"
+
#include <iostream>
using namespace std;
diff --git a/RayPlatform/routing/GraphImplementationDeBruijn.h b/RayPlatform/RayPlatform/routing/GraphImplementationDeBruijn.h
similarity index 90%
rename from RayPlatform/routing/GraphImplementationDeBruijn.h
rename to RayPlatform/RayPlatform/routing/GraphImplementationDeBruijn.h
index e1e4485..1e3a4fe 100644
--- a/RayPlatform/routing/GraphImplementationDeBruijn.h
+++ b/RayPlatform/RayPlatform/routing/GraphImplementationDeBruijn.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,7 +22,8 @@
#ifndef _GraphImplementationDeBruijn_h
#define _GraphImplementationDeBruijn_h
-#include <routing/GraphImplementation.h>
+#include "GraphImplementation.h"
+
#include <vector>
#include <stdint.h>
using namespace std;
diff --git a/RayPlatform/routing/GraphImplementationExperimental.cpp b/RayPlatform/RayPlatform/routing/GraphImplementationExperimental.cpp
similarity index 97%
rename from RayPlatform/routing/GraphImplementationExperimental.cpp
rename to RayPlatform/RayPlatform/routing/GraphImplementationExperimental.cpp
index d86b71e..a9b033d 100644
--- a/RayPlatform/routing/GraphImplementationExperimental.cpp
+++ b/RayPlatform/RayPlatform/routing/GraphImplementationExperimental.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,7 +21,8 @@
//#define CONFIG_ROUTING_KAUTZ_COMPUTE_ROUTES
-#include <routing/GraphImplementationExperimental.h>
+#include "GraphImplementationExperimental.h"
+
#include <iostream>
using namespace std;
diff --git a/RayPlatform/routing/GraphImplementationExperimental.h b/RayPlatform/RayPlatform/routing/GraphImplementationExperimental.h
similarity index 92%
rename from RayPlatform/routing/GraphImplementationExperimental.h
rename to RayPlatform/RayPlatform/routing/GraphImplementationExperimental.h
index 239261c..cf39085 100644
--- a/RayPlatform/routing/GraphImplementationExperimental.h
+++ b/RayPlatform/RayPlatform/routing/GraphImplementationExperimental.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,7 +22,8 @@
#ifndef _GraphImplementationExperimental_h
#define _GraphImplementationExperimental_h
-#include <routing/GraphImplementation.h>
+#include "GraphImplementation.h"
+
#include <vector>
#include <stdint.h>
using namespace std;
diff --git a/RayPlatform/routing/GraphImplementationGroup.cpp b/RayPlatform/RayPlatform/routing/GraphImplementationGroup.cpp
similarity index 95%
rename from RayPlatform/routing/GraphImplementationGroup.cpp
rename to RayPlatform/RayPlatform/routing/GraphImplementationGroup.cpp
index 8648fa5..fae665c 100644
--- a/RayPlatform/routing/GraphImplementationGroup.cpp
+++ b/RayPlatform/RayPlatform/routing/GraphImplementationGroup.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,7 +19,8 @@
*/
-#include <routing/GraphImplementationGroup.h>
+#include "GraphImplementationGroup.h"
+
#include <assert.h>
/**
diff --git a/RayPlatform/routing/GraphImplementationGroup.h b/RayPlatform/RayPlatform/routing/GraphImplementationGroup.h
similarity index 87%
rename from RayPlatform/routing/GraphImplementationGroup.h
rename to RayPlatform/RayPlatform/routing/GraphImplementationGroup.h
index 26fe6a1..24327b0 100644
--- a/RayPlatform/routing/GraphImplementationGroup.h
+++ b/RayPlatform/RayPlatform/routing/GraphImplementationGroup.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,7 +22,7 @@
#ifndef _GraphImplementationGroup_h
#define _GraphImplementationGroup_h
-#include <routing/GraphImplementation.h>
+#include "GraphImplementation.h"
/**
* Graph with groups
diff --git a/RayPlatform/routing/GraphImplementationKautz.cpp b/RayPlatform/RayPlatform/routing/GraphImplementationKautz.cpp
similarity index 97%
rename from RayPlatform/routing/GraphImplementationKautz.cpp
rename to RayPlatform/RayPlatform/routing/GraphImplementationKautz.cpp
index 7390f75..00c268e 100644
--- a/RayPlatform/routing/GraphImplementationKautz.cpp
+++ b/RayPlatform/RayPlatform/routing/GraphImplementationKautz.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,7 +21,8 @@
//#define CONFIG_ROUTING_KAUTZ_COMPUTE_ROUTES
-#include <routing/GraphImplementationKautz.h>
+#include "GraphImplementationKautz.h"
+
#include <iostream>
using namespace std;
diff --git a/RayPlatform/routing/GraphImplementationKautz.h b/RayPlatform/RayPlatform/routing/GraphImplementationKautz.h
similarity index 91%
rename from RayPlatform/routing/GraphImplementationKautz.h
rename to RayPlatform/RayPlatform/routing/GraphImplementationKautz.h
index b0cd50b..dd32488 100644
--- a/RayPlatform/routing/GraphImplementationKautz.h
+++ b/RayPlatform/RayPlatform/routing/GraphImplementationKautz.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,7 +22,8 @@
#ifndef _GraphImplementationKautz_h
#define _GraphImplementationKautz_h
-#include <routing/GraphImplementation.h>
+#include "GraphImplementation.h"
+
#include <vector>
#include <stdint.h>
using namespace std;
diff --git a/RayPlatform/routing/GraphImplementationRandom.cpp b/RayPlatform/RayPlatform/routing/GraphImplementationRandom.cpp
similarity index 93%
rename from RayPlatform/routing/GraphImplementationRandom.cpp
rename to RayPlatform/RayPlatform/routing/GraphImplementationRandom.cpp
index 349c9f9..362e43f 100644
--- a/RayPlatform/routing/GraphImplementationRandom.cpp
+++ b/RayPlatform/RayPlatform/routing/GraphImplementationRandom.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,7 +19,8 @@
*/
-#include <routing/GraphImplementationRandom.h>
+#include "GraphImplementationRandom.h"
+
#include <math.h> /* for log */
#include <algorithm> /* random_shuffle */
#include <assert.h>
diff --git a/RayPlatform/routing/GraphImplementationRandom.h b/RayPlatform/RayPlatform/routing/GraphImplementationRandom.h
similarity index 85%
rename from RayPlatform/routing/GraphImplementationRandom.h
rename to RayPlatform/RayPlatform/routing/GraphImplementationRandom.h
index 136bf65..ed81c86 100644
--- a/RayPlatform/routing/GraphImplementationRandom.h
+++ b/RayPlatform/RayPlatform/routing/GraphImplementationRandom.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,7 +22,7 @@
#ifndef _GraphImplementationRandom_h
#define _GraphImplementationRandom_h
-#include <routing/GraphImplementation.h>
+#include "GraphImplementation.h"
/**
* random graph with n*log(n)/2 edges
diff --git a/RayPlatform/routing/Hypercube.cpp b/RayPlatform/RayPlatform/routing/Polytope.cpp
similarity index 87%
rename from RayPlatform/routing/Hypercube.cpp
rename to RayPlatform/RayPlatform/routing/Polytope.cpp
index c8081c2..fde8ed7 100644
--- a/RayPlatform/routing/Hypercube.cpp
+++ b/RayPlatform/RayPlatform/routing/Polytope.cpp
@@ -1,8 +1,8 @@
/*
- RayPlatform
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,7 +19,8 @@
*/
-#include <routing/Hypercube.h>
+#include "Polytope.h"
+
#include <iostream>
using namespace std;
@@ -32,7 +33,7 @@ using namespace std;
/**
* TODO: move this upstream
*/
-int Hypercube::getPower(int base,int exponent){
+int Polytope::getPower(int base,int exponent){
int a=1;
while(exponent--)
a*=base;
@@ -43,7 +44,7 @@ int Hypercube::getPower(int base,int exponent){
* TODO: move this upstream
* convert a number to a vertex
*/
-void Hypercube::convertToVertex(int i,Tuple*tuple){
+void Polytope::convertToVertex(int i,Tuple*tuple){
for(int power=0;power<m_wordLength;power++){
int value=(i%getPower(m_alphabetSize,power+1))/getPower(m_alphabetSize,power);
tuple->m_digits[power]=value;
@@ -53,7 +54,7 @@ void Hypercube::convertToVertex(int i,Tuple*tuple){
/**
* TODO: move this upstream
*/
-bool Hypercube::isAPowerOf(int n,int base){
+bool Polytope::isAPowerOf(int n,int base){
int remaining=n;
while(remaining>1){
@@ -65,9 +66,9 @@ bool Hypercube::isAPowerOf(int n,int base){
}
/**
- * only implemented for the hypercube with alphabet {0,1}
+ * only implemented for the polytope with alphabet {0,1}
*/
-void Hypercube::configureGraph(int n){
+void Polytope::configureGraph(int n){
#if 0
// test some cases
@@ -101,9 +102,9 @@ void Hypercube::configureGraph(int n){
//
// (alphabetSize -1) * log(numberOfVertices) = degree * log(alphabetSize)
//
- // The alphabetSize of a hypercube is 2.
+ // The alphabetSize of a polytope is 2.
//
- // However, we can build similar things to a hypercube
+ // However, we can build similar things to a polytope
// with a different alphabetSize
//
// alphabetSize=2, 1024 vertices
@@ -150,7 +151,7 @@ void Hypercube::configureGraph(int n){
if(n == getPower(alphabetSize,wordLength)){
int degree=(alphabetSize-1)*wordLength;
if(degree==m_degree){
- cout<<"[Hypercube] found a hypercube topology that matches the provided degree "<<m_degree<<endl;
+ cout<<"[Polytope] found a polytope topology that matches the provided degree "<<m_degree<<endl;
base=alphabetSize;
}
@@ -159,7 +160,7 @@ void Hypercube::configureGraph(int n){
if(!isAPowerOf(n,base)){
- cout<<"Error: "<<n<<" is not a power of 2, can not use the hypercube."<<endl;
+ cout<<"Error: "<<n<<" is not a power of 2, can not use the polytope."<<endl;
m_alphabetSize=__INVALID_ALPHABET_SIZE;
return;
}
@@ -178,15 +179,15 @@ void Hypercube::configureGraph(int n){
m_wordLength=digits; // this is wordLength
m_size=n; // this is the number of vertices
- cout<<"[Hypercube] Size: "<<m_size<<" AlphabetSize: "<<m_alphabetSize<<" WordLength: "<<m_wordLength<<" Degree: "<<m_degree<<endl;
+ cout<<"[Polytope] Size: "<<m_size<<" AlphabetSize: "<<m_alphabetSize<<" WordLength: "<<m_wordLength<<" Degree: "<<m_degree<<endl;
start();
}
-void Hypercube::setLoad(int position,int symbol,uint64_t value){
+void Polytope::setLoad(int position,int symbol,uint64_t value){
m_loadValues[position*m_alphabetSize+symbol]=value;
}
-uint64_t Hypercube::getLoad(int position,int symbol){
+uint64_t Polytope::getLoad(int position,int symbol){
#ifdef ASSERT
assert(position<m_wordLength);
@@ -196,15 +197,15 @@ uint64_t Hypercube::getLoad(int position,int symbol){
return m_loadValues[position*m_alphabetSize+symbol];
}
-void Hypercube::makeConnections(int n){
+void Polytope::makeConnections(int n){
configureGraph(n);
if(m_verbose){
- cout<<"[Hypercube::makeConnections] using "<<m_wordLength<<" for diameter with base ";
+ cout<<"[Polytope::makeConnections] using "<<m_wordLength<<" for diameter with base ";
cout<<m_alphabetSize<<endl;
- cout<<"[Hypercube::makeConnections] The MPI graph has "<<m_size<<" vertices"<<endl;
- cout<<"[Hypercube::makeConnections] The hypercube has "<<m_size<<" vertices"<<endl;
+ cout<<"[Polytope::makeConnections] The MPI graph has "<<m_size<<" vertices"<<endl;
+ cout<<"[Polytope::makeConnections] The polytope has "<<m_size<<" vertices"<<endl;
}
// create empty sets.
@@ -236,7 +237,7 @@ void Hypercube::makeConnections(int n){
* base m_alphabetSize to base 10
* TODO: this should be moved upstream
*/
-int Hypercube::convertToBase10(Tuple*vertex){
+int Polytope::convertToBase10(Tuple*vertex){
int a=0;
for(int i=0;i<m_wordLength;i++){
a+=vertex->m_digits[i]*getPower(m_alphabetSize,i);
@@ -247,7 +248,7 @@ int Hypercube::convertToBase10(Tuple*vertex){
/**
* TODO: this should be moved upstream
*/
-void Hypercube::printVertex(Tuple*a){
+void Polytope::printVertex(Tuple*a){
for(int i=0;i<m_wordLength;i++){
if(i!=0)
cout<<",";
@@ -259,11 +260,11 @@ void Hypercube::printVertex(Tuple*a){
/**
* TODO: remove me
*/
-void Hypercube::computeRoute(Rank a,Rank b,vector<Rank>*route){}
+void Polytope::computeRoute(Rank a,Rank b,vector<Rank>*route){}
// use a round-robin algorithm.
//#define __ROUND_ROBIN__
-Rank Hypercube::getNextRankInRoute(Rank source,Rank destination,Rank rank){
+Rank Polytope::getNextRankInRoute(Rank source,Rank destination,Rank rank){
#ifdef __ROUND_ROBIN__
@@ -277,7 +278,7 @@ Rank Hypercube::getNextRankInRoute(Rank source,Rank destination,Rank rank){
}
/** with de Bruijn routing, no route are pre-computed at all */
-void Hypercube::makeRoutes(){
+void Polytope::makeRoutes(){
/* compute relay points */
computeRelayEvents();
}
@@ -325,7 +326,7 @@ void Hypercube::makeRoutes(){
// case m_digits destination can be obtained with m_digits shifts, overlap is 0
*/
-Rank Hypercube::computeNextRankInRoute(Rank source,Rank destination,Rank current){
+Rank Polytope::computeNextRankInRoute(Rank source,Rank destination,Rank current){
#ifdef ASSERT
assert(destination!=current); // we don't need any routing...
@@ -403,7 +404,7 @@ Rank Hypercube::computeNextRankInRoute(Rank source,Rank destination,Rank current
/**
* TODO: move this upstream
*/
-bool Hypercube::isConnected(Rank source,Rank destination){
+bool Polytope::isConnected(Rank source,Rank destination){
if(source==destination)
return true;
@@ -415,7 +416,7 @@ bool Hypercube::isConnected(Rank source,Rank destination){
* also, we allow any vertex to communicate with itself
* regardless of the property
*/
-bool Hypercube::computeConnection(Rank source,Rank destination){
+bool Polytope::computeConnection(Rank source,Rank destination){
if(source==destination) // no difference
return false;
@@ -440,7 +441,7 @@ bool Hypercube::computeConnection(Rank source,Rank destination){
/**
* TODO: move this upstream, instead, add a protected m_valid field.
*/
-bool Hypercube::isValid(int n){
+bool Polytope::isValid(int n){
configureGraph(n);
return m_alphabetSize!=__INVALID_ALPHABET_SIZE;
@@ -449,12 +450,12 @@ bool Hypercube::isValid(int n){
/**
* TODO: move this upstream
*/
-void Hypercube::setDegree(int degree){
+void Polytope::setDegree(int degree){
m_degree=degree;
}
-void Hypercube::printStatus(Rank rank){
- cout<<"[Hypercube] Load values:"<<endl;
+void Polytope::printStatus(Rank rank){
+ cout<<"[Polytope] Load values:"<<endl;
cout<<"AlphabetSize: "<<m_alphabetSize<<endl;
cout<<"WordLength: "<<m_wordLength<<endl;
cout<<"Self: ";
@@ -489,7 +490,7 @@ void Hypercube::printStatus(Rank rank){
}
}
-void Hypercube::start(){
+void Polytope::start(){
for(int i=0;i<m_wordLength;i++)
for(int j=0;j<m_alphabetSize;j++)
setLoad(i,j,0);
@@ -498,7 +499,7 @@ void Hypercube::start(){
m_currentSymbol=0;
}
-Rank Hypercube::computeNextRankInRouteWithRoundRobin(Rank source,Rank destination,Rank current){
+Rank Polytope::computeNextRankInRouteWithRoundRobin(Rank source,Rank destination,Rank current){
#ifdef ASSERT
assert(destination!=current); // we don't need any routing...
diff --git a/RayPlatform/routing/Hypercube.h b/RayPlatform/RayPlatform/routing/Polytope.h
similarity index 86%
copy from RayPlatform/routing/Hypercube.h
copy to RayPlatform/RayPlatform/routing/Polytope.h
index cbffeba..12588ff 100644
--- a/RayPlatform/routing/Hypercube.h
+++ b/RayPlatform/RayPlatform/routing/Polytope.h
@@ -1,8 +1,8 @@
/*
- RayPlatform
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,10 +19,11 @@
*/
-#ifndef _Hypercube_h
-#define _Hypercube_h
+#ifndef _Polytope_h
+#define _Polytope_h
+
+#include "GraphImplementation.h"
-#include <routing/GraphImplementation.h>
#include <vector>
#include <stdint.h>
using namespace std;
@@ -41,15 +42,15 @@ using namespace std;
#define __NUMBER_OF_STORED_LOAD_VALUES 256
/**
- * Hypercube
+ * Polytope
* n must be a power of 2 to be a hypercube.
* Otherwise, it is an other polytope.
- * A hypercube is a convex regular polytope.
- * This class in fact implements a generalized hypercube,
+ * A polytope is a convex regular polytope.
+ * This class in fact implements a generalized polytope,
* usually called a convex regular polytope.
* \author Sébastien Boisvert
*/
-class Hypercube : public GraphImplementation{
+class Polytope : public GraphImplementation{
uint64_t m_loadValues[__NUMBER_OF_STORED_LOAD_VALUES];
diff --git a/RayPlatform/RayPlatform/routing/Torus.cpp b/RayPlatform/RayPlatform/routing/Torus.cpp
new file mode 100644
index 0000000..6b48f63
--- /dev/null
+++ b/RayPlatform/RayPlatform/routing/Torus.cpp
@@ -0,0 +1,558 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#include "Torus.h"
+
+#include <iostream>
+using namespace std;
+
+#ifdef ASSERT
+#include <assert.h>
+#endif /* ASSERT */
+
+#define __INVALID_ALPHABET_SIZE -1
+
+/**
+ * TODO: move this upstream
+ */
+int Torus::getPower(int base,int exponent){
+ int a=1;
+ while(exponent--)
+ a*=base;
+ return a;
+}
+
+/**
+ * TODO: move this upstream
+ * convert a number to a vertex
+ */
+void Torus::convertToVertex(int i,Tuple*tuple){
+ for(int power=0;power<m_dimension;power++){
+ int value=(i%getPower(m_radix,power+1))/getPower(m_radix,power);
+ tuple->m_digits[power]=value;
+ }
+}
+
+/**
+ * TODO: move this upstream
+ */
+bool Torus::isAPowerOf(int n,int base){
+ int remaining=n;
+
+ while(remaining>1){
+ if(remaining%base != 0)
+ return false;
+ remaining/=base;
+ }
+ return true;
+}
+
+void Torus::configureGraph(int vertices){
+
+ cout<<"[Torus] configureGraph vertices= "<<vertices<<" degree="<<m_degree<<endl;
+
+ int theRadix=2;
+
+ /* try to modify the base such that radix^dimension= vertices */
+ /* and that meets the provided degree. */
+
+ for(int radix=vertices-1;radix>=2;radix--){
+ int dimension=1;
+
+ while(vertices > getPower(radix,dimension)){
+ dimension++;
+ }
+
+ if(vertices == getPower(radix,dimension)){
+
+/*
+ * For each dimension we can go left and right.
+ */
+ int degree=2*dimension;
+
+ if(degree==m_degree){
+ cout<<"[Torus] found a torus topology that matches the provided degree "<<m_degree<<endl;
+
+ theRadix=radix;
+
+ break;
+
+ }
+ }
+ }
+
+
+ if(!isAPowerOf(vertices,theRadix)){
+ cout<<"Error: "<<vertices<<" is not a power of "<<theRadix<<", can not use the torus."<<endl;
+ m_radix=__INVALID_ALPHABET_SIZE;
+ return;
+ }
+
+ int dimension=1;
+
+ while(vertices > getPower(theRadix,dimension)){
+ dimension++;
+ }
+
+ #ifdef ASSERT
+ assert(vertices ==getPower(theRadix,dimension));
+ #endif
+
+ m_radix=theRadix; // this is radix
+ m_dimension=dimension; // this is wordLength
+ m_size=vertices; // this is the number of vertices
+
+ if(m_verbose)
+ cout<<"[Torus] Vertices: "<<m_size<<" Radix: "<<m_radix<<" Dimension: "<<m_dimension<<" Degree: "<<m_degree<<endl;
+
+ start();
+}
+
+uint64_t* Torus::getLoadValueCell(int position,int symbol){
+ return m_loadValues+position*m_radix+symbol;
+}
+
+void Torus::setLoad(int position,int symbol,uint64_t value){
+
+#if 0
+ cout<<"[DEVEL] setLoad position="<<position<<" symbol="<<symbol<<" value="<<value<<endl;
+#endif
+
+ *(getLoadValueCell(position,symbol))=value;
+}
+
+uint64_t Torus::getLoad(int position,int symbol){
+
+ #ifdef ASSERT
+ assert(position<m_dimension);
+ assert(symbol<m_radix);
+ #endif /* ASSERT */
+
+ return *(getLoadValueCell(position,symbol));
+}
+
+void Torus::makeConnections(int vertices){
+
+ configureGraph(vertices);
+
+ if(m_verbose){
+ cout<<"[Torus::makeConnections] using "<<m_dimension<<" for diameter with base ";
+ cout<<m_radix<<endl;
+ cout<<"[Torus::makeConnections] The MPI graph has "<<m_size<<" vertices"<<endl;
+ cout<<"[Torus::makeConnections] The torus has "<<m_size<<" vertices"<<endl;
+ }
+
+ // create empty sets.
+ for(Rank i=0;i<m_size;i++){
+ set<Rank> b;
+ m_outcomingConnections.push_back(b);
+ m_incomingConnections.push_back(b);
+ }
+
+ // populate vertices
+ for(Rank i=0;i<m_size;i++){
+ Tuple a;
+ convertToVertex(i,&a);
+ m_graphToVertex.push_back(a);
+ }
+
+ // make all connections.
+ for(Rank i=0;i<m_size;i++){
+ for(Rank j=0;j<m_size;j++){
+ if(computeConnection(i,j)){
+ m_outcomingConnections[i].insert(j);
+ m_incomingConnections[j].insert(i);
+ }
+ }
+ }
+}
+
+/*
+ * base m_radix to base 10
+ * TODO: this should be moved upstream
+ */
+int Torus::convertToBase10(Tuple*vertex){
+ int a=0;
+ for(int i=0;i<m_dimension;i++){
+ a+=vertex->m_digits[i]*getPower(m_radix,i);
+ }
+ return a;
+}
+
+/**
+ * TODO: this should be moved upstream
+ */
+void Torus::printVertex(Tuple*a){
+ for(int i=0;i<m_dimension;i++){
+ if(i!=0)
+ cout<<",";
+ int value=a->m_digits[i];
+ cout<<value;
+ }
+}
+
+/**
+ * TODO: remove me
+ */
+void Torus::computeRoute(Rank a,Rank b,vector<Rank>*route){
+}
+
+Rank Torus::getNextRankInRoute(Rank source,Rank destination,Rank current){
+
+#if 0
+ cout<<"[DEVEL] getNextRankInRoute source="<<source<<" destination="<<destination<<" current="<<current<<endl;
+#endif
+
+ #ifdef ASSERT
+ assert(destination!=current); // we don't need any routing...
+ assert(source>=0);
+ assert(destination>=0);
+ assert(current>=0);
+ assert(source<m_size);
+ assert(destination<m_size);
+ assert(current<m_size);
+ #endif
+
+ //Tuple*sourceVertex=&(m_graphToVertex[source]);
+ Tuple*destinationVertex=&(m_graphToVertex[destination]);
+ Tuple*currentVertex=&(m_graphToVertex[current]);
+
+ int NO_VALUE=-1;
+
+ int bestPosition=NO_VALUE;
+ int bestSymbol=0;
+ uint64_t bestLoad=0;
+
+ // select the next that has the lowest load.
+ for(int position=0;position<m_dimension;position++){
+
+ //int sourceSymbol=sourceVertex[i];
+ int currentSymbol=currentVertex->m_digits[position];
+ int destinationSymbol=destinationVertex->m_digits[position];
+
+ // we don't need to test this one
+ // because the bit is already correct
+ if(currentSymbol==destinationSymbol)
+ continue;
+
+/*
+ * What is the distance in this dimension on the left ?
+ */
+ int leftDistance=getDistance(currentSymbol,destinationSymbol);
+
+ #ifdef ASSERT
+ assert(leftDistance!=0);
+ assert(leftDistance>0);
+ assert(leftDistance>=1);
+ #endif
+
+/*
+ * Likewise, what is the distance in this dimension on the right ?
+ */
+ int rightDistance=getDistance(destinationSymbol,currentSymbol);
+
+ #ifdef ASSERT
+ assert(rightDistance!=0);
+ assert(rightDistance>0);
+ assert(rightDistance>=1);
+ #endif
+
+ if(leftDistance<=rightDistance){
+
+ int nextSymbol=decrementSymbol(currentSymbol);
+
+ #ifdef ASSERT
+ assert(getMinimumDistance(currentSymbol,nextSymbol)==1);
+ assert(getMinimumDistance(nextSymbol,currentSymbol)==1);
+ #endif
+
+ // at this point, the symbol needs to be changed
+ uint64_t load=getLoad(position,nextSymbol);
+
+ // select this edge if it has a lower load
+ // or if it is the first one we are testing.
+ if(bestPosition==NO_VALUE||load<bestLoad){
+ bestPosition=position;
+ bestSymbol=nextSymbol;
+ bestLoad=load;
+ }
+ }
+
+ if(rightDistance<=leftDistance){
+
+ int nextSymbol=incrementSymbol(currentSymbol);
+
+ #ifdef ASSERT
+ assert(getMinimumDistance(currentSymbol,nextSymbol)==1);
+ assert(getMinimumDistance(nextSymbol,currentSymbol)==1);
+ #endif
+
+ // at this point, the symbol needs to be changed
+ uint64_t load=getLoad(position,nextSymbol);
+
+ // select this edge if it has a lower load
+ // or if it is the first one we are testing.
+ if(bestPosition==NO_VALUE||load<bestLoad){
+ bestPosition=position;
+ bestSymbol=nextSymbol;
+ bestLoad=load;
+ }
+ }
+ }
+
+ #ifdef ASSERT
+ assert(bestPosition!=NO_VALUE);
+ assert(bestPosition>=0);
+ assert(bestPosition<m_dimension);
+ assert(bestSymbol>=0);
+ assert(bestSymbol<m_radix);
+ assert(bestLoad>=0);
+ #endif
+
+ // here, we know where we want to go
+ //
+ // This is the agenda:
+ //
+ // * we are here
+ // *
+ // *
+ // source -> ... -> current -> next -> ... -> destination
+
+ // we just need to update next with the choice having
+ // the lowest load
+
+ Tuple next=*currentVertex;
+
+ next.m_digits[bestPosition]=bestSymbol;
+
+#if 0
+ cout<<"[DEVEL] upstream setLoad position="<<bestPosition<<" symbol="<<bestSymbol<<" value="<<bestLoad+1;
+ cout<<" current:";
+ printVertex(currentVertex);
+ cout<<" next:";
+ printVertex(&next);
+ cout<<endl;
+#endif
+
+ // we increment the load by 1
+ setLoad(bestPosition,bestSymbol,bestLoad+1);
+
+ Rank nextRank=convertToBase10(&next);
+
+ #ifdef ASSERT
+ assert(current!=nextRank);
+ assert(computeConnection(current,nextRank) == true);
+ #endif
+
+ return nextRank;
+}
+
+/** with de Bruijn routing, no route are pre-computed at all */
+void Torus::makeRoutes(){
+ /* compute relay points */
+
+ computeRelayEvents();
+}
+
+int Torus::getDistance(int end,int start){
+
+/*
+ * We could also add m_radix only if symbol-1 < 0, but out-of-order execution is
+ * faster without branching.
+ */
+
+ return (end-start+m_radix)%m_radix;
+}
+
+int Torus::incrementSymbol(int symbol){
+
+/*
+ * We could also add m_radix only if symbol-1 < 0, but out-of-order execution is
+ * faster without branching.
+ */
+
+ return (symbol+1)%m_radix;
+}
+
+int Torus::decrementSymbol(int symbol){
+
+/*
+ * We could also add m_radix only if symbol-1 < 0, but out-of-order execution is
+ * faster without branching.
+ */
+ return (symbol-1+m_radix)%m_radix;
+}
+
+/**
+ * TODO: move this upstream
+ */
+bool Torus::isConnected(Rank source,Rank destination){
+ if(source==destination)
+ return true;
+
+ return m_outcomingConnections[source].count(destination)==1;
+}
+
+int Torus::getMinimumDistance(int a,int b){
+
+ int left=getDistance(a,b);
+ int right=getDistance(b,a);
+
+ if(left<right)
+ return left;
+
+ return right;
+}
+
+/**
+ * Just verify the edge property
+ * Also, we allow any vertex to communicate with itself
+ * regardless of the property.
+ */
+bool Torus::computeConnection(Rank source,Rank destination){
+
+/*
+ * Not connected: 3,4 (23) -> 3,2 (13) Load: 150
+ *
+ */
+ if(source==destination) // no difference
+ return false;
+
+ Tuple*sourceVertex=&(m_graphToVertex[source]);
+ Tuple*destinationVertex=&(m_graphToVertex[destination]);
+
+ int differencesOf1=0;
+ int same=0;
+
+/*
+ * Count the number if dimensions that differ by only 1.
+ */
+ for(int i=0;i<m_dimension;i++){
+
+ int sourceSymbol=sourceVertex->m_digits[i];
+ int destinationSymbol=destinationVertex->m_digits[i];
+
+ if(sourceSymbol==destinationSymbol){
+ same++;
+ continue;
+ }
+
+ if(incrementSymbol(sourceSymbol)==destinationSymbol || decrementSymbol(sourceSymbol)==destinationSymbol)
+ differencesOf1++;
+ }
+
+ bool connected=false;
+
+ if((differencesOf1+same) == m_dimension && differencesOf1==1)
+ connected=true;
+
+#if 0
+ if(connected){
+ cout<<"[DEVEL] ";
+ printVertex(sourceVertex);
+ cout<<" ("<<source<<")";
+ cout<<" and ";
+ printVertex(destinationVertex);
+ cout<<" ("<<destination<<")";
+ cout<<" are connected."<<endl;
+ }
+#endif
+
+ return connected;
+}
+
+/**
+ * TODO: move this upstream, instead, add a protected m_valid field.
+ */
+bool Torus::isValid(int vertices){
+ configureGraph(vertices);
+
+ return m_radix!=__INVALID_ALPHABET_SIZE;
+}
+
+/**
+ * TODO: move this upstream
+ */
+void Torus::setDegree(int degree){
+ m_degree=degree;
+}
+
+void Torus::printStatus(Rank rank){
+
+ cout<<"[Torus] Load values:"<<endl;
+ cout<<"Radix: "<<m_radix<<endl;
+ cout<<"Dimension: "<<m_dimension<<endl;
+ cout<<"Self: ";
+ Tuple*self=&(m_graphToVertex[rank]);
+
+ printVertex(self);
+ cout<<endl;
+
+ for(int position=0;position<m_dimension;position++){
+
+ int selfSymbol=self->m_digits[position];
+
+ for(int otherSymbol=0;otherSymbol<m_radix;otherSymbol++){
+
+ if(selfSymbol==otherSymbol)// nothing to report
+ continue;
+
+ uint64_t load=getLoad(position,otherSymbol);
+
+ if(load==0)
+ continue;
+
+ Tuple copy=*self;
+ copy.m_digits[position]=otherSymbol;
+ cout<<" ";
+ printVertex(self);
+ cout<<" ("<<rank<<")";
+ cout<<" -> ";
+ printVertex(©);
+
+ Rank other=convertToBase10(©);
+ cout<<" ("<<other<<")";
+ cout<<" Load: "<<load<<endl;
+
+ #ifdef ASSERT
+ if(computeConnection(rank,other)!=true){
+ cout<<"Error: position="<<position<<" otherSymbol= "<<otherSymbol<<", load="<<load<<" but the edge does not exist."<<endl;
+ }
+ assert(computeConnection(rank,other) == true);
+ #endif
+ }
+ }
+}
+
+void Torus::start(){
+ for(int i=0;i<m_dimension;i++)
+ for(int j=0;j<m_radix;j++)
+ setLoad(i,j,0);
+
+ #ifdef ASSERT
+
+ for(int i=0;i<m_dimension;i++)
+ for(int j=0;j<m_radix;j++)
+ assert(getLoad(i,j)==0);
+ #endif
+}
+
+
diff --git a/RayPlatform/routing/Hypercube.h b/RayPlatform/RayPlatform/routing/Torus.h
similarity index 58%
rename from RayPlatform/routing/Hypercube.h
rename to RayPlatform/RayPlatform/routing/Torus.h
index cbffeba..f7710f2 100644
--- a/RayPlatform/routing/Hypercube.h
+++ b/RayPlatform/RayPlatform/routing/Torus.h
@@ -1,8 +1,8 @@
/*
- RayPlatform
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,48 +19,51 @@
*/
-#ifndef _Hypercube_h
-#define _Hypercube_h
+#ifndef _Torus_h
+#define _Torus_h
+
+#include "GraphImplementation.h"
-#include <routing/GraphImplementation.h>
#include <vector>
#include <stdint.h>
using namespace std;
-// for alphabetSize = 2 and 4096 vertices
-// we have 2^12 = 4096
-// each vertex has (2-1)*10 = 10 edges
-// in that case, the maximum index will be 12*2+11=35.
-//
-// for alphabetSize = 32
-// we have 32^2 =1024
-//
-// each vertex has (32-1)*2 = 62 edges
-// in that case, the maximum index will be 2*32+31=95
-//
#define __NUMBER_OF_STORED_LOAD_VALUES 256
/**
- * Hypercube
- * n must be a power of 2 to be a hypercube.
- * Otherwise, it is an other polytope.
- * A hypercube is a convex regular polytope.
- * This class in fact implements a generalized hypercube,
- * usually called a convex regular polytope.
+ * Torus, also known as k-ary n-cube
+ *
+ * Parameters:
+ *
+ * - dimension
+ * - radix (number of vertices per dimension)
+ * - vertices = radix ^ dimension
+ * - degree =:
+ *
+ * \see http://www.doc.ic.ac.uk/~phjk/AdvancedCompArchitecture/2001-02/Lectures.old/Ch06/node23.html
+ * \see http://www.doc.ic.ac.uk/~phjk/AdvancedCompArchitecture/2001-02/Lectures.old/Ch06/node24.html
+ * \see en.wikipedia.org/wiki/Torus#n-dimensional_torus
+ * \see http://en.wikipedia.org/wiki/Hypercube
+ *
* \author Sébastien Boisvert
*/
-class Hypercube : public GraphImplementation{
+class Torus : public GraphImplementation{
+/**
+ * Load values.
+ */
uint64_t m_loadValues[__NUMBER_OF_STORED_LOAD_VALUES];
- /** the user-provided degree */
+/**
+ * The user-provided degree.
+ */
int m_degree;
vector<Tuple> m_graphToVertex;
- int m_alphabetSize;
+ int m_radix;
- int m_wordLength;
+ int m_dimension;
void configureGraph(int n);
@@ -75,33 +78,31 @@ class Hypercube : public GraphImplementation{
void printVertex(Tuple*a);
bool isAPowerOf(int n,int base);
- int getMaximumOverlap(Tuple*a,Tuple*b);
-
- Rank computeNextRankInRoute(Rank source,Rank destination,Rank rank);
-
-
- Rank computeNextRankInRouteWithRoundRobin(Rank source,Rank destination,Rank rank);
- int m_currentPosition;
- int m_currentSymbol;
bool computeConnection(Rank source,Rank destination);
-
uint64_t getLoad(int position,int symbol);
void setLoad(int position,int symbol,uint64_t value);
+ int incrementSymbol(int symbol);
+ int decrementSymbol(int symbol);
+ int getDistance(int end,int start);
+ int getMinimumDistance(int a,int b);
+
+ uint64_t* getLoadValueCell(int position,int symbol);
protected:
void computeRoute(Rank a,Rank b,vector<Rank>*route);
Rank getNextRankInRoute(Rank source,Rank destination,Rank rank);
bool isConnected(Rank source,Rank destination);
+
public:
- void makeConnections(int n);
+ void makeConnections(int vertices);
void makeRoutes();
void setDegree(int degree);
- bool isValid(int n);
+ bool isValid(int vertices);
void printStatus(Rank rank);
void start();
};
diff --git a/RayPlatform/scheduling/SwitchMan.cpp b/RayPlatform/RayPlatform/scheduling/SwitchMan.cpp
similarity index 91%
rename from RayPlatform/scheduling/SwitchMan.cpp
rename to RayPlatform/RayPlatform/scheduling/SwitchMan.cpp
index 9b4dfdd..30ec265 100644
--- a/RayPlatform/scheduling/SwitchMan.cpp
+++ b/RayPlatform/RayPlatform/scheduling/SwitchMan.cpp
@@ -1,8 +1,8 @@
/*
- Ray
+ RayPlatform: a message-passing development framework
Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,12 +18,14 @@
see <http://www.gnu.org/licenses/>
*/
-#include <scheduling/SwitchMan.h>
+#include "SwitchMan.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/core/ComputeCore.h>
+
+#include <iostream>
#include <assert.h>
#include <vector>
-#include <core/OperatingSystem.h>
-#include <iostream>
-#include <core/ComputeCore.h>
using namespace std;
//#define CONFIG_SWITCHMAN_VERBOSITY
@@ -33,8 +35,13 @@ __CreatePlugin(SwitchMan);
__CreateMessageTagAdapter(SwitchMan,RAY_MPI_TAG_SWITCHMAN_COMPLETION_SIGNAL);
__CreateSlaveModeAdapter(SwitchMan,RAY_SLAVE_MODE_STOP);
-
void SwitchMan::constructor(Rank rank,int numberOfCores){
+
+ #ifdef ASSERT
+ assert(rank>=0);
+ assert(numberOfCores>=0);
+ #endif
+
m_rank=rank;
m_size=numberOfCores;
reset();
@@ -75,6 +82,10 @@ void SwitchMan::closeSlaveMode(Rank source){
}
void SwitchMan::runAssertions(){
+
+ if(m_counter>m_size)
+ cout<<"[SwitchMan] counter is "<<m_counter<<" but size is only "<<m_size<<endl;
+
assert(m_counter<=m_size);
assert(m_counter>=0);
}
@@ -105,7 +116,8 @@ void SwitchMan::addNextMasterMode(MasterMode a,MasterMode b){
void SwitchMan::openSlaveMode(MessageTag tag,StaticVector*outbox,Rank source,Rank destination){
#ifdef CONFIG_SWITCHMAN_VERBOSITY
- cout<<"[SwitchMan::openSlaveMode] Opening remotely slave mode on rank "<<destination<<endl;
+ cout<<"[SwitchMan::openSlaveMode] Opening remotely slave mode on rank "<<destination;
+ cout << " message on bus: " << MESSAGE_TAGS[tag] <<endl;
#endif
#ifdef ASSERT
@@ -142,7 +154,7 @@ void SwitchMan::openMasterMode(StaticVector*outbox,Rank source){
MessageTag tag=m_masterModeToTagTable[m_masterMode];
#ifdef CONFIG_SWITCHMAN_VERBOSITY
- cout<<"[SwitchMan::openMasterMode] Opening master mode on rank "<<source<<endl;
+ cout<<"[SwitchMan::openMasterMode] Opening master mode " << MASTER_MODES[m_masterMode] << " on rank "<<source<<endl;
cout<<"[SwitchMan::openMasterMode] tag= "<<MESSAGE_TAGS[tag]<<" source= "<<source<<" Outbox= "<<outbox<<endl;
#endif
@@ -196,7 +208,7 @@ void SwitchMan::sendEmptyMessage(StaticVector*outbox,Rank source,Rank destinatio
void SwitchMan::sendMessage(MessageUnit*buffer,int count,StaticVector*outbox,Rank source,Rank destination,MessageTag tag){
// send a message
Message aMessage(buffer,count,destination,tag,source);
- outbox->push_back(aMessage);
+ outbox->push_back(&aMessage);
}
void SwitchMan::sendToAll(StaticVector*outbox,Rank source,MessageTag tag){
@@ -247,7 +259,7 @@ int*SwitchMan::getSlaveModePointer(){
void SwitchMan::setSlaveMode(SlaveMode mode){
#ifdef CONFIG_SWITCHMAN_VERBOSITY
- cout<<"setSlaveMode "<<SLAVE_MODES[mode]<<endl;
+ cout<<"[SwitchMan::setSlaveMode] "<<SLAVE_MODES[mode]<<endl;
#endif
m_slaveMode=mode;
@@ -260,7 +272,7 @@ MasterMode SwitchMan::getMasterMode(){
void SwitchMan::setMasterMode(MasterMode mode){
#ifdef CONFIG_SWITCHMAN_VERBOSITY
- cout<<"setMasterMode "<<MASTER_MODES[mode]<<endl;
+ cout<<"[SwitchMan::setMasterMode] "<<MASTER_MODES[mode]<<endl;
#endif
m_masterMode=mode;
@@ -339,6 +351,9 @@ void SwitchMan::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_SWITCHMAN_COMPLETION_SIGNAL=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_SWITCHMAN_COMPLETION_SIGNAL");
__BindPlugin(SwitchMan);
+
+ __BindAdapter(SwitchMan,RAY_MPI_TAG_SWITCHMAN_COMPLETION_SIGNAL);
+ __BindAdapter(SwitchMan,RAY_SLAVE_MODE_STOP);
}
diff --git a/RayPlatform/scheduling/SwitchMan.h b/RayPlatform/RayPlatform/scheduling/SwitchMan.h
similarity index 86%
rename from RayPlatform/scheduling/SwitchMan.h
rename to RayPlatform/RayPlatform/scheduling/SwitchMan.h
index e739e1c..1938872 100644
--- a/RayPlatform/scheduling/SwitchMan.h
+++ b/RayPlatform/RayPlatform/scheduling/SwitchMan.h
@@ -1,8 +1,8 @@
/*
- Ray
+ RayPlatform: a message-passing development framework
Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,14 +21,14 @@
#ifndef _SwitchMan_H
#define _SwitchMan_H
-#include <core/master_modes.h>
-#include <structures/StaticVector.h>
-#include <communication/Message.h>
-#include <core/slave_modes.h>
-#include <core/master_modes.h>
-#include <plugins/CorePlugin.h>
-#include <handlers/MessageTagHandler.h>
-#include <handlers/SlaveModeHandler.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/core/slave_modes.h>
+#include <RayPlatform/core/master_modes.h>
+#include <RayPlatform/core/master_modes.h>
+#include <RayPlatform/plugins/CorePlugin.h>
+#include <RayPlatform/handlers/MessageTagHandler.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/structures/StaticVector.h>
#include <map>
#include <vector>
@@ -37,6 +37,9 @@ using namespace std;
class ComputeCore;
class SwitchMan;
+__DeclareMessageTagAdapter(SwitchMan,RAY_MPI_TAG_SWITCHMAN_COMPLETION_SIGNAL);
+__DeclareSlaveModeAdapter(SwitchMan,RAY_SLAVE_MODE_STOP);
+
/**
* the switchman controls the workflow on all ranks
* he is the one who decides when something must be done.
@@ -45,6 +48,9 @@ class SwitchMan;
*/
class SwitchMan: public CorePlugin {
+ __AddAdapter(SwitchMan,RAY_MPI_TAG_SWITCHMAN_COMPLETION_SIGNAL);
+ __AddAdapter(SwitchMan,RAY_SLAVE_MODE_STOP);
+
ComputeCore*m_core;
MessageTag RAY_MPI_TAG_SWITCHMAN_COMPLETION_SIGNAL;
@@ -175,4 +181,6 @@ public:
void call_RAY_SLAVE_MODE_STOP();
};
+
+
#endif
diff --git a/RayPlatform/scheduling/TaskCreator.cpp b/RayPlatform/RayPlatform/scheduling/TaskCreator.cpp
similarity index 92%
rename from RayPlatform/scheduling/TaskCreator.cpp
rename to RayPlatform/RayPlatform/scheduling/TaskCreator.cpp
index 923f350..a345489 100644
--- a/RayPlatform/scheduling/TaskCreator.cpp
+++ b/RayPlatform/RayPlatform/scheduling/TaskCreator.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,7 +18,8 @@
see <http://www.gnu.org/licenses/>
*/
-#include <scheduling/TaskCreator.h>
+#include "TaskCreator.h"
+
#include <stdlib.h> /* for NULL */
#ifdef ASSERT
diff --git a/RayPlatform/scheduling/TaskCreator.h b/RayPlatform/RayPlatform/scheduling/TaskCreator.h
similarity index 83%
rename from RayPlatform/scheduling/TaskCreator.h
rename to RayPlatform/RayPlatform/scheduling/TaskCreator.h
index 469343c..30a63c4 100644
--- a/RayPlatform/scheduling/TaskCreator.h
+++ b/RayPlatform/RayPlatform/scheduling/TaskCreator.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,9 +21,9 @@
#ifndef _TaskCreator_H
#define _TaskCreator_H
-#include <scheduling/Worker.h>
-#include <scheduling/VirtualProcessor.h>
-#include <memory/MyAllocator.h>
+#include <RayPlatform/scheduling/Worker.h>
+#include <RayPlatform/scheduling/VirtualProcessor.h>
+#include <RayPlatform/memory/MyAllocator.h>
/** this is an interface that should be implemented by child classes
* m_initialized must be set to false in the child class
diff --git a/RayPlatform/scheduling/VirtualProcessor.cpp b/RayPlatform/RayPlatform/scheduling/VirtualProcessor.cpp
similarity index 93%
rename from RayPlatform/scheduling/VirtualProcessor.cpp
rename to RayPlatform/RayPlatform/scheduling/VirtualProcessor.cpp
index 454e3c9..5201820 100644
--- a/RayPlatform/scheduling/VirtualProcessor.cpp
+++ b/RayPlatform/RayPlatform/scheduling/VirtualProcessor.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -18,8 +18,9 @@
see <http://www.gnu.org/licenses/>
*/
-#include <scheduling/VirtualProcessor.h>
-#include <core/ComputeCore.h>
+#include "VirtualProcessor.h"
+
+#include <RayPlatform/core/ComputeCore.h>
#include <iostream>
using namespace std;
@@ -162,6 +163,7 @@ bool VirtualProcessor::run(){
}else{
updateStates();
+#if 0
// add one worker to active workers
// reason is that those already in the pool don't communicate anymore --
// as for they need responses.
@@ -189,6 +191,12 @@ bool VirtualProcessor::run(){
m_virtualCommunicator->forceFlush();
}
}
+#endif
+
+ // always flush a message every time we looped over every actors
+ // if no message was flushed.
+ if(!m_virtualCommunicator->getGlobalPushedMessageStatus())
+ m_virtualCommunicator->forceFlush();
m_activeWorkerIterator=m_activeWorkers.begin();
diff --git a/RayPlatform/scheduling/VirtualProcessor.h b/RayPlatform/RayPlatform/scheduling/VirtualProcessor.h
similarity index 91%
rename from RayPlatform/scheduling/VirtualProcessor.h
rename to RayPlatform/RayPlatform/scheduling/VirtualProcessor.h
index 178af7e..3075671 100644
--- a/RayPlatform/scheduling/VirtualProcessor.h
+++ b/RayPlatform/RayPlatform/scheduling/VirtualProcessor.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,10 +21,9 @@
#ifndef _VirtualProcessor_h
#define _VirtualProcessor_h
-#include <structures/StaticVector.h>
-#include <memory/RingAllocator.h>
-#include <scheduling/Worker.h>
-
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/scheduling/Worker.h>
+#include <RayPlatform/structures/StaticVector.h>
#include <set>
#include <vector>
diff --git a/RayPlatform/scheduling/Worker.h b/RayPlatform/RayPlatform/scheduling/Worker.h
similarity index 80%
rename from RayPlatform/scheduling/Worker.h
rename to RayPlatform/RayPlatform/scheduling/Worker.h
index 322586c..abdfcec 100644
--- a/RayPlatform/scheduling/Worker.h
+++ b/RayPlatform/RayPlatform/scheduling/Worker.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,9 +21,10 @@
#ifndef _Worker_h
#define _Worker_h
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+
#include <stdint.h>
-#include <memory/RingAllocator.h>
-#include <communication/VirtualCommunicator.h>
/** a general worker class
* \author Sébastien Boisvert
diff --git a/RayPlatform/RayPlatform/store/CarriageableItem.h b/RayPlatform/RayPlatform/store/CarriageableItem.h
new file mode 100644
index 0000000..ec7f598
--- /dev/null
+++ b/RayPlatform/RayPlatform/store/CarriageableItem.h
@@ -0,0 +1,61 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#ifndef CarriageableItemHeader
+#define CarriageableItemHeader
+
+#include <stdint.h>
+
+/**
+ * Interface to carry an item across a media.
+ *
+ *
+ * TODO: add a dependency on the message size.
+ * TODO add the concept of part numbers
+ *
+ * \author Sébastien Boisvert
+ *
+ */
+class CarriageableItem {
+
+public:
+
+ /**
+ * load the object from a stream
+ *
+ * \returns bytes loaded
+ */
+ virtual int load(const char * buffer) = 0;
+
+ /**
+ *
+ * dump the object in a stream
+ *
+ * \returns bytes written
+ */
+ virtual int dump(char * buffer) const = 0;
+
+ virtual int getRequiredNumberOfBytes() const = 0;
+
+ virtual ~CarriageableItem() {}
+};
+
+#endif
diff --git a/RayPlatform/RayPlatform/store/KeyValueStore.cpp b/RayPlatform/RayPlatform/store/KeyValueStore.cpp
new file mode 100644
index 0000000..b20c50f
--- /dev/null
+++ b/RayPlatform/RayPlatform/store/KeyValueStore.cpp
@@ -0,0 +1,604 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+
+#include "KeyValueStore.h"
+
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/cryptography/crypto.h>
+
+#include <string.h> /* for memcpy */
+
+#ifdef CONFIG_ASSERT
+#include <assert.h>
+#endif
+
+//#define KeyValueStore_DEBUG_NOW "Yes"
+
+__CreatePlugin(KeyValueStore);
+
+__CreateMessageTagAdapter(KeyValueStore, RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART);
+__CreateMessageTagAdapter(KeyValueStore, RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART_REPLY);
+
+KeyValueStore::KeyValueStore() {
+
+ clear();
+}
+
+
+void KeyValueStore::pullRemoteKey(const string & key, const Rank & rank, KeyValueStoreRequest & request) {
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "[DEBUG] pullRemoteKey Request: key= " << key << " rank= " << rank << endl;
+#endif
+
+ request.initialize(key, rank);
+ request.setTypeToPullRequest();
+
+}
+
+void KeyValueStore::initialize(Rank rank, int size, RingAllocator * outboxAllocator, StaticVector * inbox, StaticVector * outbox) {
+
+ m_rank = rank;
+ m_size = size;
+ m_outboxAllocator = outboxAllocator;
+ m_inbox = inbox;
+ m_outbox = outbox;
+
+ int chunkSize = 33554432; // 32 MiB 4194304 is 4 MiB
+ m_memoryAllocator.constructor(chunkSize, "/allocator/KeyValueStore.DRAM", false);
+}
+
+bool KeyValueStore::getStringKey(const char * key, int keyLength, string & keyObject) {
+
+ #define MAXIMUM_KEY_LENGTH 256
+ char buffer[MAXIMUM_KEY_LENGTH];
+
+#ifdef CONFIG_ASSERT
+ assert(keyLength + 1 <= MAXIMUM_KEY_LENGTH);
+#endif
+
+ if(keyLength + 1 > MAXIMUM_KEY_LENGTH)
+ return false;
+
+ memcpy(buffer, key, keyLength);
+ buffer[keyLength] = '\0';
+
+ keyObject = buffer;
+
+ return true;
+}
+
+bool KeyValueStore::insertLocalKey(const string & key, char * value, int valueLength) {
+
+ return insertLocalKeyWithLength(key.c_str(), key.length(), value, valueLength);
+}
+
+/**
+ * TODO: remove limit on key length
+ */
+bool KeyValueStore::insertLocalKeyWithLength(const char * key, int keyLength, char * value, int valueLength) {
+
+ string keyObject = "";
+
+ if(!getStringKey(key, keyLength, keyObject))
+ return false;
+
+ if(m_items.count(keyObject) != 0)
+ return false;
+
+ KeyValueStoreItem item(value, valueLength);
+ m_items[keyObject] = item;
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "[DEBUG] KeyValueStore registered key " << keyObject << " (length: " << keyLength;
+ cout << ")";
+ cout << " with value (length: " << valueLength << ")";
+ cout << endl;
+#endif /* KeyValueStore_DEBUG_NOW */
+
+ return true;
+}
+
+bool KeyValueStore::removeLocalKey(const string & key) {
+
+ string keyObject = key;
+
+ // nothing to remove
+ if(m_items.count(keyObject) == 0)
+ return true;
+
+ int size = m_items[keyObject].getValueLength();
+ char * content = m_items[keyObject].getValue();
+
+ m_memoryAllocator.free(content, size);
+
+ m_items.erase(keyObject);
+
+#ifdef CONFIG_ASSERT
+ assert(m_items.count(keyObject) == 0);
+#endif ////// CONFIG_ASSERT
+
+ return true;
+}
+
+bool KeyValueStore::getLocalKey(const string & key, char * & value, int & valueLength) {
+
+ return getLocalKeyWithLength(key.c_str(), key.length(), &value, &valueLength);
+}
+
+bool KeyValueStore::getLocalKeyWithLength(const char * key, int keyLength, char ** value, int * valueLength) {
+
+ string keyObject;
+ if(!getStringKey(key, keyLength, keyObject))
+ return false;
+
+ // nothing to remove
+ if(m_items.count(keyObject) == 0)
+ return false;
+
+ KeyValueStoreItem & item = m_items[keyObject];
+
+
+ if(!item.isItemReady())
+ return false;
+
+ (*value) = item.getValue();
+ (*valueLength) = item.getValueLength();
+
+ return true;
+}
+
+bool KeyValueStore::pushLocalKeyWithLength(const char * key, int keyLength, Rank destination) {
+
+ // not implemented.
+
+ return false;
+}
+
+KeyValueStoreItem * KeyValueStore::getLocalItemFromKey(const char * key, int keyLength) {
+
+ string keyObject;
+ if(!getStringKey(key, keyLength, keyObject))
+ return NULL;
+
+ if(m_items.count(keyObject) == 0)
+ return NULL;
+
+ return &(m_items[keyObject]);
+}
+
+/**
+ * This method sends RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART to another rank.
+ * The chunk is received in a message RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART_REPLY.
+ *
+ * If the new offset is lower than the value length, then the transfer is not complete.
+ */
+bool KeyValueStore::pullRemoteKeyWithLength(const char * key, int keyLength, Rank source) {
+
+ KeyValueStoreItem * item = getLocalItemFromKey(key, keyLength);
+
+ if(item == NULL) {
+
+ // insert an item with an empty value
+ insertLocalKeyWithLength(key, keyLength, NULL, 0);
+
+ item = getLocalItemFromKey(key, keyLength);
+
+ // this will mark the key as not ready.
+ item->startDownload();
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "[DEBUG] KeyValueStore marked key " << key << " for download." << endl;
+#endif
+
+ return false;
+ }
+
+ // the transfer has completed.
+ if(item != NULL && item->isItemReady()){
+
+#if 0
+ cout << "[DEBUG] item is ready to be used." << endl;
+#endif
+
+ return true;
+
+ } else if(item != NULL && !item->messageWasSent()) {
+
+ char * buffer=(char *) m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+
+ uint32_t valueSize = item->getValueLength();
+ uint32_t offset = item->getOffset();
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "[DEBUG] KeyValueStore::pullRemoteKeyWithLength key= " << key << " valueSize= " << valueSize;
+ cout << " offset= " << offset;
+ cout << endl;
+#endif /* KeyValueStore_DEBUG_NOW */
+
+ int outputPosition = 0;
+
+ outputPosition += dumpMessageHeader(key, valueSize, offset, buffer);
+
+ int units = outputPosition / sizeof(MessageUnit);
+ if(outputPosition % sizeof(MessageUnit))
+ units ++;
+
+ Message aMessage((MessageUnit*)buffer, units, source,
+ RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART, m_rank);
+
+ m_outbox->push_back(&aMessage);
+
+ // this will mark the item as being being currently fetched.
+ item->sendMessage();
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "[DEBUG] message RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART sent to " << source;
+ cout << endl;
+#endif
+ return false;
+ }
+
+ return false;
+}
+
+void KeyValueStore::clear() {
+ m_rank = 0;
+ m_size = 0;
+ m_outboxAllocator = NULL;
+ m_inbox = NULL;
+ m_outbox = NULL;
+
+ m_items.clear();
+
+ // free used memory, but don't give it back to the
+ // operatign system
+ m_memoryAllocator.reset();
+}
+
+void KeyValueStore::destroy() {
+ clear();
+
+ // give back the memory to the operating system
+ m_memoryAllocator.clear();
+}
+
+char * KeyValueStore::allocateMemory(int bytes) {
+ char * address = (char*)m_memoryAllocator.allocate(bytes);
+
+ return address;
+}
+
+int KeyValueStore::loadMessageHeader(string & key, uint32_t & valueSize, uint32_t & offset, const char * buffer) const {
+
+ // read the message header
+ key = buffer;
+
+ int inputPosition = 0;
+ int size = key.length();
+ inputPosition += size;
+ inputPosition ++; // for the \0
+
+ size = sizeof(uint32_t);
+ memcpy(&valueSize, buffer + inputPosition, size);
+ inputPosition += size;
+
+ memcpy(&offset, buffer + inputPosition, size);
+ inputPosition += size;
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "[DEBUG] loadMessageHeader key= " << key << " valueSize= " << valueSize;
+ cout << " offset= " << offset << " headerSize= " << inputPosition << endl;
+#endif
+
+ return inputPosition;
+}
+
+int KeyValueStore::dumpMessageHeader(const char* key, uint32_t valueLength, uint32_t offset, char * buffer) const {
+
+ int outputPosition = 0;
+
+ int size = strlen(key);
+
+ memcpy(buffer + outputPosition, key, size);
+ outputPosition += size;
+ buffer[outputPosition++] = '\0';
+
+ size = sizeof(uint32_t);
+
+ memcpy(buffer + outputPosition, &valueLength, size);
+ outputPosition += size;
+ memcpy(buffer + outputPosition, &offset, size);
+ outputPosition += size;
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "[DEBUG] dumpMessageHeader key= " << key << " valueLength= " << valueLength;
+ cout << " offset= " << offset << " HeaderSize= " << outputPosition;
+ cout << endl;
+#endif
+
+
+
+ return outputPosition;
+}
+
+/*
+ * The header contains:
+ *
+ * * the key, strlen(buffer) is the length of this key
+ * * the length of the value (0 if the information is not known by the sender) uint32_t
+ * * the offset (uint32_t) at buffer + length of key
+ *
+ * TODO: we need something to indicate that the object does not exist
+ * here. add this in the header...
+ */
+void KeyValueStore::call_RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART(Message * message) {
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "***" << endl;
+ cout << "call_RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART" << endl;
+#endif
+
+ char * buffer = (char*)message->getBuffer();
+
+#ifdef CONFIG_ASSERT
+ assert(buffer != NULL);
+#endif /////// CONFIG_ASSERT
+
+ // load the message header
+ string key = "";
+ uint32_t valueLength = 0;
+ uint32_t offset = 0;
+
+ int position = 0;
+ position += loadMessageHeader(key, valueLength, offset, buffer + position);
+
+ char * responseBuffer = (char *) m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+
+ char * value = NULL;
+ int actualValueLength = 0;
+
+ // if the key is inside, we respond with
+ if(m_items.count(key) > 0) {
+
+ KeyValueStoreItem * item = getLocalItemFromKey(key.c_str(), key.length());
+ value = item->getValue();
+ actualValueLength = item->getValueLength();
+ }
+
+ // TODO: if object does not exist, a message will be returned regardless
+ // with size 0, which should not be the case.
+
+
+ // we need a header even when the key is not found
+ // in that case, we assign a value length of 0 in the response.
+ //
+ int outputPosition = 0;
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "[DEBUG] actualValueLength= " << actualValueLength << endl;
+#endif
+
+ outputPosition += dumpMessageHeader(key.c_str(), actualValueLength, offset, responseBuffer);
+
+ if(value != NULL) {
+
+ int bytesToCopy = actualValueLength - offset;
+
+ int remainingBytes = MAXIMUM_MESSAGE_SIZE_IN_BYTES - outputPosition;
+
+ if(remainingBytes < bytesToCopy)
+ bytesToCopy = remainingBytes;
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "[DEBUG] KeyValueStore::call_RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART key= " << key << " offset= " << offset;
+ cout << " actualValueLength= " << actualValueLength;
+ cout << " bytesToCopy= " << bytesToCopy;
+ cout << " source= " << message->getSource();
+ cout << endl;
+#endif /* KeyValueStore_DEBUG_NOW */
+
+ int size = bytesToCopy;
+ memcpy(responseBuffer + outputPosition, value + offset, size);
+ outputPosition += size;
+ }
+
+#ifdef CONFIG_ASSERT
+ assert(outputPosition <= MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+#endif
+
+ int units = outputPosition / sizeof(MessageUnit);
+ if(outputPosition % sizeof(MessageUnit))
+ units ++;
+
+ Message aMessage((MessageUnit*) responseBuffer, units, message->getSource(),
+ RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART_REPLY, message->getDestination());
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "DEBUG reply RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART_REPLY " << units << " units ";
+ cout << outputPosition << " bytes" << endl;
+#endif
+
+ // and hop we go, the message is now on the message bus
+ // let's hope that the message reaches its destination safely.
+
+ m_outbox->push_back(&aMessage);
+}
+
+/*
+ * The header contains:
+ *
+ * * the key, strlen(buffer) is the length of this key
+ * * the length of the value (0 if the information is not known by the sender) uint32_t
+ * * the offset (uint32_t) at buffer + length of key
+ *
+ * The rest of the buffer contains bytes.
+ *
+ */
+void KeyValueStore::call_RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART_REPLY(Message * message) {
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "***" << endl;
+ cout << "[DEBUG] received call_RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART_REPLY" << endl;
+ cout << " units " << message->getCount() << endl;
+#endif
+
+ char * buffer = (char*)message->getBuffer();
+
+#ifdef CONFIG_ASSERT
+ assert(buffer != NULL);
+#endif /////// CONFIG_ASSERT
+
+
+ string key = "";
+ uint32_t valueLength = 0;
+ uint32_t offset = 0;
+
+ int inputPosition = 0;
+
+ inputPosition += loadMessageHeader(key, valueLength, offset, buffer);
+
+ KeyValueStoreItem * item = getLocalItemFromKey(key.c_str(), key.length());
+
+#ifdef CONFIG_ASSERT
+ assert(item != NULL);
+#endif
+
+ // if the offset is 0
+ if(offset == 0) {
+
+#ifdef CONFIG_ASSERT
+ assert(item->getValue() == NULL);
+ assert(item->getValueLength() == 0);
+#endif
+
+ if(valueLength != 0) {
+
+ char * value = allocateMemory(valueLength);
+ item->setValue(value);
+ item->setValueLength(valueLength);
+ }
+
+ // it is 0 bytes, the meta-data of the object
+ // are already registered correctly
+ //
+ // This was done when the first message of type
+ // RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART
+ // was sent.
+ }
+
+ // TODO check if the object is invalid (does not exist on remote host)
+
+ // if we have 0 bytes to copy, we have nothing left to do.
+ if(item->getValueLength() == 0) {
+ item->receiveMessage();
+ return;
+ }
+
+ // get the local buffer copy
+
+ char * value = item->getValue();
+
+#ifdef CONFIG_ASSERT
+ int dummySize = item->getValueLength();
+
+ assert(value != NULL);
+ assert(dummySize == (int)valueLength);
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "DEBUG dummySize= " << dummySize << endl;
+#endif
+#endif
+
+ int bytesToCopy = valueLength - offset;
+
+ int remainingBytes = MAXIMUM_MESSAGE_SIZE_IN_BYTES - inputPosition;
+
+ if(remainingBytes < bytesToCopy)
+ bytesToCopy = remainingBytes;
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "[DEBUG] memcpy destination= " << offset;
+ cout << " inputPosition= " << inputPosition << " bytesToCopy= " << bytesToCopy << endl;
+#endif
+
+ // copy into the buffer what we received from
+ // a remote rank
+ memcpy(value + offset, buffer + inputPosition, bytesToCopy);
+
+#ifdef CONFIG_ASSERT
+ assert((int)(offset + bytesToCopy) <= (int)item->getValueLength());
+#endif
+
+ uint32_t newOffset = offset + bytesToCopy;
+
+ item->setDownloadedSize(newOffset);
+
+#ifdef KeyValueStore_DEBUG_NOW
+ cout << "[DEBUG] setDownloadedSize " << newOffset << endl;
+#endif
+
+ // update data for next message
+ item->receiveMessage();
+
+ // the progression will occur with a call to pullRemoteKey by the end-user
+ // code.
+}
+
+bool KeyValueStore::test(KeyValueStoreRequest & request) {
+
+ const string & key = request.getKey();
+ const Rank & rank = request.getRank();
+
+#if 0
+ cout << "[DEBUG] test Request: key= " << key << " rank= " << rank << endl;
+#endif
+
+ if(request.isAPullRequest()) {
+
+ if(pullRemoteKeyWithLength(key.c_str(), key.length(), rank)) {
+
+ return true;
+ }
+
+ }
+
+ return false;
+}
+
+void KeyValueStore::registerPlugin(ComputeCore * core){
+
+ m_core=core;
+ m_plugin=core->allocatePluginHandle();
+
+ core->setPluginName(m_plugin, "KeyValueStore");
+ core->setPluginDescription(m_plugin, "A key-value store for transporting items between processes");
+ core->setPluginAuthors(m_plugin,"Sébastien Boisvert");
+ core->setPluginLicense(m_plugin,"GNU Lesser General License version 3");
+
+ __ConfigureMessageTagHandler(KeyValueStore, RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART);
+ __ConfigureMessageTagHandler(KeyValueStore, RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART_REPLY);
+}
+
+void KeyValueStore::resolveSymbols(ComputeCore*core){
+
+}
diff --git a/RayPlatform/RayPlatform/store/KeyValueStore.h b/RayPlatform/RayPlatform/store/KeyValueStore.h
new file mode 100644
index 0000000..d803f0b
--- /dev/null
+++ b/RayPlatform/RayPlatform/store/KeyValueStore.h
@@ -0,0 +1,177 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#ifndef KeyValueStoreHeader
+#define KeyValueStoreHeader
+
+#include "KeyValueStoreItem.h"
+#include "KeyValueStoreRequest.h"
+
+#include <RayPlatform/core/types.h>
+#include <RayPlatform/plugins/CorePlugin.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/memory/MyAllocator.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/handlers/MessageTagHandler.h>
+
+#include <map>
+#include <string>
+using namespace std;
+
+#include <stdint.h>
+
+__DeclarePlugin(KeyValueStore);
+
+__DeclareMessageTagAdapter(KeyValueStore, RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART);
+__DeclareMessageTagAdapter(KeyValueStore, RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART_REPLY);
+
+/**
+ *
+ * This is an initial design for a MPI key-value store in
+ * RayPlatform.
+ *
+ * Each Rank has its own KeyValueStore, and these can send and
+ * receive keys between each others.
+ *
+ * The purpose of this is to ease synchronization between ranks
+ * when the message size is bounded.
+ *
+ * DONE =========> implement the code necessary to have more than one active transfer
+ * at any given time.
+ *
+ * DONE ==========> the system does not support empty values
+ *
+ * Key-value items can be manipulated locally or remotely.
+ *
+ *
+ * \author Sébastien Boisvert
+ */
+class KeyValueStore : public CorePlugin {
+
+ MessageTag RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART;
+ MessageTag RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART_REPLY;
+
+ __AddAdapter(KeyValueStore, RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART);
+ __AddAdapter(KeyValueStore, RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART_REPLY);
+
+ MyAllocator m_memoryAllocator;
+
+ map<string,KeyValueStoreItem> m_items;
+
+ Rank m_rank;
+ int m_size;
+ RingAllocator*m_outboxAllocator;
+ StaticVector*m_inbox;
+ StaticVector*m_outbox;
+
+ bool getStringKey(const char * key, int keyLength, string & keyObject);
+ int dumpMessageHeader(const char* key, uint32_t valueLength, uint32_t offset, char * buffer) const;
+ int loadMessageHeader(string & key, uint32_t & valueLength, uint32_t & offset, const char * buffer) const;
+
+ KeyValueStoreItem * getLocalItemFromKey(const char * key, int keyLength);
+
+ bool pushLocalKeyWithLength(const char * key, int keyLength, Rank destination);
+
+ /**
+ * Get a key-value entry from a source.
+ *
+ * Usage:
+ *
+ * if(m_store.receiveKeyAndValueFromRank("joe", 3, 42)) {
+ * char * value;
+ * int valueLength;
+ * m_store.get("joe", 3, &value, &valueLength);
+ *
+ * cout << " retrieved KEY <joe> from rank 42, VALUE has length ";
+ * cout << valueLength << " bytes" << endl;
+ * }
+ */
+ bool pullRemoteKeyWithLength(const char * key, int keyLength, Rank source);
+ bool insertLocalKeyWithLength(const char * key, int keyLength, char * value, int valueLength);
+ bool getLocalKeyWithLength(const char * key, int keyLength, char ** value, int * valueLength);
+
+public:
+
+ KeyValueStore();
+
+ void initialize(Rank rank, int size, RingAllocator * outboxAllocator, StaticVector * inbox, StaticVector * outbox);
+
+ /**
+ * Insert a local key
+ *
+ * \param value is the value already allocated using allocateMemory()
+ */
+ bool insertLocalKey(const string & key, char * value, int valueLength);
+
+ /**
+ * Remove a local key
+ */
+ bool removeLocalKey(const string & key);
+
+ /**
+ * Get a local key
+ *
+ * \param output parameter
+ * \param valueLength output parameter
+ */
+ bool getLocalKey(const string & key, char * & value, int & valueLength);
+
+ // TODO: implement the update method.
+ // Also, implement
+ //
+ // getRemoteKey(key, value, valueLength, rank, request)
+ //
+ // too !
+
+ /**
+ * Pull a key from a remote site.
+ *
+ * If you use ASCII keys (or any string key), you can also use this
+ * API call:
+ *
+ * After this calling this method, the test method needs to be called using the request
+ * object.
+ *
+ */
+ void pullRemoteKey(const string & key, const Rank & source, KeyValueStoreRequest & request);
+
+ /**
+ * Test a request for completion
+ */
+ bool test(KeyValueStoreRequest & request);
+
+ /**
+ * allocate memory using the slab allocator
+ * of the key-value store.
+ */
+ char * allocateMemory(int bytes);
+
+ void clear();
+ void destroy();
+
+ void call_RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART_REPLY(Message * message);
+ void call_RAYPLATFORM_MESSAGE_TAG_DOWNLOAD_OBJECT_PART(Message * message);
+
+ void resolveSymbols(ComputeCore*core);
+ void registerPlugin(ComputeCore * core);
+};
+
+#endif /* KeyValueStoreHeader */
diff --git a/RayPlatform/RayPlatform/store/KeyValueStoreItem.cpp b/RayPlatform/RayPlatform/store/KeyValueStoreItem.cpp
new file mode 100644
index 0000000..72be55f
--- /dev/null
+++ b/RayPlatform/RayPlatform/store/KeyValueStoreItem.cpp
@@ -0,0 +1,105 @@
+/*
+ * Author: Sébastien Boisvert
+ * Project: RayPlatform
+ * Licence: LGPL3
+ */
+
+#include "KeyValueStoreItem.h"
+
+#include <iostream>
+using namespace std;
+
+#ifdef CONFIG_ASSERT
+#include <assert.h>
+#endif
+
+KeyValueStoreItem::KeyValueStoreItem() {
+
+}
+
+KeyValueStoreItem::KeyValueStoreItem(char * value, int valueSize) {
+ m_value = value;
+ m_valueSize = valueSize;
+ m_downloadedSize = valueSize;
+
+ m_ready = true;
+}
+
+char * KeyValueStoreItem::getValue() {
+ return m_value;
+}
+
+int KeyValueStoreItem::getValueLength() const {
+ return m_valueSize;
+}
+
+void KeyValueStoreItem::markItemAsReady() {
+
+#ifdef CONFIG_ASSERT
+ assert(m_downloadedSize == m_valueSize);
+ assert( (m_value == NULL && m_valueSize == 0 && m_downloadedSize == 0) || (m_value != NULL && m_valueSize != 0));
+#endif
+
+#if 0
+ cout << "[DEBUG] markItemAsReady " << m_downloadedSize << "/" << m_valueSize << endl;
+#endif
+
+ m_ready = true;
+}
+
+bool KeyValueStoreItem::isItemReady() const {
+ return m_ready == true;
+}
+
+void KeyValueStoreItem::setDownloadedSize(int downloadedSize) {
+
+#ifdef CONFIG_ASSERT
+ assert(downloadedSize <= m_valueSize);
+#endif
+
+ m_downloadedSize = downloadedSize;
+}
+
+void KeyValueStoreItem::sendMessage() {
+
+#ifdef CONFIG_ASSERT
+ assert(m_messageWasSent == false);
+#endif
+
+ m_messageWasSent = true;
+ m_ready = false;
+}
+
+void KeyValueStoreItem::receiveMessage() {
+
+#ifdef CONFIG_ASSERT
+ assert(m_messageWasSent == true);
+#endif
+
+ m_messageWasSent = false;
+
+ if(m_downloadedSize == m_valueSize)
+ markItemAsReady();
+}
+
+bool KeyValueStoreItem::messageWasSent() const {
+ return m_messageWasSent;
+}
+
+void KeyValueStoreItem::setValue(char * value) {
+ m_value = value;
+}
+
+void KeyValueStoreItem::setValueLength(int valueSize) {
+ m_valueSize = valueSize;
+}
+
+void KeyValueStoreItem::startDownload() {
+
+ m_ready = false;
+ m_messageWasSent = false;
+}
+
+int KeyValueStoreItem::getOffset() const {
+ return m_downloadedSize;
+}
diff --git a/RayPlatform/RayPlatform/store/KeyValueStoreItem.h b/RayPlatform/RayPlatform/store/KeyValueStoreItem.h
new file mode 100644
index 0000000..64f2f26
--- /dev/null
+++ b/RayPlatform/RayPlatform/store/KeyValueStoreItem.h
@@ -0,0 +1,50 @@
+/*
+ * Author: Sébastien Boisvert
+ * Project: RayPlatform
+ * Licence: LGPL3
+ */
+
+#ifndef KeyValueStoreItem_Header
+#define KeyValueStoreItem_Header
+
+/**
+ * A key-value entry in the distributed store.
+ *
+ * \author Sébastien Boisvert
+ */
+class KeyValueStoreItem {
+
+ char * m_value;
+ int m_valueSize;
+ int m_downloadedSize;
+ bool m_messageWasSent;
+ bool m_ready;
+
+ void markItemAsReady();
+
+public:
+
+ KeyValueStoreItem();
+
+ KeyValueStoreItem(char * value, int valueLength);
+ char * getValue();
+ int getValueLength() const;
+
+ bool isItemReady() const;
+
+ void sendMessage();
+ void receiveMessage();
+
+ bool messageWasSent() const;
+
+ void setValue(char * value);
+ void setValueLength(int size);
+
+ void setDownloadedSize(int downloadedSize);
+
+ void startDownload();
+
+ int getOffset() const;
+};
+
+#endif // KeyValueStoreItem_Header
diff --git a/RayPlatform/RayPlatform/store/KeyValueStoreRequest.cpp b/RayPlatform/RayPlatform/store/KeyValueStoreRequest.cpp
new file mode 100644
index 0000000..bdaace2
--- /dev/null
+++ b/RayPlatform/RayPlatform/store/KeyValueStoreRequest.cpp
@@ -0,0 +1,66 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#include "KeyValueStoreRequest.h"
+
+#define KEY_VALUE_STORE_OPERATION_NONE 0x0
+#define KEY_VALUE_STORE_OPERATION_PULL_REQUEST 0x1
+#define KEY_VALUE_STORE_OPERATION_PUSH_REQUEST 0x2
+
+KeyValueStoreRequest::KeyValueStoreRequest() {
+ m_type = KEY_VALUE_STORE_OPERATION_NONE;
+}
+
+void KeyValueStoreRequest::initialize(const string & key, const Rank & rank) {
+
+ m_key = key;
+ m_rank = rank;
+}
+
+void KeyValueStoreRequest::setTypeToPullRequest() {
+
+ m_type = KEY_VALUE_STORE_OPERATION_PULL_REQUEST;
+}
+
+void KeyValueStoreRequest::setTypeToPushRequest() {
+
+ m_type = KEY_VALUE_STORE_OPERATION_PUSH_REQUEST;
+}
+
+const Rank & KeyValueStoreRequest::getRank() const {
+
+ return m_rank;
+}
+
+bool KeyValueStoreRequest::isAPullRequest() const {
+
+ return m_type == KEY_VALUE_STORE_OPERATION_PULL_REQUEST;
+}
+
+bool KeyValueStoreRequest::isAPushRequest() const {
+
+ return m_type == KEY_VALUE_STORE_OPERATION_PUSH_REQUEST;
+}
+
+const string & KeyValueStoreRequest::getKey() const {
+
+ return m_key;
+}
diff --git a/RayPlatform/RayPlatform/store/KeyValueStoreRequest.h b/RayPlatform/RayPlatform/store/KeyValueStoreRequest.h
new file mode 100644
index 0000000..a16c182
--- /dev/null
+++ b/RayPlatform/RayPlatform/store/KeyValueStoreRequest.h
@@ -0,0 +1,54 @@
+/*
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://github.com/sebhtml/RayPlatform
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You have received a copy of the GNU Lesser General Public License
+ along with this program (lgpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#ifndef _KeyValueStore_h
+#define _KeyValueStore_h
+
+#include <RayPlatform/core/types.h>
+
+#include <string>
+using namespace std;
+
+/**
+ * A key-value store request.
+ *
+ * \author Sébastien Boisvert
+ */
+class KeyValueStoreRequest {
+
+ int m_type;
+ string m_key;
+ Rank m_rank;
+
+public:
+
+ KeyValueStoreRequest();
+
+ void initialize(const string & key, const Rank & rank);
+ void setTypeToPullRequest();
+ void setTypeToPushRequest();
+ const Rank & getRank() const;
+ bool isAPullRequest() const;
+ bool isAPushRequest() const;
+ const string & getKey() const;
+};
+
+#endif
diff --git a/RayPlatform/structures/MyHashTable.h b/RayPlatform/RayPlatform/structures/MyHashTable.h
similarity index 98%
rename from RayPlatform/structures/MyHashTable.h
rename to RayPlatform/RayPlatform/structures/MyHashTable.h
index 4302f24..9761a2e 100644
--- a/RayPlatform/structures/MyHashTable.h
+++ b/RayPlatform/RayPlatform/structures/MyHashTable.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -39,14 +39,16 @@
*/
#define MAX_SAVED_PROBE 16
-#include <stdint.h>
-#include <time.h>
-#include "structures/MyHashTableGroup.h"
-#include <memory/ChunkAllocatorWithDefragmentation.h>
+#include "MyHashTableGroup.h"
+
+#include <RayPlatform/memory/ChunkAllocatorWithDefragmentation.h>
+#include <RayPlatform/memory/allocator.h> /* for __Malloc */
+
#include <iostream>
#include <assert.h>
#include <string.h> /* for strcpy */
-#include <memory/allocator.h> /* for __Malloc */
+#include <stdint.h>
+#include <time.h>
using namespace std;
/**
diff --git a/RayPlatform/structures/MyHashTableGroup.h b/RayPlatform/RayPlatform/structures/MyHashTableGroup.h
similarity index 98%
rename from RayPlatform/structures/MyHashTableGroup.h
rename to RayPlatform/RayPlatform/structures/MyHashTableGroup.h
index 62dd2aa..dbd2d5d 100644
--- a/RayPlatform/structures/MyHashTableGroup.h
+++ b/RayPlatform/RayPlatform/structures/MyHashTableGroup.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,9 +22,10 @@
#ifndef _MyHashTableGroup_hh
#define _MyHashTableGroup_hh
+#include <RayPlatform/memory/ChunkAllocatorWithDefragmentation.h>
+
#include <stdint.h>
#include <time.h>
-#include <memory/ChunkAllocatorWithDefragmentation.h>
#include <iostream>
#include <assert.h>
using namespace std;
diff --git a/RayPlatform/structures/MyHashTableIterator.h b/RayPlatform/RayPlatform/structures/MyHashTableIterator.h
similarity index 91%
rename from RayPlatform/structures/MyHashTableIterator.h
rename to RayPlatform/RayPlatform/structures/MyHashTableIterator.h
index 087c826..66a0d91 100644
--- a/RayPlatform/structures/MyHashTableIterator.h
+++ b/RayPlatform/RayPlatform/structures/MyHashTableIterator.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,7 +22,8 @@
#ifndef _MyHashTableIterator_H
#define _MyHashTableIterator_H
-#include <structures/MyHashTable.h>
+#include "MyHashTable.h"
+
#include <assert.h>
/**
diff --git a/RayPlatform/structures/MyStack.h b/RayPlatform/RayPlatform/structures/MyStack.h
similarity index 89%
rename from RayPlatform/structures/MyStack.h
rename to RayPlatform/RayPlatform/structures/MyStack.h
index b88290d..1604abd 100644
--- a/RayPlatform/structures/MyStack.h
+++ b/RayPlatform/RayPlatform/structures/MyStack.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,7 +21,7 @@
#ifndef _MyStack
#define _MyStack
-#include<assert.h>
+#include <assert.h>
#define _MAX_STACK_SIZE 1024
/**
diff --git a/RayPlatform/structures/SplayNode.h b/RayPlatform/RayPlatform/structures/SplayNode.h
similarity index 92%
rename from RayPlatform/structures/SplayNode.h
rename to RayPlatform/RayPlatform/structures/SplayNode.h
index 2a05d75..18edc87 100644
--- a/RayPlatform/structures/SplayNode.h
+++ b/RayPlatform/RayPlatform/structures/SplayNode.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,7 +21,7 @@
#ifndef _SplayNode
#define _SplayNode
-#include<iostream>
+#include <iostream>
using namespace std;
/**
diff --git a/RayPlatform/structures/SplayTree.h b/RayPlatform/RayPlatform/structures/SplayTree.h
similarity index 95%
rename from RayPlatform/structures/SplayTree.h
rename to RayPlatform/RayPlatform/structures/SplayTree.h
index e64d1c1..25e1ef3 100644
--- a/RayPlatform/structures/SplayTree.h
+++ b/RayPlatform/RayPlatform/structures/SplayTree.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -21,12 +21,14 @@
#ifndef _SplayTree
#define _SplayTree
-#include<structures/MyStack.h>
-#include<vector>
-#include<stdlib.h>
-#include<structures/SplayNode.h>
-#include<iostream>
-#include<memory/MyAllocator.h>
+#include "MyStack.h"
+#include "SplayNode.h"
+
+#include <RayPlatform/memory/MyAllocator.h>
+
+#include <iostream>
+#include <vector>
+#include <stdlib.h>
using namespace std;
/**
diff --git a/RayPlatform/structures/SplayTreeIterator.h b/RayPlatform/RayPlatform/structures/SplayTreeIterator.h
similarity index 89%
rename from RayPlatform/structures/SplayTreeIterator.h
rename to RayPlatform/RayPlatform/structures/SplayTreeIterator.h
index 456eb71..8f88eee 100644
--- a/RayPlatform/structures/SplayTreeIterator.h
+++ b/RayPlatform/RayPlatform/structures/SplayTreeIterator.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,11 +22,12 @@
#ifndef _SplayTreeIterator
#define _SplayTreeIterator
-#include<structures/MyStack.h>
-#include<structures/SplayNode.h>
-#include<structures/SplayTree.h>
-#include<assert.h>
-#include<stdlib.h>
+#include "MyStack.h"
+#include "SplayNode.h"
+#include "SplayTree.h"
+
+#include <assert.h>
+#include <stdlib.h>
/**
* An iterator on SplayTree
@@ -72,6 +73,11 @@ void SplayTreeIterator<TREE_KEY_TYPE,TREE_VALUE_TYPE>::constructor(SplayTree<TRE
m_tree=tree;
m_processed=0;
m_treeSize=m_tree->size();
+
+ // TODO not implemented.
+ // This solves a compilation warning.
+ m_rank = -1;
+
if(m_tree!=NULL && m_tree->getRoot()!=NULL){
m_stack.push(tree->getRoot());
}
@@ -79,7 +85,8 @@ void SplayTreeIterator<TREE_KEY_TYPE,TREE_VALUE_TYPE>::constructor(SplayTree<TRE
template<class TREE_KEY_TYPE,class TREE_VALUE_TYPE>
bool SplayTreeIterator<TREE_KEY_TYPE,TREE_VALUE_TYPE>::hasNext()const{
- #ifdef ASSERT
+
+ #ifdef CONFIG_ASSERT
if(m_stack.size()==0 && m_tree!=NULL){
if(m_processed!=(int)m_tree->size()){
cout<<"Rank="<<m_rank<<" id="<<m_id<<" Processed="<<m_processed<<" Tree="<<m_tree->size()<<" onRecord="<<m_treeSize<<endl;
@@ -87,6 +94,7 @@ bool SplayTreeIterator<TREE_KEY_TYPE,TREE_VALUE_TYPE>::hasNext()const{
assert(m_processed==(int)m_tree->size());
}
#endif
+
return m_stack.size()>0;
}
diff --git a/RayPlatform/structures/StaticVector.cpp b/RayPlatform/RayPlatform/structures/StaticVector.cpp
similarity index 79%
rename from RayPlatform/structures/StaticVector.cpp
rename to RayPlatform/RayPlatform/structures/StaticVector.cpp
index 57068e8..b537eea 100644
--- a/RayPlatform/structures/StaticVector.cpp
+++ b/RayPlatform/RayPlatform/structures/StaticVector.cpp
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -19,8 +19,9 @@
*/
-#include <structures/StaticVector.h>
-#include <memory/allocator.h>
+#include "StaticVector.h"
+
+#include <RayPlatform/memory/allocator.h>
#include <assert.h>
#include <string.h>
@@ -48,9 +49,9 @@ Message*StaticVector::at(int i){
return m_messages+i;
}
-// TODO: the message a should be passed as a pointer
-void StaticVector::push_back(Message a){
- m_messages[m_size++]=a;
+// The message is be passed as a pointer.
+void StaticVector::push_back(Message*a){
+ m_messages[m_size++]=*a;
}
int StaticVector::size(){
diff --git a/RayPlatform/structures/StaticVector.h b/RayPlatform/RayPlatform/structures/StaticVector.h
similarity index 73%
rename from RayPlatform/structures/StaticVector.h
rename to RayPlatform/RayPlatform/structures/StaticVector.h
index 563e637..e4a181a 100644
--- a/RayPlatform/structures/StaticVector.h
+++ b/RayPlatform/RayPlatform/structures/StaticVector.h
@@ -1,8 +1,8 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ RayPlatform: a message-passing development framework
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
- http://DeNovoAssembler.SourceForge.Net/
+ http://github.com/sebhtml/RayPlatform
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
@@ -22,12 +22,12 @@
#ifndef _StaticVector
#define _StaticVector
-#include <communication/Message.h>
-#include <communication/mpi_tags.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/communication/mpi_tags.h>
/**
* A static vector of Message.
- * This is used for the inbox and the outbox in Ray
+ * This is used for the inbox and the outbox in Ray ranks or mini-ranks
*
* clear is basically O(1), it just sets m_size=0
* \author Sébastien Boisvert
@@ -41,8 +41,8 @@ public:
Message*operator[](int i);
Message*at(int i);
- // TODO: messages should be passed by reference or pointer
- void push_back(Message a);
+ // Messages are passed by reference or pointer.
+ void push_back(Message*a);
int size();
void clear();
void constructor(int size,const char*type,bool show);
diff --git a/RayPlatform/common.mk b/RayPlatform/common.mk
index f6f1bb3..e0aa117 100644
--- a/RayPlatform/common.mk
+++ b/RayPlatform/common.mk
@@ -1,63 +1,91 @@
MPICXX = mpicxx
AR = ar
-CXXFLAGS= -O3 -Wall -ansi
RM = rm
+ECHO = echo
+
+Q=@
+
+ASSERT=n
+ASSERT-$(ASSERT)= -DASSERT
+CXXFLAGS= -O3 -Wall -std=c++98 $(ASSERT-y)
#memory
-obj-y += memory/ReusableMemoryStore.o memory/MyAllocator.o memory/RingAllocator.o
-obj-y += memory/allocator.o
-obj-y += memory/DefragmentationGroup.o memory/ChunkAllocatorWithDefragmentation.o memory/DefragmentationLane.o
+obj-y += RayPlatform/memory/ReusableMemoryStore.o
+obj-y += RayPlatform/memory/MyAllocator.o
+obj-y += RayPlatform/memory/RingAllocator.o
+obj-y += RayPlatform/memory/allocator.o
+obj-y += RayPlatform/memory/DefragmentationGroup.o
+obj-y += RayPlatform/memory/ChunkAllocatorWithDefragmentation.o
+obj-y += RayPlatform/memory/DefragmentationLane.o
+obj-y += RayPlatform/memory/DirtyBuffer.o
# routing stuff for option -route-messages
-#
-obj-y += routing/GraphImplementation.o
-obj-y += routing/GraphImplementationRandom.o
-obj-y += routing/GraphImplementationComplete.o
-obj-y += routing/GraphImplementationDeBruijn.o
-obj-y += routing/GraphImplementationKautz.o
-obj-y += routing/GraphImplementationExperimental.o
-obj-y += routing/GraphImplementationGroup.o
-obj-y += routing/Hypercube.o
-obj-y += routing/ConnectionGraph.o
-
-#communication
-obj-y += communication/mpi_tags.o communication/VirtualCommunicator.o communication/BufferedData.o \
-communication/Message.o communication/MessagesHandler.o
-obj-y += communication/MessageRouter.o
-
-# scheduling stuff
-obj-y += scheduling/VirtualProcessor.o
-obj-y += scheduling/TaskCreator.o
-obj-y += scheduling/SwitchMan.o
-
-#core
-obj-y += core/slave_modes.o
-obj-y += core/OperatingSystem.o
-obj-y += core/master_modes.o
-obj-y += core/ComputeCore.o
-obj-y += core/statistics.o
-
-# plugin architecture
-
-obj-y += plugins/CorePlugin.o
-obj-y += plugins/RegisteredPlugin.o
+obj-y += RayPlatform/routing/ConnectionGraph.o
+obj-y += RayPlatform/routing/GraphImplementation.o
+obj-y += RayPlatform/routing/GraphImplementationRandom.o
+obj-y += RayPlatform/routing/GraphImplementationComplete.o
+obj-y += RayPlatform/routing/GraphImplementationDeBruijn.o
+obj-y += RayPlatform/routing/GraphImplementationKautz.o
+obj-y += RayPlatform/routing/GraphImplementationExperimental.o
+obj-y += RayPlatform/routing/GraphImplementationGroup.o
+obj-y += RayPlatform/routing/Polytope.o
+obj-y += RayPlatform/routing/Torus.o
+
+# communication
+obj-y += RayPlatform/communication/mpi_tags.o
+obj-y += RayPlatform/communication/VirtualCommunicator.o
+obj-y += RayPlatform/communication/BufferedData.o
+obj-y += RayPlatform/communication/Message.o
+obj-y += RayPlatform/communication/MessagesHandler.o
+obj-y += RayPlatform/communication/MessageQueue.o
+obj-y += RayPlatform/communication/MessageRouter.o
+
+# scheduling
+obj-y += RayPlatform/scheduling/VirtualProcessor.o
+obj-y += RayPlatform/scheduling/TaskCreator.o
+obj-y += RayPlatform/scheduling/SwitchMan.o
+
+# core
+obj-y += RayPlatform/core/ComputeCore.o
+obj-y += RayPlatform/core/MiniRank.o
+obj-y += RayPlatform/core/slave_modes.o
+obj-y += RayPlatform/core/OperatingSystem.o
+obj-y += RayPlatform/core/master_modes.o
+obj-y += RayPlatform/core/statistics.o
+
+# modular plugin architecture
+obj-y += RayPlatform/plugins/CorePlugin.o
+obj-y += RayPlatform/plugins/RegisteredPlugin.o
# structures
-obj-y += structures/StaticVector.o
+obj-y += RayPlatform/structures/StaticVector.o
# profiling
-obj-y += profiling/Profiler.o
-obj-y += profiling/Derivative.o
-obj-y += profiling/TickLogger.o
-obj-y += profiling/TimePrinter.o
+obj-y += RayPlatform/profiling/Profiler.o
+obj-y += RayPlatform/profiling/Derivative.o
+obj-y += RayPlatform/profiling/TickLogger.o
+obj-y += RayPlatform/profiling/TimePrinter.o
+obj-y += RayPlatform/profiling/ProcessStatus.o
# handlers
-
-obj-y += handlers/MasterModeExecutor.o
-obj-y += handlers/SlaveModeExecutor.o
-obj-y += handlers/MessageTagExecutor.o
+obj-y += RayPlatform/handlers/MasterModeExecutor.o
+obj-y += RayPlatform/handlers/SlaveModeExecutor.o
+obj-y += RayPlatform/handlers/MessageTagExecutor.o
#cryptography
-obj-y += cryptography/crypto.o
+obj-y += RayPlatform/cryptography/crypto.o
+
+#store
+obj-y += RayPlatform/store/KeyValueStore.o
+obj-y += RayPlatform/store/KeyValueStoreItem.o
+obj-y += RayPlatform/store/KeyValueStoreRequest.o
+# actor model for the win (FTW)
+# Gul Agha, Massachusetts Institute of Technology, Cambridge, MA
+# Actors: a model of concurrent computation in distributed systems
+# http://dl.acm.org/citation.cfm?id=7929
+obj-y += RayPlatform/actors/Actor.o
+obj-y += RayPlatform/actors/Playground.o
+# file operations
+obj-y += RayPlatform/files/FileReader.o
diff --git a/RayPlatform/communication/Message.cpp b/RayPlatform/communication/Message.cpp
deleted file mode 100644
index a5eb27c..0000000
--- a/RayPlatform/communication/Message.cpp
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
-
- http://DeNovoAssembler.SourceForge.Net/
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Lesser General Public License as published by
- the Free Software Foundation, version 3 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Lesser General Public License for more details.
-
- You have received a copy of the GNU Lesser General Public License
- along with this program (lgpl-3.0.txt).
- see <http://www.gnu.org/licenses/>
-
-*/
-
-#include <communication/Message.h>
-#include <iostream>
-#include <communication/mpi_tags.h>
-using namespace std;
-
-/** buffer must be allocated or else it will CORE DUMP. */
-Message::Message(MessageUnit*b,int c,Rank dest,MessageTag tag,Rank source){
- m_buffer=b;
- m_count=c;
- m_destination=dest;
- m_tag=tag;
- m_source=source;
-}
-
-MessageUnit*Message::getBuffer(){
- return m_buffer;
-}
-
-int Message::getCount(){
- return m_count;
-}
-
-Rank Message::getDestination(){
- return m_destination;
-}
-
-MessageTag Message::getTag(){
- return m_tag;
-}
-
-Message::Message(){}
-
-int Message::getSource(){
- return m_source;
-}
-
-void Message::print(){
- uint8_t shortTag=getTag();
-
- cout<<"Source: "<<getSource()<<" Destination: "<<getDestination()<<" Tag: "<<MESSAGE_TAGS[shortTag]<<" Count: "<<getCount();
- if(m_count > 0){
- cout<<" Overlay: "<<hex<<m_buffer[0]<<dec;
- }
-}
-
-void Message::setBuffer(MessageUnit*buffer){
- m_buffer = buffer;
-}
-
-void Message::setTag(MessageTag tag){
- m_tag=tag;
-}
-
-void Message::setCount(int count){
- m_count=count;
-}
-
-void Message::setSource(Rank source){
- m_source=source;
-}
-
-void Message::setDestination(Rank destination){
- m_destination=destination;
-}
diff --git a/RayPlatform/communication/Message.h b/RayPlatform/communication/Message.h
deleted file mode 100644
index 71836eb..0000000
--- a/RayPlatform/communication/Message.h
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
-
- http://DeNovoAssembler.SourceForge.Net/
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Lesser General Public License as published by
- the Free Software Foundation, version 3 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Lesser General Public License for more details.
-
- You have received a copy of the GNU Lesser General Public License
- along with this program (lgpl-3.0.txt).
- see <http://www.gnu.org/licenses/>
-
-*/
-
-#ifndef _Message_H
-#define _Message_H
-
-#include <stdint.h>
-#include <core/types.h>
-
-/**
- * In Ray, every message is a Message.
- * the inbox and the outbox are arrays of Message's
- * All the code in Ray utilise Message to communicate information.
- * MPI_Datatype is always MPI_UNSIGNED_LONG_LONG
- * m_count is >=0 and <= MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(uint64_t)
- * (default is 4000/8 = 500 ).
- * \author Sébastien Boisvert
- */
-class Message{
- /** the message body, contains data
- * if NULL, m_count must be 0 */
- MessageUnit*m_buffer;
-
- /** the number of uint64_t that the m_buffer contains
- * can be 0 regardless of m_buffer value
- * */
- int m_count;
-
- /** the Message-passing interface rank destination
- * Must be >=0 and <= MPI_Comm_size()-1 */
- Rank m_destination;
-
- /**
- * Ray message-passing interface message tags are named RAY_MPI_TAG_<something>
- * see mpi_tag_macros.h
- */
- MessageTag m_tag;
-
- /** the message-passing interface rank source
- * Must be >=0 and <= MPI_Comm_size()-1 */
- Rank m_source;
-public:
- Message();
- Message(MessageUnit*b,int c,Rank dest,MessageTag tag,Rank source);
- MessageUnit*getBuffer();
- int getCount();
-/**
- * Returns the destination MPI rank
- */
- Rank getDestination();
-
-/**
- * Returns the message tag (RAY_MPI_TAG_something)
- */
- MessageTag getTag();
-/**
- * Gets the source MPI rank
- */
- Rank getSource();
-
- void print();
-
- void setBuffer(MessageUnit*buffer);
-
- void setTag(MessageTag tag);
-
- void setSource(Rank source);
-
- void setDestination(Rank destination);
-
- void setCount(int count);
-};
-
-#endif
diff --git a/RayPlatform/git.txt b/RayPlatform/git.txt
deleted file mode 100644
index e48ae69..0000000
--- a/RayPlatform/git.txt
+++ /dev/null
@@ -1 +0,0 @@
-git://github.com/sebhtml/RayPlatform.git
diff --git a/RayPlatform/github.txt b/RayPlatform/github.txt
deleted file mode 100644
index 292990c..0000000
--- a/RayPlatform/github.txt
+++ /dev/null
@@ -1 +0,0 @@
-https://github.com/sebhtml/RayPlatform
diff --git a/RayPlatform/handlers/MasterModeHandler.h b/RayPlatform/handlers/MasterModeHandler.h
deleted file mode 100644
index 41e102e..0000000
--- a/RayPlatform/handlers/MasterModeHandler.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
-
- http://DeNovoAssembler.SourceForge.Net/
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Lesser General Public License as published by
- the Free Software Foundation, version 3 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Lesser General Public License for more details.
-
- You have received a copy of the GNU Lesser General Public License
- along with this program (lgpl-3.0.txt).
- see <http://www.gnu.org/licenses/>
-*/
-
-#ifndef _MasterModeHandler_h
-#define _MasterModeHandler_h
-
-/* this is a macro to create the cpp code for an adapter */
-#define __CreateMasterModeAdapter( corePlugin,handle )\
-void __GetAdapter( corePlugin, handle ) () { \
- __GetPlugin( corePlugin ) -> __GetMethod( handle ) () ; \
-}
-
-
-#include <core/types.h>
-#include <core/master_modes.h>
-
-/**
- * base class for handling master modes
- * \author Sébastien Boisvert
- * with help from Élénie Godzaridis for the design
- * \date 2012-08-08 replaced this with function pointers
- */
-typedef void (*MasterModeHandler) () /* */ ;
-
-#endif
diff --git a/RayPlatform/handlers/MessageTagHandler.h b/RayPlatform/handlers/MessageTagHandler.h
deleted file mode 100644
index 51f426c..0000000
--- a/RayPlatform/handlers/MessageTagHandler.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
-
- http://DeNovoAssembler.SourceForge.Net/
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Lesser General Public License as published by
- the Free Software Foundation, version 3 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Lesser General Public License for more details.
-
- You have received a copy of the GNU Lesser General Public License
- along with this program (lgpl-3.0.txt).
- see <http://www.gnu.org/licenses/>
-*/
-
-#ifndef _MessageTagHandler_h
-#define _MessageTagHandler_h
-
-
-/* this is a macro to create the cpp code for an adapter */
-#define __CreateMessageTagAdapter( corePlugin, handle ) \
-void __GetAdapter( corePlugin, handle) ( Message*message ) { \
- __GetPlugin( corePlugin ) -> __GetMethod( handle ) ( message ) ; \
-}
-
-
-
-#include <communication/Message.h>
-#include <core/types.h>
-#include <communication/mpi_tags.h>
-
-/**
- * base class for handling message tags
- * \author Sébastien Boisvert
- * with help from Élénie Godzaridis for the design
- * \date 2012-08-08 replaced this with function pointers
- */
-typedef void (*MessageTagHandler) (Message*message) /* */ ;
-
-
-#endif
diff --git a/RayPlatform/handlers/SlaveModeHandler.h b/RayPlatform/handlers/SlaveModeHandler.h
deleted file mode 100644
index 6146d29..0000000
--- a/RayPlatform/handlers/SlaveModeHandler.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
-
- http://DeNovoAssembler.SourceForge.Net/
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Lesser General Public License as published by
- the Free Software Foundation, version 3 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Lesser General Public License for more details.
-
- You have received a copy of the GNU Lesser General Public License
- along with this program (lgpl-3.0.txt).
- see <http://www.gnu.org/licenses/>
-*/
-
-#ifndef _SlaveModeHandler_h
-#define _SlaveModeHandler_h
-
-#include <core/slave_modes.h>
-#include <core/types.h>
-
-/* this is a macro to create the cpp code for an adapter */
-#define __CreateSlaveModeAdapter( corePlugin,handle ) \
-void __GetAdapter( corePlugin, handle ) () { \
- __GetPlugin( corePlugin ) -> __GetMethod( handle ) () ; \
-}
-
-/**
- * base class for handling slave modes
- * \author Sébastien Boisvert
- * with help from Élénie Godzaridis for the design
- * \date 2012-08-08 replaced this with function pointers
- */
-typedef void (*SlaveModeHandler) () /* */ ;
-
-
-#endif
diff --git a/RayPlatform/memory/RingAllocator.cpp b/RayPlatform/memory/RingAllocator.cpp
deleted file mode 100644
index 33ac7a9..0000000
--- a/RayPlatform/memory/RingAllocator.cpp
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
-
- http://DeNovoAssembler.SourceForge.Net/
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Lesser General Public License as published by
- the Free Software Foundation, version 3 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Lesser General Public License for more details.
-
- You have received a copy of the GNU Lesser General Public License
- along with this program (lgpl-3.0.txt).
- see <http://www.gnu.org/licenses/>
-
-*/
-
-#include <memory/RingAllocator.h>
-#include <memory/allocator.h>
-
-#include <string.h>
-#include <assert.h>
-#include <iostream>
-using namespace std;
-
-// #define CONFIG_RING_VERBOSE
-
-#define BUFFER_STATE_AVAILABLE 0x0
-#define BUFFER_STATE_DIRTY 0x1
-
-void RingAllocator::constructor(int chunks,int size,const char*type,bool show){
-
- resetCount();
-
- /* m_max should never be 0 */
- #ifdef ASSERT
- assert(size>0);
- assert(chunks>0);
- #endif /* ASSERT */
-
- m_chunks=chunks;// number of buffers
-
- m_max=size;// maximum buffer size in bytes
-
- #ifdef ASSERT
- assert(m_chunks==chunks);
- assert(m_max==size);
- #endif /* ASSERT */
-
- strcpy(m_type,type);
-
- m_numberOfBytes=m_chunks*m_max;
-
- #ifdef ASSERT
- assert(m_numberOfBytes>0);
- #endif /* ASSERT */
-
- m_memory=(uint8_t*)__Malloc(sizeof(uint8_t)*m_numberOfBytes,m_type,show);
- m_bufferStates=(uint8_t*)__Malloc(m_chunks*sizeof(uint8_t),m_type,show);
-
- for(int i=0;i<m_chunks;i++)
- m_bufferStates[i]= BUFFER_STATE_AVAILABLE;
-
- m_current=0;// the current head for allocation operations
-
- m_show=show;
-
- // in the beginning, all buffers are available
- m_availableBuffers=m_chunks;
-
- #if 0
- cout<<"[RingAllocator] m_chunks: "<<m_chunks<<" m_max: "<<m_max<<endl;
- #endif
-
- #ifdef ASSERT
- assert(size>0);
- assert(chunks>0);
-
- if(m_chunks==0)
- cout<<"Error: chunks: "<<chunks<<" m_chunks: "<<m_chunks<<endl;
-
- assert(m_chunks>0);
- assert(m_max>0);
- #endif /* ASSERT */
-}
-
-RingAllocator::RingAllocator(){}
-
-/*
- * allocate a chunk of m_max bytes in constant time
- */
-void*RingAllocator::allocate(int a){
- #ifdef ASSERT
- m_count++;
- if(a>m_max){
- cout<<"Request "<<a<<" but maximum is "<<m_max<<endl;
- }
- assert(a<=m_max);
- #endif
-
- int origin=m_current;
-
- // first half of the circle
- // from origin to N-1
- while(m_bufferStates[m_current] == BUFFER_STATE_DIRTY
- && m_current < m_chunks){
-
- m_current++;
-
- }
-
- // start from 0 if we completed.
- // set i to 0
- if(m_current==m_chunks){
- m_current=0;
- }
-
- // from 0 to origin-1
- while(m_bufferStates[m_current] == BUFFER_STATE_DIRTY
- && m_current < origin){
-
- m_current++;
- }
-
- // if all buffers are dirty, we throw a runtime error
- #ifdef ASSERT
- if(m_current==origin && m_bufferStates[m_current]==BUFFER_STATE_DIRTY){
- cout<<"Error: all buffers are dirty !, chunks: "<<m_chunks<<endl;
- assert(m_current!=origin);
- }
- #endif
-
- void*address=(void*)(m_memory+m_current*m_max);
-
-
- m_current++;
-
- // depending on the architecture
- // branching (if) can be faster than integer division/modulo
-
- if(m_current==m_chunks){
- m_current=0;
- }
-
- return address;
-}
-
-void RingAllocator::salvageBuffer(void*buffer){
- int bufferNumber=getBufferHandle(buffer);
-
- m_bufferStates[bufferNumber]= BUFFER_STATE_AVAILABLE;
-
- #ifdef CONFIG_RING_VERBOSE
- cout<<"[RingAllocator::salvageBuffer] "<<bufferNumber<<" -> BUFFER_STATE_AVAILABLE"<<endl;
- #endif
-
- m_availableBuffers++;
-
- #ifdef ASSERT
- assert(m_availableBuffers>=1);
- #endif
-
-}
-
-void RingAllocator::markBufferAsDirty(void*buffer){
- int bufferNumber=getBufferHandle(buffer);
-
- m_bufferStates[bufferNumber]= BUFFER_STATE_DIRTY;
-
- #ifdef CONFIG_RING_VERBOSE
- cout<<"[RingAllocator::markBufferAsDirty] "<<bufferNumber<<" -> BUFFER_STATE_DIRTY"<<endl;
- #endif
-
- #ifdef ASSERT
- assert(m_availableBuffers>=1);
- #endif
-
- m_availableBuffers--;
-
- #ifdef ASSERT
- assert(m_availableBuffers>=0);
- #endif
-}
-
-int RingAllocator::getBufferHandle(void*buffer){
-
- void*origin=m_memory;
-
- uint64_t originValue=(uint64_t)origin;
- uint64_t bufferValue=(uint64_t)buffer;
-
- uint64_t difference=bufferValue-originValue;
-
- int index=difference/(m_max*sizeof(uint8_t));
-
- return index;
-}
-
-int RingAllocator::getSize(){
- return m_max;
-}
-
-void RingAllocator::clear(){
- __Free(m_memory,m_type,m_show);
- m_memory=NULL;
-}
-
-void RingAllocator::resetCount(){
- m_count=0;
-}
-
-int RingAllocator::getCount(){
- return m_count;
-}
-
-int RingAllocator::getNumberOfBuffers(){
- return m_chunks;
-}
diff --git a/RayPlatform/tag.txt b/RayPlatform/tag.txt
deleted file mode 100644
index b74810d..0000000
--- a/RayPlatform/tag.txt
+++ /dev/null
@@ -1 +0,0 @@
-7ceb2d018d4f549ddce375d1ef0f28ae2112c87b
diff --git a/code/plugin_Amos/Amos.cpp b/code/Amos/Amos.cpp
similarity index 86%
rename from code/plugin_Amos/Amos.cpp
rename to code/Amos/Amos.cpp
index 99e1f51..3fcf7aa 100644
--- a/code/plugin_Amos/Amos.cpp
+++ b/code/Amos/Amos.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,27 +19,27 @@
*/
-#include <plugin_Amos/Amos.h>
+#include "Amos.h"
+
+#include <code/Mock/constants.h>
+#include <code/Mock/Parameters.h>
+#include <code/SequencesLoader/ReadHandle.h>
+
+#include <RayPlatform/communication/mpi_tags.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/core/master_modes.h>
+#include <RayPlatform/core/slave_modes.h>
+
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <stdio.h>
-#include <communication/mpi_tags.h>
-#include <communication/Message.h>
-#include <application_core/constants.h>
-#include <core/master_modes.h>
-#include <core/slave_modes.h>
-#include <application_core/Parameters.h>
__CreatePlugin(Amos);
- /**/
-__CreateMasterModeAdapter(Amos,RAY_MASTER_MODE_AMOS); /**/
- /**/
-__CreateSlaveModeAdapter(Amos,RAY_SLAVE_MODE_AMOS); /**/
- /**/
- /**/
-
+__CreateMasterModeAdapter(Amos,RAY_MASTER_MODE_AMOS);
+__CreateSlaveModeAdapter(Amos,RAY_SLAVE_MODE_AMOS);
void Amos::constructor(Parameters*parameters,RingAllocator*outboxAllocator,StaticVector*outbox,
FusionData*fusionData,ExtensionData*extensionData,int*masterMode,int*slaveMode,Scaffolder*scaffolder,
@@ -62,7 +62,7 @@ void Amos::call_RAY_MASTER_MODE_AMOS(){
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(MessageUnit));
message[0]=m_ed->m_EXTENSION_currentPosition;
Message aMessage(message,1,m_ed->m_EXTENSION_rank,RAY_MPI_TAG_WRITE_AMOS,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_ed->m_EXTENSION_rank++;
m_ed->m_EXTENSION_currentRankIsDone=false;
m_ed->m_EXTENSION_currentRankIsStarted=true;
@@ -103,7 +103,7 @@ void Amos::call_RAY_SLAVE_MODE_AMOS(){
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(MessageUnit));
message[0]=m_ed->m_EXTENSION_currentPosition;
Message aMessage(message,1,MASTER_RANK,RAY_MPI_TAG_WRITE_AMOS_REPLY,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
fclose(m_amosFile);
*m_slave_mode=RAY_SLAVE_MODE_DO_NOTHING;
// iterate over the next one
@@ -131,7 +131,7 @@ void Amos::call_RAY_SLAVE_MODE_AMOS(){
#if defined(RAY_64_BITS)
fprintf(m_amosFile,"{CTG\niid:%u\neid:contig-%lu\ncom:\nSoftware: Ray, MPI rank: %i\n.\nseq:\n%s\n.\nqlt:\n%s\n.\n",
m_ed->m_EXTENSION_currentPosition+1,
- m_ed->m_EXTENSION_identifiers[m_contigId],
+ m_ed->m_EXTENSION_identifiers[m_contigId].getValue(),
m_parameters->getRank(),
seq.c_str(),
qlt
@@ -139,7 +139,7 @@ void Amos::call_RAY_SLAVE_MODE_AMOS(){
#elif defined(RAY_32_BITS)
fprintf(m_amosFile,"{CTG\niid:%u\neid:contig-%llu\ncom:\nSoftware: Ray, MPI rank: %i\n.\nseq:\n%s\n.\nqlt:\n%s\n.\n",
m_ed->m_EXTENSION_currentPosition+1,
- m_ed->m_EXTENSION_identifiers[m_contigId],
+ m_ed->m_EXTENSION_identifiers[m_contigId].getValue(),
m_parameters->getRank(),
seq.c_str(),
qlt
@@ -157,7 +157,8 @@ void Amos::call_RAY_SLAVE_MODE_AMOS(){
m_ed->m_EXTENSION_reads_requested=true;
m_ed->m_EXTENSION_reads_received=false;
- Kmer vertex=m_ed->m_EXTENSION_contigs[m_contigId][m_mode_send_vertices_sequence_id_position];
+ Kmer vertex;
+ m_ed->m_EXTENSION_contigs[m_contigId].at(m_mode_send_vertices_sequence_id_position,&vertex);
m_readFetcher.constructor(&vertex,m_outboxAllocator,m_inbox,m_outbox,m_parameters,m_virtualCommunicator,m_workerId,
RAY_MPI_TAG_REQUEST_VERTEX_READS);
@@ -190,7 +191,7 @@ void Amos::call_RAY_SLAVE_MODE_AMOS(){
int readLength=result[0];
int forwardOffset=result[1];
int reverseOffset=result[2];
- ReadHandle globalIdentifier=m_parameters->getGlobalIdFromRankAndLocalId(readRank,idOnRank)+1;
+ ReadHandle globalIdentifier = m_parameters->getGlobalIdFromRankAndLocalId(readRank,idOnRank)+1;
int start=forwardOffset;
int theEnd=readLength-1;
int offset=m_mode_send_vertices_sequence_id_position;
@@ -204,10 +205,10 @@ void Amos::call_RAY_SLAVE_MODE_AMOS(){
}
#if defined(RAY_64_BITS)
- fprintf(m_amosFile,"{TLE\nsrc:%li\noff:%i\nclr:%i,%i\n}\n",globalIdentifier,offset,
+ fprintf(m_amosFile,"{TLE\nsrc:%li\noff:%i\nclr:%i,%i\n}\n",globalIdentifier.getValue(), offset,
start,theEnd);
#elif defined(RAY_32_BITS)
- fprintf(m_amosFile,"{TLE\nsrc:%lli\noff:%i\nclr:%i,%i\n}\n",globalIdentifier,offset,
+ fprintf(m_amosFile,"{TLE\nsrc:%lli\noff:%i\nclr:%i,%i\n}\n",globalIdentifier.getValue(), offset,
start,theEnd);
#endif
@@ -227,6 +228,7 @@ void Amos::call_RAY_SLAVE_MODE_AMOS(){
}
void Amos::registerPlugin(ComputeCore*core){
+ m_core=core;
PluginHandle plugin=core->allocatePluginHandle();
m_plugin=plugin;
@@ -235,13 +237,10 @@ void Amos::registerPlugin(ComputeCore*core){
core->setPluginAuthors(plugin,"Sébastien Boisvert");
core->setPluginLicense(plugin,"GNU General Public License version 3");
- RAY_SLAVE_MODE_AMOS=core->allocateSlaveModeHandle(plugin);
- core->setSlaveModeObjectHandler(plugin,RAY_SLAVE_MODE_AMOS, __GetAdapter(Amos,RAY_SLAVE_MODE_AMOS));
- core->setSlaveModeSymbol(plugin,RAY_SLAVE_MODE_AMOS,"RAY_SLAVE_MODE_AMOS");
+ __ConfigureMasterModeHandler(Amos, RAY_MASTER_MODE_AMOS);
+ __ConfigureSlaveModeHandler(Amos, RAY_SLAVE_MODE_AMOS);
- RAY_MASTER_MODE_AMOS=core->allocateMasterModeHandle(plugin);
- core->setMasterModeObjectHandler(plugin,RAY_MASTER_MODE_AMOS, __GetAdapter(Amos,RAY_MASTER_MODE_AMOS));
- core->setMasterModeSymbol(plugin,RAY_MASTER_MODE_AMOS,"RAY_MASTER_MODE_AMOS");
+ __BindPlugin(Amos);
}
void Amos::resolveSymbols(ComputeCore*core){
@@ -255,6 +254,4 @@ void Amos::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_WRITE_AMOS=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_WRITE_AMOS");
RAY_MPI_TAG_WRITE_AMOS_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_WRITE_AMOS_REPLY");
RAY_MPI_TAG_REQUEST_VERTEX_READS=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_REQUEST_VERTEX_READS");
-
- __BindPlugin(Amos);
}
diff --git a/code/plugin_Amos/Amos.h b/code/Amos/Amos.h
similarity index 77%
rename from code/plugin_Amos/Amos.h
rename to code/Amos/Amos.h
index aec0f55..6cfd3ca 100644
--- a/code/plugin_Amos/Amos.h
+++ b/code/Amos/Amos.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -21,22 +21,26 @@
#ifndef _Amos
#define _Amos
-#include <application_core/Parameters.h>
-#include <structures/StaticVector.h>
-#include <memory/RingAllocator.h>
-#include <plugin_FusionData/FusionData.h>
-#include <plugin_SeedExtender/ExtensionData.h>
-#include <plugin_Scaffolder/Scaffolder.h>
-#include <plugin_SeedExtender/ReadFetcher.h>
-#include <communication/VirtualCommunicator.h>
-#include <core/ComputeCore.h>
+#include <code/Mock/Parameters.h>
+#include <code/FusionData/FusionData.h>
+#include <code/Scaffolder/Scaffolder.h>
+#include <code/SeedExtender/ExtensionData.h>
+#include <code/SeedExtender/ReadFetcher.h>
+
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <stdio.h>
#include <stdint.h>
#include <vector>
using namespace std;
+__DeclarePlugin(Amos);
+__DeclareMasterModeAdapter(Amos,RAY_MASTER_MODE_AMOS);
+__DeclareSlaveModeAdapter(Amos,RAY_SLAVE_MODE_AMOS);
/**
* AMOS specification is available : http://sourceforge.net/apps/mediawiki/amos/index.php?title=AMOS
@@ -45,6 +49,9 @@ using namespace std;
*/
class Amos : public CorePlugin{
+ __AddAdapter(Amos,RAY_MASTER_MODE_AMOS);
+ __AddAdapter(Amos,RAY_SLAVE_MODE_AMOS);
+
MessageTag RAY_MPI_TAG_REQUEST_VERTEX_READS;
MessageTag RAY_MPI_TAG_ASK_READ_LENGTH;
MessageTag RAY_MPI_TAG_WRITE_AMOS;
diff --git a/code/Amos/Makefile b/code/Amos/Makefile
new file mode 100644
index 0000000..0d250f4
--- /dev/null
+++ b/code/Amos/Makefile
@@ -0,0 +1,3 @@
+Amos-y += code/Amos/Amos.o
+
+obj-y += $(Amos-y)
diff --git a/code/plugin_CoverageGatherer/CoverageDistribution.cpp b/code/CoverageGatherer/CoverageDistribution.cpp
similarity index 93%
rename from code/plugin_CoverageGatherer/CoverageDistribution.cpp
rename to code/CoverageGatherer/CoverageDistribution.cpp
index fade283..e1c82cd 100644
--- a/code/plugin_CoverageGatherer/CoverageDistribution.cpp
+++ b/code/CoverageGatherer/CoverageDistribution.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,7 +19,8 @@
*/
-#include <plugin_CoverageGatherer/CoverageDistribution.h>
+#include "CoverageDistribution.h"
+
#include <iostream>
#include <fstream>
#include <map>
@@ -30,9 +31,14 @@ using namespace std;
CoverageDistribution::CoverageDistribution(map<CoverageDepth,LargeCount>*distributionOfCoverage,string*file){
+
if(file!=NULL){
ofstream f;
f.open(file->c_str());
+
+ f<<"# KmerCoverage Frequency"<<endl;
+ f<<"# Any frequency is a even number because of odd k-mer length"<<endl;
+
for(map<CoverageDepth,LargeCount>::iterator i=distributionOfCoverage->begin();i!=distributionOfCoverage->end();i++){
f<<""<<i->first<<" "<<i->second<<endl;
}
diff --git a/code/plugin_CoverageGatherer/CoverageDistribution.h b/code/CoverageGatherer/CoverageDistribution.h
similarity index 91%
rename from code/plugin_CoverageGatherer/CoverageDistribution.h
rename to code/CoverageGatherer/CoverageDistribution.h
index 6e6ee30..d623e53 100644
--- a/code/plugin_CoverageGatherer/CoverageDistribution.h
+++ b/code/CoverageGatherer/CoverageDistribution.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,8 +22,10 @@
#ifndef _CoverageDistribution
#define _CoverageDistribution
-#include <application_core/constants.h>
-#include <core/types.h>
+#include <code/Mock/constants.h>
+
+#include <RayPlatform/core/types.h>
+
#include <map>
#include <string>
#include <vector>
diff --git a/code/plugin_CoverageGatherer/CoverageGatherer.cpp b/code/CoverageGatherer/CoverageGatherer.cpp
similarity index 82%
rename from code/plugin_CoverageGatherer/CoverageGatherer.cpp
rename to code/CoverageGatherer/CoverageGatherer.cpp
index e6c4a17..b89fa91 100644
--- a/code/plugin_CoverageGatherer/CoverageGatherer.cpp
+++ b/code/CoverageGatherer/CoverageGatherer.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,36 +14,35 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_CoverageGatherer/CoverageGatherer.h>
-#include <core/OperatingSystem.h>
+#include "CoverageGatherer.h"
+
+#include <code/Mock/constants.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/VerticesExtractor/GridTableIterator.h>
+#include <code/VerticesExtractor/Vertex.h>
+
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/core/slave_modes.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/communication/mpi_tags.h>
+
+#include <sstream>
+#include <stdio.h>
+#include <stdint.h>
+using namespace std;
+
#ifdef ASSERT
#include <assert.h>
#endif
-#include <communication/Message.h>
-#include <communication/mpi_tags.h>
-#include <core/slave_modes.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <application_core/constants.h>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <plugin_VerticesExtractor/GridTableIterator.h>
-#include <plugin_VerticesExtractor/Vertex.h>
-#include <sstream>
__CreatePlugin(CoverageGatherer);
- /**/
- /**/
-__CreateSlaveModeAdapter(CoverageGatherer,RAY_SLAVE_MODE_SEND_DISTRIBUTION); /**/
- /**/
- /**/
-
-using namespace std;
+__CreateSlaveModeAdapter(CoverageGatherer,RAY_SLAVE_MODE_SEND_DISTRIBUTION);
void CoverageGatherer::writeKmers(){
#ifdef ASSERT
@@ -54,42 +53,22 @@ void CoverageGatherer::writeKmers(){
(*m_slaveMode)=RAY_SLAVE_MODE_DO_NOTHING;
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_COVERAGE_END,
m_parameters->getRank());
- (*m_outbox).push_back(aMessage);
+ m_outbox->push_back(&aMessage);
return;
}
GridTableIterator iterator;
iterator.constructor(m_subgraph,m_parameters->getWordSize(),m_parameters);
- FILE*kmerFile=NULL;
+ FILE* kmerFile=NULL;
+ ostringstream buffer;
ostringstream name;
name<<m_parameters->getPrefix()<<"/kmers.txt";
if(m_parameters->getRank()==0)
- kmerFile=fopen(name.str().c_str(),"w"); // create empty file
+ kmerFile=fopen(name.str().c_str(),"w"); // create empty file
else
- kmerFile=fopen(name.str().c_str(),"a"); // append to file
+ kmerFile=fopen(name.str().c_str(),"a"); // append to file
- fprintf(kmerFile,"# Generated by Ray %s\n",RAY_VERSION);
- fprintf(kmerFile,"# The length of k-mers is %i\n",m_parameters->getWordSize());
- if(m_parameters->getColorSpaceMode())
- fprintf(kmerFile,"# This file contains the k-mer graph (subgraph of the de Bruijn graph for alphabet={0,1,2,3})\n");
- else
- fprintf(kmerFile,"# This file contains the k-mer graph (subgraph of the de Bruijn graph for alphabet={A,C,G,T})\n");
- fprintf(kmerFile,"# The genome sequence you are looking for is a path in this maze\n");
- fprintf(kmerFile,"# You can kickstart your assembly algorithm development by loading this file\n");
- fprintf(kmerFile,"# Note that vertices with a coverage of 1 are not considered.\n");
- fprintf(kmerFile,"# Format:\n");
- if(m_parameters->getColorSpaceMode())
- fprintf(kmerFile,"# k-mer color sequence; coverage value; first color of parents; last color of children\n");
- else
- fprintf(kmerFile,"# k-mer nucleotide sequence; coverage value; first nucleotide of parents; last nucleotide of children\n");
- if(m_parameters->getColorSpaceMode()){
- fprintf(kmerFile,"# Example in color space:\n# 0312;10;3 2;1 0\n# 0312 has a coverage value of 10\n");
- fprintf(kmerFile,"# Ingoing arcs: 3031 -> 0312 and 2031 -> 0312\n");
- fprintf(kmerFile,"# Outgoing arcs: 0312 -> 3121 and 0312 -> 3120\n");
- }else{
- fprintf(kmerFile,"# Example in nucleotide space:\n# ATCG;10;T G;C A\n# ATCG has a coverage value of 10\n");
- fprintf(kmerFile,"# Ingoing arcs: TATC -> ATCG and GATC -> ATCG\n");
- fprintf(kmerFile,"# Outgoing arcs: ATCG -> TCGC and ATCG -> TCGA\n");
- }
+ if(m_parameters->getRank()==MASTER_RANK)
+ writeHeader(kmerFile);
while(iterator.hasNext()){
Vertex*node=iterator.next();
@@ -102,26 +81,36 @@ void CoverageGatherer::writeKmers(){
string kmerSequence=key.idToWord(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
vector<Kmer> parents=node->getIngoingEdges(&key,m_parameters->getWordSize());
vector<Kmer> children=node->getOutgoingEdges(&key,m_parameters->getWordSize());
- fprintf(kmerFile,"%s;%i;",kmerSequence.c_str(),coverage);
+
+ //fprintf(kmerFile,"%s;%i;",kmerSequence.c_str(),coverage);
+ buffer << kmerSequence << ";" << coverage << ";";
for(int i=0;i<(int)parents.size();i++){
string printableVersion=parents[i].idToWord(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
if(i!=0)
- fprintf(kmerFile," ");
+ buffer << " ";
+ //fprintf(kmerFile," ");
- fprintf(kmerFile,"%c",printableVersion[0]);
+ //fprintf(kmerFile,"%c",printableVersion[0]);
+ buffer << printableVersion[0];
}
- fprintf(kmerFile,";");
+ //fprintf(kmerFile,";");
+ buffer << ";";
for(int i=0;i<(int)children.size();i++){
string printableVersion=children[i].idToWord(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
if(i!=0)
- fprintf(kmerFile," ");
+ buffer << " ";
+ //fprintf(kmerFile," ");
- fprintf(kmerFile,"%c",printableVersion[m_parameters->getWordSize()-1]);
+ //fprintf(kmerFile,"%c",printableVersion[m_parameters->getWordSize()-1]);
+ buffer << printableVersion[m_parameters->getWordSize()-1];
}
- fprintf(kmerFile,"\n");
+ //fprintf("\n");
+ buffer << endl;
+ flushFileOperationBuffer_FILE(false, &buffer, kmerFile, CONFIG_FILE_IO_BUFFER_SIZE);
}
+ flushFileOperationBuffer_FILE(true, &buffer, kmerFile, CONFIG_FILE_IO_BUFFER_SIZE);
fclose(kmerFile);
-
+
#ifdef ASSERT
if(n!=m_subgraph->size()){
cout<<"n="<<n<<" size="<<m_subgraph->size()<<endl;
@@ -143,7 +132,7 @@ void CoverageGatherer::call_RAY_SLAVE_MODE_SEND_DISTRIBUTION(){
(*m_slaveMode)=RAY_SLAVE_MODE_DO_NOTHING;
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_COVERAGE_END,
m_parameters->getRank());
- (*m_outbox).push_back(aMessage);
+ m_outbox->push_back(&aMessage);
return;
}
GridTableIterator iterator;
@@ -158,7 +147,7 @@ void CoverageGatherer::call_RAY_SLAVE_MODE_SEND_DISTRIBUTION(){
n++;
#endif
}
-
+
#ifdef ASSERT
if(n!=m_subgraph->size()){
cout<<"Expected (from iterator)="<<n<<" Actual (->size())="<<m_subgraph->size()<<endl;
@@ -187,15 +176,15 @@ void CoverageGatherer::call_RAY_SLAVE_MODE_SEND_DISTRIBUTION(){
if(count!=0){
Message aMessage(messageContent,count,MASTER_RANK,RAY_MPI_TAG_COVERAGE_DATA,
m_parameters->getRank());
-
- (*m_outbox).push_back(aMessage);
+
+ m_outbox->push_back(&aMessage);
m_waiting=true;
}else{
m_distributionOfCoverage.clear();
(*m_slaveMode)=RAY_SLAVE_MODE_DO_NOTHING;
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_COVERAGE_END,
m_parameters->getRank());
- (*m_outbox).push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
}
}
@@ -210,7 +199,36 @@ void CoverageGatherer::constructor(Parameters*parameters,StaticVector*inbox,Stat
m_subgraph=subgraph;
}
+void CoverageGatherer::writeHeader(FILE*kmerFile){
+
+ fprintf(kmerFile,"# Generated by Ray %s\n",CONFIG_RAY_VERSION);
+ fprintf(kmerFile,"# The length of k-mers is %i\n",m_parameters->getWordSize());
+ if(m_parameters->getColorSpaceMode())
+ fprintf(kmerFile,"# This file contains the k-mer graph (subgraph of the de Bruijn graph for alphabet={0,1,2,3})\n");
+ else
+ fprintf(kmerFile,"# This file contains the k-mer graph (subgraph of the de Bruijn graph for alphabet={A,C,G,T})\n");
+ fprintf(kmerFile,"# The genome sequence you are looking for is a path in this maze\n");
+ fprintf(kmerFile,"# You can kickstart your assembly algorithm development by loading this file\n");
+ fprintf(kmerFile,"# Note that vertices with a coverage of 1 are not considered.\n");
+ fprintf(kmerFile,"# Format:\n");
+ if(m_parameters->getColorSpaceMode())
+ fprintf(kmerFile,"# k-mer color sequence; coverage value; first color of parents; last color of children\n");
+ else
+ fprintf(kmerFile,"# k-mer nucleotide sequence; coverage value; first nucleotide of parents; last nucleotide of children\n");
+ if(m_parameters->getColorSpaceMode()){
+ fprintf(kmerFile,"# Example in color space:\n# 0312;10;3 2;1 0\n# 0312 has a coverage value of 10\n");
+ fprintf(kmerFile,"# Ingoing arcs: 3031 -> 0312 and 2031 -> 0312\n");
+ fprintf(kmerFile,"# Outgoing arcs: 0312 -> 3121 and 0312 -> 3120\n");
+ }else{
+ fprintf(kmerFile,"# Example in nucleotide space:\n# ATCG;10;T G;C A\n# ATCG has a coverage value of 10\n");
+ fprintf(kmerFile,"# Ingoing arcs: TATC -> ATCG and GATC -> ATCG\n");
+ fprintf(kmerFile,"# Outgoing arcs: ATCG -> TCGC and ATCG -> TCGA\n");
+ }
+
+}
+
void CoverageGatherer::registerPlugin(ComputeCore*core){
+ m_core=core;
PluginHandle plugin=core->allocatePluginHandle();
m_plugin=plugin;
@@ -219,9 +237,7 @@ void CoverageGatherer::registerPlugin(ComputeCore*core){
core->setPluginAuthors(plugin,"Sébastien Boisvert");
core->setPluginLicense(plugin,"GNU General Public License version 3");
- RAY_SLAVE_MODE_SEND_DISTRIBUTION=core->allocateSlaveModeHandle(plugin);
- core->setSlaveModeObjectHandler(plugin,RAY_SLAVE_MODE_SEND_DISTRIBUTION, __GetAdapter(CoverageGatherer,RAY_SLAVE_MODE_SEND_DISTRIBUTION));
- core->setSlaveModeSymbol(plugin,RAY_SLAVE_MODE_SEND_DISTRIBUTION,"RAY_SLAVE_MODE_SEND_DISTRIBUTION");
+ __ConfigureSlaveModeHandler(CoverageGatherer, RAY_SLAVE_MODE_SEND_DISTRIBUTION);
RAY_MPI_TAG_COVERAGE_DATA_REPLY=core->allocateMessageTagHandle(plugin);
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_COVERAGE_DATA_REPLY,"RAY_MPI_TAG_COVERAGE_DATA_REPLY");
@@ -235,6 +251,7 @@ void CoverageGatherer::registerPlugin(ComputeCore*core){
RAY_MPI_TAG_GET_COVERAGE_AND_PATHS_REPLY=core->allocateMessageTagHandle(plugin);
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_GET_COVERAGE_AND_PATHS_REPLY,"RAY_MPI_TAG_GET_COVERAGE_AND_PATHS_REPLY");
+ __BindPlugin(CoverageGatherer);
}
void CoverageGatherer::resolveSymbols(ComputeCore*core){
@@ -249,7 +266,4 @@ void CoverageGatherer::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION_REPLY");
RAY_MPI_TAG_GET_COVERAGE_AND_MARK_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_COVERAGE_AND_MARK_REPLY");
RAY_MPI_TAG_GET_COVERAGE_AND_PATHS_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_COVERAGE_AND_PATHS_REPLY");
-
- __BindPlugin(CoverageGatherer);
-
}
diff --git a/code/plugin_CoverageGatherer/CoverageGatherer.h b/code/CoverageGatherer/CoverageGatherer.h
similarity index 79%
rename from code/plugin_CoverageGatherer/CoverageGatherer.h
rename to code/CoverageGatherer/CoverageGatherer.h
index 359b4c8..64ba409 100644
--- a/code/plugin_CoverageGatherer/CoverageGatherer.h
+++ b/code/CoverageGatherer/CoverageGatherer.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,23 +22,28 @@
#ifndef _CoverageGatherer_H
#define _CoverageGatherer_H
-#include <application_core/Parameters.h>
-#include <structures/StaticVector.h>
-#include <plugin_VerticesExtractor/GridTable.h>
-#include <memory/RingAllocator.h>
-#include <core/ComputeCore.h>
+#include <code/Mock/Parameters.h>
+#include <code/VerticesExtractor/GridTable.h>
+
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <stdint.h>
#include <map>
using namespace std;
+__DeclarePlugin(CoverageGatherer);
+__DeclareSlaveModeAdapter(CoverageGatherer,RAY_SLAVE_MODE_SEND_DISTRIBUTION);
/**
* \author Sébastien Boisvert
*/
class CoverageGatherer : public CorePlugin{
+ __AddAdapter(CoverageGatherer,RAY_SLAVE_MODE_SEND_DISTRIBUTION);
+
MessageTag RAY_MPI_TAG_COVERAGE_DATA_REPLY;
MessageTag RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION_REPLY;
MessageTag RAY_MPI_TAG_GET_COVERAGE_AND_MARK_REPLY;
@@ -60,6 +65,9 @@ class CoverageGatherer : public CorePlugin{
int*m_slaveMode;
GridTable*m_subgraph;
RingAllocator*m_outboxAllocator;
+
+ void writeHeader(FILE*kmerFile);
+
public:
void constructor(Parameters*parameters,StaticVector*inbox,StaticVector*outbox,int*slaveMode,
GridTable*subgraph,RingAllocator*outboxAllocator);
diff --git a/code/CoverageGatherer/Makefile b/code/CoverageGatherer/Makefile
new file mode 100644
index 0000000..f36ac16
--- /dev/null
+++ b/code/CoverageGatherer/Makefile
@@ -0,0 +1,4 @@
+CoverageGatherer-y += code/CoverageGatherer/CoverageGatherer.o
+CoverageGatherer-y += code/CoverageGatherer/CoverageDistribution.o
+
+obj-y += $(CoverageGatherer-y)
diff --git a/code/plugin_EdgePurger/EdgePurger.cpp b/code/EdgePurger/EdgePurger.cpp
similarity index 71%
rename from code/plugin_EdgePurger/EdgePurger.cpp
rename to code/EdgePurger/EdgePurger.cpp
index 6dabb8d..d7d02b3 100644
--- a/code/plugin_EdgePurger/EdgePurger.cpp
+++ b/code/EdgePurger/EdgePurger.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,19 +19,19 @@
*/
-#include <plugin_EdgePurger/EdgePurger.h>
+#include "EdgePurger.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
+
#include <stdlib.h>
#include <stdio.h>
-#include <core/OperatingSystem.h>
+#include <sstream>
+#include <fstream>
+using namespace std;
__CreatePlugin(EdgePurger);
- /**/
- /**/
__CreateSlaveModeAdapter(EdgePurger,RAY_SLAVE_MODE_PURGE_NULL_EDGES); /**/
- /**/
- /**/
-
//#define DEBUG_EdgePurger
@@ -53,6 +53,11 @@ void EdgePurger::constructor(StaticVector*outbox,StaticVector*inbox,RingAllocato
/* for TaskCreator */
m_initialized=false;
m_virtualProcessor=virtualProcessor;
+
+/*
+ * Reserve the number of slots for number of vertices.
+ */
+ m_graphSizes.resize(m_parameters->getSize());
}
void EdgePurger::call_RAY_SLAVE_MODE_PURGE_NULL_EDGES(){
@@ -61,21 +66,43 @@ void EdgePurger::call_RAY_SLAVE_MODE_PURGE_NULL_EDGES(){
/* master control */
if(m_inbox->size()>0&&m_inbox->at(0)->getTag()==RAY_MPI_TAG_PURGE_NULL_EDGES_REPLY){
+
+ Message*message=m_inbox->at(0);
+ Rank source=message->getSource();
+ MessageUnit*buffer=message->getBuffer();
+ LargeCount numberOfVertices=buffer[0];
+
+ #ifdef ASSERT
+ assert(source>=0);
+ assert(source<(int)m_graphSizes.size());
+ #endif
+
+ m_graphSizes[source]=numberOfVertices;
+
m_masterCountFinished++;
//cout<<"Receiving RAY_MPI_TAG_PURGE_NULL_EDGES_REPLY"<<endl;
if(m_masterCountFinished==m_parameters->getSize()){
+
+ writeGraphPartition();
+
(*m_masterMode)=RAY_MASTER_MODE_WRITE_KMERS;
}
}
+
if(m_done){
return;
}
MACRO_COLLECT_PROFILING_INFORMATION();
+
if(!m_checkedCheckpoint){
if(m_parameters->hasCheckpoint("GenomeGraph")){
- Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_PURGE_NULL_EDGES_REPLY,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+
+ MessageUnit*messageBuffer=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(MessageUnit));
+ int bufferSize=0;
+ messageBuffer[bufferSize++]=m_subgraph->size();
+ Message aMessage(messageBuffer,bufferSize,MASTER_RANK,RAY_MPI_TAG_PURGE_NULL_EDGES_REPLY,m_parameters->getRank());
+ m_outbox->push_back(&aMessage);
m_done=true;
return;
}
@@ -91,12 +118,16 @@ void EdgePurger::call_RAY_SLAVE_MODE_PURGE_NULL_EDGES(){
void EdgePurger::finalizeMethod(){
printf("Rank %i is purging edges [%i/%i] (completed)\n",m_parameters->getRank(),(int)m_subgraph->size(),(int)m_subgraph->size());
- fflush(stdout);
m_done=true;
- Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_PURGE_NULL_EDGES_REPLY,
- m_parameters->getRank());
- m_outbox->push_back(aMessage);
+
+ MessageUnit*messageBuffer=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(MessageUnit));
+ int bufferSize=0;
+ messageBuffer[bufferSize++]=m_subgraph->size();
+
+ Message aMessage(messageBuffer,bufferSize,MASTER_RANK,RAY_MPI_TAG_PURGE_NULL_EDGES_REPLY,
+ m_parameters->getRank());
+ m_outbox->push_back(&aMessage);
m_derivative.writeFile(&cout);
@@ -138,12 +169,58 @@ Worker* EdgePurger::assignNextTask(){
return worker;
}
+void EdgePurger::writeGraphPartition(){
+ ostringstream fileName;
+ fileName<<m_parameters->getPrefix()<<"GraphPartition.txt";
+
+ string theFileName=fileName.str();
+ ofstream fileStream(theFileName.c_str());
+
+
+ LargeCount total=0;
+
+ for(int i=0;i<(int)m_graphSizes.size();i++){
+ total+=m_graphSizes[i];
+ }
+
+ LargeCount best=total/m_graphSizes.size();
+
+ fileStream<<"#Rank NumberOfKmers IdealNumberOfKmers Difference RelativeDifference"<<endl;
+ fileStream<<"#TotalKmers: "<<total<<endl;
+ fileStream<<"#Ranks: "<<m_graphSizes.size()<<endl;
+ fileStream<<"#IdealNumberOfKmers: "<<best<<endl;
+
+ for(int i=0;i<(int)m_graphSizes.size();i++){
+ LargeCount actual=m_graphSizes[i];
+
+ int difference=0;
+
+ if(actual<best){
+ difference=best-actual;
+ difference=-difference;
+ }else{
+ difference=actual-best;
+ }
+
+ double relativeDifference=difference*100;
+
+ if(best!=0)
+ relativeDifference/=best;
+
+ fileStream<<i<<" "<<actual<<" "<<best<<" "<<difference;
+ fileStream<<" "<<relativeDifference<<"%"<<endl;
+ }
+
+ fileStream.close();
+
+ m_graphSizes.clear();
+}
+
void EdgePurger::processWorkerResult(Worker*){
/* nothing to do */
if(m_completedJobs%50000==0){
cout<<"Rank "<<m_parameters->getRank()<<" is purging edges ["<<m_completedJobs+1;
cout<<"/"<<m_subgraph->size()<<"]"<<endl;
- cout.flush();
m_derivative.addX(m_completedJobs);
m_derivative.printStatus(SLAVE_MODES[RAY_SLAVE_MODE_PURGE_NULL_EDGES],RAY_SLAVE_MODE_PURGE_NULL_EDGES);
@@ -206,4 +283,6 @@ void EdgePurger::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT_REPLY");
__BindPlugin(EdgePurger);
+
+ __BindAdapter(EdgePurger,RAY_SLAVE_MODE_PURGE_NULL_EDGES);
}
diff --git a/code/plugin_EdgePurger/EdgePurger.h b/code/EdgePurger/EdgePurger.h
similarity index 76%
rename from code/plugin_EdgePurger/EdgePurger.h
rename to code/EdgePurger/EdgePurger.h
index 0c8c8ce..a75e2b6 100644
--- a/code/plugin_EdgePurger/EdgePurger.h
+++ b/code/EdgePurger/EdgePurger.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,23 +22,29 @@
#ifndef _EdgePurger_H
#define _EdgePurger_H
-#include <application_core/Parameters.h>
-#include <memory/RingAllocator.h>
-#include <structures/StaticVector.h>
-#include <communication/VirtualCommunicator.h>
-#include <plugin_VerticesExtractor/GridTable.h>
-#include <plugin_VerticesExtractor/GridTableIterator.h>
-#include <profiling/Profiler.h>
-#include <profiling/Derivative.h>
-#include <plugin_EdgePurger/EdgePurgerWorker.h>
-#include <scheduling/TaskCreator.h>
-#include <core/ComputeCore.h>
+#include "EdgePurgerWorker.h"
+
+#include <code/Mock/Parameters.h>
+#include <code/VerticesExtractor/GridTable.h>
+#include <code/VerticesExtractor/GridTableIterator.h>
+
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/profiling/Profiler.h>
+#include <RayPlatform/profiling/Derivative.h>
+#include <RayPlatform/scheduling/TaskCreator.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <map>
#include <set>
+#include <vector>
#include <stdint.h>
using namespace std;
+__DeclarePlugin(EdgePurger);
+
+__DeclareSlaveModeAdapter(EdgePurger,RAY_SLAVE_MODE_PURGE_NULL_EDGES);
/**
* VerticesExtractor.cpp adds k-mers and ingoing and outgoing edges
@@ -50,6 +56,8 @@ using namespace std;
*/
class EdgePurger : public TaskCreator, public CorePlugin {
+ __AddAdapter(EdgePurger,RAY_SLAVE_MODE_PURGE_NULL_EDGES);
+
MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT_REPLY;
MessageTag RAY_MPI_TAG_PURGE_NULL_EDGES_REPLY;
@@ -58,8 +66,6 @@ class EdgePurger : public TaskCreator, public CorePlugin {
MasterMode RAY_MASTER_MODE_WRITE_KMERS;
SlaveMode RAY_SLAVE_MODE_PURGE_NULL_EDGES;
-
-
Profiler*m_profiler;
/** checkpointing */
@@ -81,6 +87,9 @@ class EdgePurger : public TaskCreator, public CorePlugin {
MyAllocator m_workerAllocator;
int*m_masterMode;
bool m_done;
+ vector<LargeCount> m_graphSizes;
+
+ void writeGraphPartition();
public:
void constructor(StaticVector*outbox,StaticVector*inbox,RingAllocator*outboxAllocator,Parameters*parameters,
diff --git a/code/plugin_EdgePurger/EdgePurgerWorker.cpp b/code/EdgePurger/EdgePurgerWorker.cpp
similarity index 95%
rename from code/plugin_EdgePurger/EdgePurgerWorker.cpp
rename to code/EdgePurger/EdgePurgerWorker.cpp
index a4c464d..cbb2030 100644
--- a/code/plugin_EdgePurger/EdgePurgerWorker.cpp
+++ b/code/EdgePurger/EdgePurgerWorker.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,7 +19,7 @@
*/
-#include <plugin_EdgePurger/EdgePurgerWorker.h>
+#include "EdgePurgerWorker.h"
bool EdgePurgerWorker::isDone(){
return m_isDone;
@@ -35,7 +35,7 @@ void EdgePurgerWorker::work(){
}else if(m_iterator<(int)m_edges.size()){
Kmer vertex=m_edges[m_iterator];
if(!m_coverageRequested){
- Rank sendTo=m_parameters->_vertexRank(&vertex);
+ Rank sendTo=m_parameters->vertexRank(&vertex);
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(MessageUnit));
int bufferPosition=0;
vertex.pack(message,&bufferPosition);
@@ -72,7 +72,7 @@ void EdgePurgerWorker::work(){
}else if(m_iterator<(int)m_edges.size()){
Kmer vertex=m_edges[m_iterator];
if(!m_coverageRequested){
- Rank sendTo=m_parameters->_vertexRank(&vertex);
+ Rank sendTo=m_parameters->vertexRank(&vertex);
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(MessageUnit));
int bufferPosition=0;
vertex.pack(message,&bufferPosition);
diff --git a/code/plugin_EdgePurger/EdgePurgerWorker.h b/code/EdgePurger/EdgePurgerWorker.h
similarity index 82%
rename from code/plugin_EdgePurger/EdgePurgerWorker.h
rename to code/EdgePurger/EdgePurgerWorker.h
index bac19d9..c382b48 100644
--- a/code/plugin_EdgePurger/EdgePurgerWorker.h
+++ b/code/EdgePurger/EdgePurgerWorker.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,15 +22,17 @@
#ifndef _EdgePurgerWorker_H
#define _EdgePurgerWorker_H
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/VerticesExtractor/Vertex.h>
+#include <code/VerticesExtractor/GridTable.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/scheduling/Worker.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+
#include <stdint.h>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <plugin_VerticesExtractor/Vertex.h>
-#include <memory/RingAllocator.h>
-#include <plugin_VerticesExtractor/GridTable.h>
-#include <application_core/Parameters.h>
-#include <structures/StaticVector.h>
-#include <scheduling/Worker.h>
-#include <communication/VirtualCommunicator.h>
/**
* An EdgePurgerWorker actually does the job.
diff --git a/code/EdgePurger/Makefile b/code/EdgePurger/Makefile
new file mode 100644
index 0000000..0c19ff2
--- /dev/null
+++ b/code/EdgePurger/Makefile
@@ -0,0 +1,4 @@
+EdgePurger-y += code/EdgePurger/EdgePurger.o
+EdgePurger-y += code/EdgePurger/EdgePurgerWorker.o
+
+obj-y += $(EdgePurger-y)
diff --git a/code/Example/Example.cpp b/code/Example/Example.cpp
new file mode 100644
index 0000000..d3ee9a4
--- /dev/null
+++ b/code/Example/Example.cpp
@@ -0,0 +1,273 @@
+/*
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2013 Charles Joly Beauparlant
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+ This is derived from RayPlatform-example.
+
+*/
+
+#include "Example.h"
+
+__CreatePlugin(Example);
+
+__CreateMasterModeAdapter(Example,RAY_MASTER_MODE_STEP_A);
+__CreateMasterModeAdapter(Example,RAY_MASTER_MODE_STEP_B);
+__CreateMasterModeAdapter(Example,RAY_MASTER_MODE_STEP_C);
+
+__CreateSlaveModeAdapter(Example,RAY_SLAVE_MODE_STEP_A);
+__CreateSlaveModeAdapter(Example,RAY_SLAVE_MODE_STEP_B);
+__CreateSlaveModeAdapter(Example,RAY_SLAVE_MODE_STEP_C);
+
+__CreateMessageTagAdapter(Example,RAY_MPI_TAG_STOP_AND_DIE);
+__CreateMessageTagAdapter(Example,RAY_MPI_TAG_TIME_BOMB);
+
+Example::Example(){
+ m_doneA=false;
+ m_doneB=false;
+ m_doneC=false;
+}
+
+void Example::call_RAY_MASTER_MODE_STEP_A(){
+ if(m_doneA==false){
+ m_doneA=true;
+ cout<<"Rank "<<m_core->getRank()<<" call_RAY_MASTER_MODE_STEP_A"<<endl;
+
+ m_core->getSwitchMan()->openMasterMode(m_core->getOutbox(),m_core->getRank());
+ }else if(m_core->getSwitchMan()->allRanksAreReady()){
+ m_core->getSwitchMan()->closeMasterMode();
+ }
+}
+
+void Example::call_RAY_MASTER_MODE_STEP_B(){
+ if(m_doneB==false){
+ m_doneB=true;
+ cout<<"Rank "<<MASTER_RANK<<" call_RAY_MASTER_MODE_STEP_B"<<endl;
+ m_core->getSwitchMan()->openMasterMode(m_core->getOutbox(),m_core->getRank());
+ }else if(m_core->getSwitchMan()->allRanksAreReady()){
+ m_core->getSwitchMan()->closeMasterMode();
+ }
+}
+
+void Example::call_RAY_MASTER_MODE_STEP_C(){
+ if(m_doneC==false){
+ m_doneC=true;
+ cout<<"Rank "<<MASTER_RANK<<" call_RAY_MASTER_MODE_STEP_C"<<endl;
+ m_core->getSwitchMan()->openMasterMode(m_core->getOutbox(),m_core->getRank());
+ }else if(m_core->getSwitchMan()->allRanksAreReady()){
+ m_core->getSwitchMan()->closeMasterMode();
+ }
+}
+
+void Example::call_RAY_SLAVE_MODE_STEP_A(){
+ cout<<"I am "<<m_core->getRank()<<" doing call_RAY_SLAVE_MODE_STEP_A"<<endl;
+
+ m_core->getSwitchMan()->closeSlaveModeLocally(m_core->getOutbox(),m_core->getRank());
+}
+
+void Example::call_RAY_SLAVE_MODE_STEP_B(){
+ cout<<"I am "<<m_core->getRank()<<" doing call_RAY_SLAVE_MODE_STEP_B"<<endl;
+
+ m_core->getSwitchMan()->closeSlaveModeLocally(m_core->getOutbox(),m_core->getRank());
+}
+
+void Example::call_RAY_SLAVE_MODE_STEP_C(){
+
+ cout<<"I am "<<m_core->getRank()<<" doing call_RAY_SLAVE_MODE_STEP_C, now I die"<<endl;
+ cout<<"This is over "<<endl;
+
+ if(m_core->getRank()==0){
+ uint64_t*buffer=(uint64_t*)m_core->getOutboxAllocator()->allocate(1*sizeof(uint64_t));
+ buffer[0]=100;
+
+ // compute the next destination
+ Rank destination=m_core->getRank()+1;
+ if(destination==m_core->getSize())
+ destination=0;
+
+ Message aMessage(buffer,1,destination,RAY_MPI_TAG_TIME_BOMB,m_core->getRank());
+
+ // send the bomb to another rank
+ m_core->getOutbox()->push_back(&aMessage);
+ }
+
+ // do nothing now
+ m_core->getSwitchMan()->setSlaveMode(RAY_SLAVE_MODE_DO_NOTHING);
+}
+
+void Example::call_RAY_MPI_TAG_TIME_BOMB(Message*message){
+ uint64_t*buffer=message->getBuffer();
+
+ if(buffer[0]==0){
+ cout<<"The bomb exploded on rank "<<m_core->getRank()<<" !"<<endl;
+
+ // kill everyone
+
+ m_core->sendEmptyMessageToAll(RAY_MPI_TAG_STOP_AND_DIE);
+ }else{
+ uint64_t*bufferOut=(uint64_t*)m_core->getOutboxAllocator()->allocate(1*sizeof(uint64_t));
+ bufferOut[0]=buffer[0]-1;
+
+ cout<<"Remaining time before the explosion is "<<bufferOut[0]<<" according to rank "<<m_core->getRank()<<endl;
+
+ // compute the next destination
+ Rank destination=m_core->getRank()+1;
+ if(destination==m_core->getSize())
+ destination=0;
+
+ Message aMessage(bufferOut,1,destination,RAY_MPI_TAG_TIME_BOMB,m_core->getRank());
+
+ // send the bomb to another rank
+ m_core->getOutbox()->push_back(&aMessage);
+ }
+}
+
+void Example::call_RAY_MPI_TAG_STOP_AND_DIE(Message*message){
+
+ cout<<"rank "<<m_core->getRank()<<" received message RAY_MPI_TAG_STOP_AND_DIE, this kills the batman"<<endl;
+
+ m_core->stop();
+}
+
+void Example::registerPlugin(ComputeCore*core){
+
+ /** register the m_plugin with the core **/
+
+ m_plugin=core->allocatePluginHandle();
+
+ core->setPluginName(m_plugin,"Example");
+ core->setPluginDescription(m_plugin,"Minimal plugin example");
+ core->setPluginAuthors(m_plugin,"Charles Joly Beauparlant");
+ core->setPluginLicense(m_plugin,"GNU General Public License version 3");
+
+ // for each master mode, we allocate a handle after that, we register a handler for it
+ //
+ // allocateMasterModeHandle takes 1 arguments
+ // - the plugin handle
+
+ RAY_MASTER_MODE_STEP_A=core->allocateMasterModeHandle(m_plugin);
+ core->setMasterModeObjectHandler(m_plugin,RAY_MASTER_MODE_STEP_A,
+ __GetAdapter(Example,RAY_MASTER_MODE_STEP_A));
+
+ core->setMasterModeSymbol(m_plugin,RAY_MASTER_MODE_STEP_A,"RAY_MASTER_MODE_STEP_A");
+
+ core->setMasterModePublicAccess(m_plugin, RAY_MASTER_MODE_STEP_A);
+
+ RAY_MASTER_MODE_STEP_B=core->allocateMasterModeHandle(m_plugin);
+ core->setMasterModeObjectHandler(m_plugin,RAY_MASTER_MODE_STEP_B,
+ __GetAdapter(Example,RAY_MASTER_MODE_STEP_B));
+
+ core->setMasterModeSymbol(m_plugin,RAY_MASTER_MODE_STEP_B,"RAY_MASTER_MODE_STEP_B");
+
+ RAY_MASTER_MODE_STEP_C=core->allocateMasterModeHandle(m_plugin);
+ core->setMasterModeObjectHandler(m_plugin,RAY_MASTER_MODE_STEP_C,
+ __GetAdapter(Example,RAY_MASTER_MODE_STEP_C));
+
+ core->setMasterModeSymbol(m_plugin,RAY_MASTER_MODE_STEP_C,"RAY_MASTER_MODE_STEP_C");
+
+ // for each slave mode, we allocate a handle after that, we register a handler for it
+
+ RAY_SLAVE_MODE_STEP_A=core->allocateSlaveModeHandle(m_plugin);
+ core->setSlaveModeObjectHandler(m_plugin,RAY_SLAVE_MODE_STEP_A,
+ __GetAdapter(Example,RAY_SLAVE_MODE_STEP_A));
+
+ core->setSlaveModeSymbol(m_plugin,RAY_SLAVE_MODE_STEP_A,"RAY_SLAVE_MODE_STEP_A");
+
+ RAY_SLAVE_MODE_STEP_B=core->allocateSlaveModeHandle(m_plugin);
+ core->setSlaveModeObjectHandler(m_plugin,RAY_SLAVE_MODE_STEP_B,
+ __GetAdapter(Example,RAY_SLAVE_MODE_STEP_B));
+
+ core->setSlaveModeSymbol(m_plugin,RAY_SLAVE_MODE_STEP_B,"RAY_SLAVE_MODE_STEP_B");
+
+ RAY_SLAVE_MODE_STEP_C=core->allocateSlaveModeHandle(m_plugin);
+ core->setSlaveModeObjectHandler(m_plugin,RAY_SLAVE_MODE_STEP_C,
+ __GetAdapter(Example,RAY_SLAVE_MODE_STEP_C));
+
+ core->setSlaveModeSymbol(m_plugin,RAY_SLAVE_MODE_STEP_C,"RAY_SLAVE_MODE_STEP_C");
+
+ RAY_MPI_TAG_START_STEP_A=core->allocateMessageTagHandle(m_plugin);
+ core->setMessageTagSymbol(m_plugin, RAY_MPI_TAG_START_STEP_A, "RAY_MPI_TAG_START_STEP_A");
+ RAY_MPI_TAG_START_STEP_B=core->allocateMessageTagHandle(m_plugin);
+ core->setMessageTagSymbol(m_plugin, RAY_MPI_TAG_START_STEP_B, "RAY_MPI_TAG_START_STEP_B");
+ RAY_MPI_TAG_START_STEP_C=core->allocateMessageTagHandle(m_plugin);
+ core->setMessageTagSymbol(m_plugin, RAY_MPI_TAG_START_STEP_C, "RAY_MPI_TAG_START_STEP_C");
+
+ // now, we register the order of the master modes
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_STEP_A,RAY_MASTER_MODE_STEP_B);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_STEP_B,RAY_MASTER_MODE_STEP_C);
+
+ // configure which control message to send given a master mode
+ core->setMasterModeToMessageTagSwitch(m_plugin,RAY_MASTER_MODE_STEP_A,RAY_MPI_TAG_START_STEP_A);
+ core->setMasterModeToMessageTagSwitch(m_plugin,RAY_MASTER_MODE_STEP_B,RAY_MPI_TAG_START_STEP_B);
+ core->setMasterModeToMessageTagSwitch(m_plugin,RAY_MASTER_MODE_STEP_C,RAY_MPI_TAG_START_STEP_C);
+
+ // configure which slave mode to set given a message tag
+ core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_START_STEP_A,RAY_SLAVE_MODE_STEP_A);
+ core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_START_STEP_B,RAY_SLAVE_MODE_STEP_B);
+ core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_START_STEP_C,RAY_SLAVE_MODE_STEP_C);
+
+ m_core=core;
+
+ // configure the two message tags
+ RAY_MPI_TAG_TIME_BOMB=core->allocateMessageTagHandle(m_plugin);
+ core->setMessageTagObjectHandler(m_plugin,RAY_MPI_TAG_TIME_BOMB,
+ __GetAdapter(Example,RAY_MPI_TAG_TIME_BOMB));
+
+ core->setMessageTagSymbol(m_plugin, RAY_MPI_TAG_TIME_BOMB, "RAY_MPI_TAG_TIME_BOMB");
+
+ RAY_MPI_TAG_STOP_AND_DIE=core->allocateMessageTagHandle(m_plugin);
+ core->setMessageTagObjectHandler(m_plugin,RAY_MPI_TAG_STOP_AND_DIE,
+ __GetAdapter(Example,RAY_MPI_TAG_STOP_AND_DIE));
+
+ core->setMessageTagSymbol(m_plugin, RAY_MPI_TAG_STOP_AND_DIE, "RAY_MPI_TAG_STOP_AND_DIE");
+
+ /* a plugin can share any of its object with other plugins **/
+ /** other plugins have to resolve the symbol. **/
+ void*object=&m_doneA;
+ core->setObjectSymbol(m_plugin,object,"BooleanState");
+}
+
+void Example::resolveSymbols(ComputeCore*core){
+
+ // here, we resolve symbols owned by other m_plugins
+ // but needed by the current m_plugin
+ // obviously, this is not needed here because
+ // we only have one m_plugin
+
+ RAY_SLAVE_MODE_STEP_A=core->getSlaveModeFromSymbol(m_plugin,"RAY_SLAVE_MODE_STEP_A");
+
+ // this slave mode is registered somewhere in RayPlatform
+ RAY_SLAVE_MODE_DO_NOTHING=core->getSlaveModeFromSymbol(m_plugin,"RAY_SLAVE_MODE_DO_NOTHING");
+
+ bool*example=(bool*) core->getObjectFromSymbol(m_plugin,"BooleanState");
+
+ if(*example)
+ *example=false;
+
+ __BindPlugin(Example);
+
+ __BindAdapter(Example,RAY_MASTER_MODE_STEP_A);
+ __BindAdapter(Example,RAY_MASTER_MODE_STEP_B);
+ __BindAdapter(Example,RAY_MASTER_MODE_STEP_C);
+
+ __BindAdapter(Example,RAY_SLAVE_MODE_STEP_A);
+ __BindAdapter(Example,RAY_SLAVE_MODE_STEP_B);
+ __BindAdapter(Example,RAY_SLAVE_MODE_STEP_C);
+
+ __BindAdapter(Example,RAY_MPI_TAG_STOP_AND_DIE);
+ __BindAdapter(Example,RAY_MPI_TAG_TIME_BOMB);
+}
diff --git a/code/Example/Example.h b/code/Example/Example.h
new file mode 100644
index 0000000..01fb444
--- /dev/null
+++ b/code/Example/Example.h
@@ -0,0 +1,117 @@
+/*
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2013 Charles Joly Beauparlant
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+ This is derived from RayPlatform-example.
+
+*/
+
+#ifndef _Example_h
+#define _Example_h
+
+#include <RayPlatform/core/ComputeCore.h>
+
+__DeclarePlugin(Example);
+
+__DeclareMasterModeAdapter(Example,RAY_MASTER_MODE_STEP_A);
+__DeclareMasterModeAdapter(Example,RAY_MASTER_MODE_STEP_B);
+__DeclareMasterModeAdapter(Example,RAY_MASTER_MODE_STEP_C);
+
+__DeclareSlaveModeAdapter(Example,RAY_SLAVE_MODE_STEP_A);
+__DeclareSlaveModeAdapter(Example,RAY_SLAVE_MODE_STEP_B);
+__DeclareSlaveModeAdapter(Example,RAY_SLAVE_MODE_STEP_C);
+
+__DeclareMessageTagAdapter(Example,RAY_MPI_TAG_STOP_AND_DIE);
+__DeclareMessageTagAdapter(Example,RAY_MPI_TAG_TIME_BOMB);
+
+/**
+ * The plugin Example
+ *
+ * \author Charles Joly Beauparlant
+ **/
+class Example: public CorePlugin{
+
+ __AddAdapter(Example,RAY_MASTER_MODE_STEP_A);
+ __AddAdapter(Example,RAY_MASTER_MODE_STEP_B);
+ __AddAdapter(Example,RAY_MASTER_MODE_STEP_C);
+
+ __AddAdapter(Example,RAY_SLAVE_MODE_STEP_A);
+ __AddAdapter(Example,RAY_SLAVE_MODE_STEP_B);
+ __AddAdapter(Example,RAY_SLAVE_MODE_STEP_C);
+
+ __AddAdapter(Example,RAY_MPI_TAG_STOP_AND_DIE);
+ __AddAdapter(Example,RAY_MPI_TAG_TIME_BOMB);
+
+/**
+ * A list of master modes
+ */
+ MasterMode RAY_MASTER_MODE_STEP_A;
+ MasterMode RAY_MASTER_MODE_STEP_B;
+ MasterMode RAY_MASTER_MODE_STEP_C;
+
+/** slave modes **/
+ SlaveMode RAY_SLAVE_MODE_STEP_A;
+ SlaveMode RAY_SLAVE_MODE_STEP_B;
+ SlaveMode RAY_SLAVE_MODE_STEP_C;
+ SlaveMode RAY_SLAVE_MODE_DO_NOTHING;
+
+
+ MessageTag RAY_MPI_TAG_START_STEP_A;
+ MessageTag RAY_MPI_TAG_START_STEP_B;
+ MessageTag RAY_MPI_TAG_START_STEP_C;
+
+ MessageTag RAY_MPI_TAG_TIME_BOMB;
+ MessageTag RAY_MPI_TAG_STOP_AND_DIE;
+
+ ComputeCore*m_core;
+
+/** states for progression **/
+ bool m_doneA;
+ bool m_doneB;
+ bool m_doneC;
+ bool m_doneExample;
+
+/** Example state **/
+ bool m_example;
+public:
+
+ Example();
+
+/** callbacks for master modes **/
+ void call_RAY_MASTER_MODE_STEP_A();
+ void call_RAY_MASTER_MODE_STEP_B();
+ void call_RAY_MASTER_MODE_STEP_C();
+
+/** callbacks for slave modes **/
+ void call_RAY_SLAVE_MODE_STEP_A();
+ void call_RAY_SLAVE_MODE_STEP_B();
+ void call_RAY_SLAVE_MODE_STEP_C();
+
+ void call_RAY_MPI_TAG_TIME_BOMB(Message*message);
+ void call_RAY_MPI_TAG_STOP_AND_DIE(Message*message);
+
+/** the following two methods are required by the interface CorePlugin **/
+
+/** register the plugin with the core **/
+ void registerPlugin(ComputeCore*core);
+
+/** resolve symbols not owned by the current plugin **/
+ void resolveSymbols(ComputeCore*core);
+};
+
+#endif
diff --git a/code/Example/Makefile b/code/Example/Makefile
new file mode 100644
index 0000000..8ca70c6
--- /dev/null
+++ b/code/Example/Makefile
@@ -0,0 +1,3 @@
+Example-y += code/Example/Example.o
+
+obj-y += $(Example-y)
diff --git a/code/plugin_FusionData/FusionData.cpp b/code/FusionData/FusionData.cpp
similarity index 91%
rename from code/plugin_FusionData/FusionData.cpp
rename to code/FusionData/FusionData.cpp
index e67368c..855f29f 100644
--- a/code/plugin_FusionData/FusionData.cpp
+++ b/code/FusionData/FusionData.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,11 +19,13 @@
*/
-#include <assert.h>
-#include <plugin_FusionData/FusionData.h>
-#include <core/OperatingSystem.h>
+#include "FusionData.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/communication/Message.h>
+
#include <sstream>
-#include <communication/Message.h>
+#include <assert.h>
__CreatePlugin(FusionData);
@@ -37,7 +39,7 @@ using namespace std;
#define SHOW_FUSION
-void FusionData::call_RAY_SLAVE_MODE_DISTRIBUTE_FUSIONS(){
+void FusionData::processCheckpoints(){
/** read the checkpoint ContigPaths */
if(!m_processedCheckpoint){
@@ -60,11 +62,12 @@ void FusionData::call_RAY_SLAVE_MODE_DISTRIBUTE_FUSIONS(){
int vertices=0;
f.read((char*)&name,sizeof(PathHandle));
f.read((char*)&vertices,sizeof(int));
- vector<Kmer> path;
+ GraphPath path;
+ path.setKmerLength(m_parameters->getWordSize());
for(int j=0;j<vertices;j++){
Kmer kmer;
kmer.read(&f);
- path.push_back(kmer);
+ path.push_back(&kmer);
}
#ifdef ASSERT
@@ -78,6 +81,11 @@ void FusionData::call_RAY_SLAVE_MODE_DISTRIBUTE_FUSIONS(){
f.close();
}
}
+}
+
+void FusionData::call_RAY_SLAVE_MODE_DISTRIBUTE_FUSIONS(){
+
+ processCheckpoints();
if(!isReady()){
return;
@@ -89,9 +97,9 @@ void FusionData::call_RAY_SLAVE_MODE_DISTRIBUTE_FUSIONS(){
}else if(m_buffers.isEmpty() && m_seedingData->m_SEEDING_i==(LargeCount)m_ed->m_EXTENSION_contigs.size()){
printf("Rank %i is distributing fusions [%i/%i] (completed)\n",getRank(),(int)m_ed->m_EXTENSION_contigs.size(),(int)m_ed->m_EXTENSION_contigs.size());
- fflush(stdout);
+
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
m_buffers.showStatistics(m_parameters->getRank());
m_buffers.clear();
@@ -110,7 +118,6 @@ void FusionData::call_RAY_SLAVE_MODE_DISTRIBUTE_FUSIONS(){
if(m_ed->m_EXTENSION_currentPosition==0){
if(m_seedingData->m_SEEDING_i%10==0){
printf("Rank %i is distributing fusions [%i/%i]\n",getRank(),(int)(m_seedingData->m_SEEDING_i+1),(int)m_ed->m_EXTENSION_contigs.size());
- fflush(stdout);
if(m_parameters->showMemoryUsage()){
showMemoryUsage(getRank());
@@ -125,14 +132,16 @@ void FusionData::call_RAY_SLAVE_MODE_DISTRIBUTE_FUSIONS(){
assert(m_ed->m_EXTENSION_contigs[m_seedingData->m_SEEDING_i].size() > 0);
#endif
- Kmer vertex=m_ed->m_EXTENSION_contigs[m_seedingData->m_SEEDING_i][m_ed->m_EXTENSION_currentPosition];
- int destination=m_parameters->_vertexRank(&vertex);
+ Kmer vertex;
+ m_ed->m_EXTENSION_contigs[m_seedingData->m_SEEDING_i].at(m_ed->m_EXTENSION_currentPosition,&vertex);
+
+ Rank destination=m_parameters->vertexRank(&vertex);
for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
m_buffers.addAt(destination,vertex.getU64(i));
}
- m_buffers.addAt(destination,m_ed->m_EXTENSION_identifiers[m_seedingData->m_SEEDING_i]);
+ m_buffers.addAt(destination,m_ed->m_EXTENSION_identifiers[m_seedingData->m_SEEDING_i].getValue());
m_buffers.addAt(destination,m_ed->m_EXTENSION_currentPosition);
if(m_buffers.flush(destination,KMER_U64_ARRAY_SIZE+2,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_WITH_REPLY,m_outboxAllocator,m_outbox,getRank(),false)){
@@ -213,7 +222,6 @@ bool FusionData::isReady(){
void FusionData::finishFusions(){
if(m_seedingData->m_SEEDING_i==(LargeCount)m_ed->m_EXTENSION_contigs.size()){
printf("Rank %i is finishing fusions [%i/%i] (completed)\n",getRank(),(int)m_ed->m_EXTENSION_contigs.size(),(int)m_ed->m_EXTENSION_contigs.size());
- fflush(stdout);
if(m_parameters->showMemoryUsage()){
showMemoryUsage(m_rank);
@@ -223,7 +231,7 @@ void FusionData::finishFusions(){
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(MessageUnit));
message[0]=m_FINISH_fusionOccured;
Message aMessage(message,1,MASTER_RANK,RAY_MPI_TAG_FINISH_FUSIONS_FINISHED,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
m_cacheForRepeatedVertices.clear();
@@ -248,7 +256,6 @@ void FusionData::finishFusions(){
if(m_seedingData->m_SEEDING_i%10==0){
printf("Rank %i is finishing fusions [%i/%i]\n",getRank(),(int)m_seedingData->m_SEEDING_i+1,(int)m_ed->m_EXTENSION_contigs.size());
- fflush(stdout);
if(m_parameters->showMemoryUsage()){
showMemoryUsage(getRank());
@@ -297,7 +304,13 @@ void FusionData::finishFusions(){
m_Machine_getPaths_DONE=true;
m_Machine_getPaths_result.clear();// avoids major leak... LOL
}else{
- getPaths(m_ed->m_EXTENSION_contigs[m_seedingData->m_SEEDING_i][m_ed->m_EXTENSION_currentPosition]);
+ int position=m_ed->m_EXTENSION_currentPosition;
+ GraphPath*path=&(m_ed->m_EXTENSION_contigs[m_seedingData->m_SEEDING_i]);
+
+ Kmer kmerObjectAtPosition;
+ path->at(position,&kmerObjectAtPosition);
+
+ getPaths(kmerObjectAtPosition);
}
}else{
// at this point, we have the paths that has the said vertex in them.
@@ -319,24 +332,32 @@ void FusionData::finishFusions(){
if(m_seedingData->m_SEEDING_i%10==0){
printf("Rank %i is finishing fusions [%i/%i]\n",getRank(),(int)m_seedingData->m_SEEDING_i+1,(int)m_ed->m_EXTENSION_contigs.size());
- fflush(stdout);
if(m_parameters->showMemoryUsage()){
showMemoryUsage(getRank());
showDate();
}
}
- vector<Kmer> a;
- m_FINISH_newFusions.push_back(a);
- vector<int> b;
+ GraphPath aPath;
+ aPath.setKmerLength(m_parameters->getWordSize());
+
+ m_FINISH_newFusions.push_back(aPath);
+
+// TODO: GraphPath provides a way to store coverage too !
+ vector<CoverageDepth> b;
m_FINISH_coverages.clear();
m_FINISH_vertex_requested=false;
m_FUSION_eliminated.insert(currentId);
m_FUSION_pathLengthRequested=false;
m_checkedValidity=false;
}
- Kmer vertex=m_ed->m_EXTENSION_contigs[m_seedingData->m_SEEDING_i][m_ed->m_EXTENSION_currentPosition];
- m_FINISH_newFusions[m_FINISH_newFusions.size()-1].push_back(vertex);
+
+ int position=m_ed->m_EXTENSION_currentPosition;
+ Kmer vertex;
+ m_ed->m_EXTENSION_contigs[m_seedingData->m_SEEDING_i].at(position,&vertex);
+
+ m_FINISH_newFusions[m_FINISH_newFusions.size()-1].push_back(&vertex);
+
m_Machine_getPaths_DONE=false;
m_Machine_getPaths_INITIALIZED=false;
m_Machine_getPaths_result.clear();
@@ -441,7 +462,13 @@ void FusionData::finishFusions(){
assert(m_seedingData->m_SEEDING_i<m_ed->m_EXTENSION_contigs.size());
assert(m_ed->m_EXTENSION_currentPosition<(int)m_ed->m_EXTENSION_contigs[m_seedingData->m_SEEDING_i].size());
#endif
- getPaths(m_ed->m_EXTENSION_contigs[m_seedingData->m_SEEDING_i][m_ed->m_EXTENSION_currentPosition]);
+
+ int position=m_ed->m_EXTENSION_currentPosition;
+ GraphPath*path=&(m_ed->m_EXTENSION_contigs[m_seedingData->m_SEEDING_i]);
+ Kmer kmerObject;
+ path->at(position,&kmerObject);
+
+ getPaths(kmerObject);
}else{
bool found=false;
@@ -488,14 +515,14 @@ void FusionData::finishFusions(){
if(!m_FUSION_pathLengthRequested){
Rank rankId=getRankFromPathUniqueId(pathId);
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(sizeof(MessageUnit));
- message[0]=pathId;
+ message[0]=pathId.getValue();
#ifdef ASSERT
assert(rankId<m_size);
#endif
Message aMessage(message,1,rankId,RAY_MPI_TAG_GET_PATH_LENGTH,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_FUSION_pathLengthRequested=true;
m_FUSION_pathLengthReceived=false;
}else if(m_FUSION_pathLengthReceived){
@@ -513,15 +540,15 @@ void FusionData::finishFusions(){
if(!m_FINISH_vertex_requested){
Rank rankId=getRankFromPathUniqueId(pathId);
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(sizeof(MessageUnit)*2);
- message[0]=pathId;
+ message[0]=pathId.getValue();
message[1]=nextPosition;
Message aMessage(message,2,rankId,RAY_MPI_TAG_GET_PATH_VERTEX,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_FINISH_vertex_requested=true;
m_FINISH_vertex_received=false;
}else if(m_FINISH_vertex_received){
- m_FINISH_newFusions[m_FINISH_newFusions.size()-1].push_back(m_FINISH_received_vertex);
+ m_FINISH_newFusions[m_FINISH_newFusions.size()-1].push_back(&m_FINISH_received_vertex);
m_FINISH_vertex_requested=false;
m_selectedPosition++;
m_FINISH_fusionOccured=true;
@@ -596,8 +623,8 @@ void FusionData::getPaths(Kmer vertex){
vertex.pack(message,&bufferPosition);
message[bufferPosition++]=0;
Message aMessage(message,bufferPosition,
- m_parameters->_vertexRank(&vertex),RAY_MPI_TAG_ASK_VERTEX_PATHS,getRank());
- m_outbox->push_back(aMessage);
+ m_parameters->vertexRank(&vertex),RAY_MPI_TAG_ASK_VERTEX_PATHS,getRank());
+ m_outbox->push_back(&aMessage);
m_FUSION_paths_requested=true;
m_FUSION_paths_received=false;
m_FUSION_receivedPaths.clear();
@@ -671,4 +698,6 @@ void FusionData::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_WITH_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_WITH_REPLY");
__BindPlugin(FusionData);
+
+ __BindAdapter(FusionData,RAY_SLAVE_MODE_DISTRIBUTE_FUSIONS);
}
diff --git a/code/plugin_FusionData/FusionData.h b/code/FusionData/FusionData.h
similarity index 83%
rename from code/plugin_FusionData/FusionData.h
rename to code/FusionData/FusionData.h
index 0c2ebe4..c154e4d 100644
--- a/code/plugin_FusionData/FusionData.h
+++ b/code/FusionData/FusionData.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,25 +22,28 @@
#ifndef _FusionData
#define _FusionData
-class SeedingData;
-
-#include <plugin_SeedExtender/Direction.h>
-#include <application_core/Parameters.h>
-#include <structures/StaticVector.h>
-#include <plugin_SeedingData/SeedingData.h>
-#include <plugin_SeedExtender/ExtensionData.h>
-#include <memory/RingAllocator.h>
-#include <communication/BufferedData.h>
-#include <profiling/TimePrinter.h>
-#include <handlers/SlaveModeHandler.h>
+#include <code/Mock/Parameters.h>
+#include <code/SeedingData/SeedingData.h>
+#include <code/SeedExtender/Direction.h>
+#include <code/SeedExtender/ExtensionData.h>
-#include <core/ComputeCore.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/communication/BufferedData.h>
+#include <RayPlatform/profiling/TimePrinter.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <vector>
#include <map>
#include <set>
using namespace std;
+class SeedingData;
+
+__DeclarePlugin(FusionData);
+
+__DeclareSlaveModeAdapter(FusionData,RAY_SLAVE_MODE_DISTRIBUTE_FUSIONS);
/*
* Contains information regarding fusion of extensions.
@@ -51,6 +54,8 @@ using namespace std;
*/
class FusionData : public CorePlugin {
+ __AddAdapter(FusionData,RAY_SLAVE_MODE_DISTRIBUTE_FUSIONS);
+
MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS;
MessageTag RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED;
MessageTag RAY_MPI_TAG_FINISH_FUSIONS_FINISHED;
@@ -94,11 +99,13 @@ class FusionData : public CorePlugin {
PathHandle m_selectedPath;
int m_selectedPosition;
+ void processCheckpoints();
+
public:
bool m_FINISH_vertex_received;
bool m_FINISH_fusionOccured;
- vector<vector<Kmer > > m_FINISH_newFusions;
+ vector<GraphPath> m_FINISH_newFusions;
vector<int> m_FINISH_coverages;
map<PathHandle,int> m_FINISH_pathLengths;
Kmer m_FINISH_received_vertex;
diff --git a/code/FusionData/Makefile b/code/FusionData/Makefile
new file mode 100644
index 0000000..8636b6d
--- /dev/null
+++ b/code/FusionData/Makefile
@@ -0,0 +1,3 @@
+FusionData-y += code/FusionData/FusionData.o
+
+obj-y += $(FusionData-y)
diff --git a/code/plugin_FusionTaskCreator/FusionTaskCreator.cpp b/code/FusionTaskCreator/FusionTaskCreator.cpp
similarity index 93%
rename from code/plugin_FusionTaskCreator/FusionTaskCreator.cpp
rename to code/FusionTaskCreator/FusionTaskCreator.cpp
index 79e288e..77dfacb 100644
--- a/code/plugin_FusionTaskCreator/FusionTaskCreator.cpp
+++ b/code/FusionTaskCreator/FusionTaskCreator.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,18 +19,14 @@
*/
-#include <plugin_FusionTaskCreator/FusionTaskCreator.h>
-#include <plugin_FusionTaskCreator/FusionWorker.h>
-#include <core/OperatingSystem.h>
+#include "FusionTaskCreator.h"
+#include "FusionWorker.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
__CreatePlugin(FusionTaskCreator);
- /**/
- /**/
__CreateSlaveModeAdapter(FusionTaskCreator,RAY_SLAVE_MODE_FUSION); /**/
- /**/
- /**/
-
void FusionTaskCreator::call_RAY_SLAVE_MODE_FUSION(){
@@ -39,7 +35,7 @@ void FusionTaskCreator::call_RAY_SLAVE_MODE_FUSION(){
void FusionTaskCreator::constructor(VirtualProcessor*virtualProcessor,StaticVector*outbox,
RingAllocator*outboxAllocator,int*mode,Parameters*parameters,
- vector<vector<Kmer> >*paths,vector<PathHandle>*pathIdentifiers,
+ vector<GraphPath>*paths,vector<PathHandle>*pathIdentifiers,
set<PathHandle>*eliminated,VirtualCommunicator*virtualCommunicator){
m_virtualCommunicator=virtualCommunicator;
@@ -99,7 +95,7 @@ void FusionTaskCreator::finalizeMethod(){
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(sizeof(MessageUnit));
message[0]=removedPaths;
Message aMessage(message,1,MASTER_RANK,RAY_MPI_TAG_FUSION_DONE,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
/* set the mode */
(*m_slaveMode)=RAY_SLAVE_MODE_DO_NOTHING;
@@ -133,7 +129,7 @@ bool FusionTaskCreator::hasUnassignedTask(){
*/
Worker*FusionTaskCreator::assignNextTask(){
- if(m_currentWorkerIdentifier % 10== 0){
+ if(m_currentWorkerIdentifier % 100== 0){
cout<<"Rank "<<m_parameters->getRank()<<" FusionTaskCreator assignNextTask ["<<m_currentWorkerIdentifier<<"/"<<m_paths->size()*2<<"]"<<endl;
if(m_parameters->showMemoryUsage()){
@@ -165,7 +161,7 @@ Worker*FusionTaskCreator::assignNextTask(){
/** get the result of a worker */
void FusionTaskCreator::processWorkerResult(Worker*worker){
- if(m_completedJobs % 10== 0){
+ if(m_completedJobs % 10000== 0){
cout<<"Rank "<<m_parameters->getRank()<<" FusionTaskCreator processWorkerResult ["<<m_completedJobs<<"/"<<m_paths->size()*2<<"]"<<endl;
if(m_parameters->showMemoryUsage()){
@@ -210,7 +206,7 @@ void FusionTaskCreator::registerPlugin(ComputeCore*core){
m_finishedInPreviousCycle=false;
- // TODO: this is buggy
+ // TODO: this is buggy, don't turn this on !
m_fastRun=false;
}
@@ -225,4 +221,7 @@ void FusionTaskCreator::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY_REPLY");
__BindPlugin(FusionTaskCreator);
+
+ __BindAdapter(FusionTaskCreator,RAY_SLAVE_MODE_FUSION);
+
}
diff --git a/code/plugin_FusionTaskCreator/FusionTaskCreator.h b/code/FusionTaskCreator/FusionTaskCreator.h
similarity index 70%
rename from code/plugin_FusionTaskCreator/FusionTaskCreator.h
rename to code/FusionTaskCreator/FusionTaskCreator.h
index bcd8b8a..35e1d54 100644
--- a/code/plugin_FusionTaskCreator/FusionTaskCreator.h
+++ b/code/FusionTaskCreator/FusionTaskCreator.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,46 +22,55 @@
#ifndef _FusionTaskCreator_H
#define _FusionTaskCreator_H
-#include <scheduling/Worker.h>
-#include <scheduling/TaskCreator.h>
-#include <scheduling/VirtualProcessor.h>
-#include <structures/StaticVector.h>
-#include <application_core/Parameters.h>
-#include <communication/VirtualCommunicator.h>
-#include <memory/RingAllocator.h>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <handlers/MasterModeHandler.h>
-#include <handlers/SlaveModeHandler.h>
-#include <core/ComputeCore.h>
+#include <code/Mock/Parameters.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/SeedingData/GraphPath.h>
+
+#include <RayPlatform/scheduling/Worker.h>
+#include <RayPlatform/scheduling/TaskCreator.h>
+#include <RayPlatform/scheduling/VirtualProcessor.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/handlers/MasterModeHandler.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <vector>
#include <set>
using namespace std;
+__DeclarePlugin(FusionTaskCreator);
+__DeclareSlaveModeAdapter(FusionTaskCreator,RAY_SLAVE_MODE_FUSION);
/**
* The class creates and kills workers for the fusion of
- * similar paths */
+ * similar paths.
+ *
+ * If path is self-contained in another, it is eliminated.
+ *
+ * \author Sébastien Boisvert
+ */
class FusionTaskCreator: public TaskCreator, public CorePlugin {
+ __AddAdapter(FusionTaskCreator,RAY_SLAVE_MODE_FUSION);
+
MessageTag RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY_REPLY;
MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH;
MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
MessageTag RAY_MPI_TAG_GET_PATH_LENGTH;
MessageTag RAY_MPI_TAG_FUSION_DONE;
-
SlaveMode RAY_SLAVE_MODE_FUSION;
SlaveMode RAY_SLAVE_MODE_DO_NOTHING;
-
VirtualCommunicator*m_virtualCommunicator;
RingAllocator*m_outboxAllocator;
Parameters*m_parameters;
StaticVector*m_outbox;
int*m_slaveMode;
- vector<vector<Kmer> >*m_paths;
+ vector<GraphPath>*m_paths;
vector<PathHandle>*m_pathIdentifiers;
set<PathHandle>*m_eliminated;
@@ -78,7 +87,7 @@ class FusionTaskCreator: public TaskCreator, public CorePlugin {
public:
void constructor( VirtualProcessor*virtualProcessor,StaticVector*outbox,
- RingAllocator*outboxAllocator,int*mode,Parameters*parameters,vector<vector<Kmer> >*paths,vector<PathHandle >*pathIdentifiers,
+ RingAllocator*outboxAllocator,int*mode,Parameters*parameters,vector<GraphPath>*paths,vector<PathHandle >*pathIdentifiers,
set<PathHandle>*eliminated,VirtualCommunicator*virtualCommunicator);
void call_RAY_SLAVE_MODE_FUSION();
diff --git a/code/plugin_FusionTaskCreator/FusionWorker.cpp b/code/FusionTaskCreator/FusionWorker.cpp
similarity index 92%
rename from code/plugin_FusionTaskCreator/FusionWorker.cpp
rename to code/FusionTaskCreator/FusionWorker.cpp
index 3ed8d71..313fce0 100644
--- a/code/plugin_FusionTaskCreator/FusionWorker.cpp
+++ b/code/FusionTaskCreator/FusionWorker.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,8 +19,8 @@
*/
+#include "FusionWorker.h"
-#include <plugin_FusionTaskCreator/FusionWorker.h>
#include <iostream>
using namespace std;
@@ -61,7 +61,8 @@ TODO: does the code pay attention when the coverage indicates a repeated k-mer ?
cout<<"FusionWorker "<<m_workerIdentifier<<" position: ["<<m_position<<"/"<<m_path->size()<<endl;
}
*/
- Kmer kmer=m_path->at(m_position);
+ Kmer kmer;
+ m_path->at(m_position,&kmer);
if(m_reverseStrand)
kmer=kmer.complementVertex(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
@@ -107,7 +108,9 @@ TODO: does the code pay attention when the coverage indicates a repeated k-mer ?
}else if(m_receivedNumberOfPaths && m_pathIndex < m_numberOfPaths){
/* request a path */
if(!m_requestedPath){
- Kmer kmer=m_path->at(m_position);
+ Kmer kmer;
+ m_path->at(m_position,&kmer);
+
if(m_reverseStrand)
kmer=kmer.complementVertex(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
@@ -178,7 +181,7 @@ TODO: does the code pay attention when the coverage indicates a repeated k-mer ?
PathHandle hitName=m_hitNames[m_hitIterator];
Rank destination=getRankFromPathUniqueId(hitName);
- message[0]=hitName;
+ message[0]=hitName.getValue();
Message aMessage(message,1,destination,
RAY_MPI_TAG_GET_PATH_LENGTH,m_parameters->getRank());
@@ -237,6 +240,17 @@ TODO: does the code pay attention when the coverage indicates a repeated k-mer ?
if(ratio < 0.7)
continue;
+/*
+ * We want to make sure that no sequence path is lost.
+ * They should be merged since it is not totally included.
+ */
+ int biologicalObjectsWithoutMatch=selfLength-matches;
+
+ int maximumAllowedLost=1024;
+
+ if(biologicalObjectsWithoutMatch>maximumAllowedLost)
+ continue;
+
/* the other is longer anyway */
if(hitLength > selfLength){
m_eliminated=true;
@@ -256,7 +270,10 @@ WorkerHandle FusionWorker::getWorkerIdentifier(){
return m_workerIdentifier;
}
-void FusionWorker::constructor(WorkerHandle number,vector<Kmer>*path,PathHandle identifier,bool reverseStrand,
+/**
+ * This class takes a path, and try to find another path that contains it.
+ */
+void FusionWorker::constructor(WorkerHandle number,GraphPath*path,PathHandle identifier,bool reverseStrand,
VirtualCommunicator*virtualCommunicator,Parameters*parameters,RingAllocator*outboxAllocator,
MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH,
diff --git a/code/plugin_FusionTaskCreator/FusionWorker.h b/code/FusionTaskCreator/FusionWorker.h
similarity index 81%
rename from code/plugin_FusionTaskCreator/FusionWorker.h
rename to code/FusionTaskCreator/FusionWorker.h
index efba841..8f9edfe 100644
--- a/code/plugin_FusionTaskCreator/FusionWorker.h
+++ b/code/FusionTaskCreator/FusionWorker.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,10 +22,12 @@
#ifndef _FusionWorker_H
#define _FusionWorker_H
-#include <communication/VirtualCommunicator.h>
-#include <scheduling/Worker.h>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <application_core/Parameters.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/Mock/Parameters.h>
+#include <code/SeedingData/GraphPath.h>
+
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/scheduling/Worker.h>
#include <stdint.h>
#include <map>
@@ -44,7 +46,11 @@ class FusionWorker: public Worker{
bool m_requestedNumberOfPaths;
WorkerHandle m_workerIdentifier;
bool m_isDone;
- vector<Kmer>*m_path;
+
+/*
+ * TODO: in the future, a PathHandle object will be included in any GraphPath object.
+ */
+ GraphPath*m_path;
PathHandle m_identifier;
bool m_reverseStrand;
bool m_eliminated;
@@ -65,7 +71,7 @@ class FusionWorker: public Worker{
bool m_receivedPath;
bool m_requestedPath;
public:
- void constructor(WorkerHandle i,vector<Kmer>*path,PathHandle identifier,bool reverseStrand,
+ void constructor(WorkerHandle i,GraphPath*path,PathHandle identifier,bool reverseStrand,
VirtualCommunicator*virtualCommunicator,Parameters*parameters,RingAllocator*outboxAllocator,
MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH,
MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
diff --git a/code/FusionTaskCreator/Makefile b/code/FusionTaskCreator/Makefile
new file mode 100644
index 0000000..765c1ea
--- /dev/null
+++ b/code/FusionTaskCreator/Makefile
@@ -0,0 +1,4 @@
+FusionTaskCreator-y += code/FusionTaskCreator/FusionWorker.o
+FusionTaskCreator-y += code/FusionTaskCreator/FusionTaskCreator.o
+
+obj-y += $(FusionTaskCreator-y)
diff --git a/code/plugin_GeneOntology/GeneOntology.cpp b/code/GeneOntology/GeneOntology.cpp
similarity index 98%
rename from code/plugin_GeneOntology/GeneOntology.cpp
rename to code/GeneOntology/GeneOntology.cpp
index 4ecfc44..da7f38e 100644
--- a/code/plugin_GeneOntology/GeneOntology.cpp
+++ b/code/GeneOntology/GeneOntology.cpp
@@ -22,22 +22,20 @@
#define SENTINEL_VALUE_FOR_TOTAL 123456789
-#include <plugin_GeneOntology/GeneOntology.h>
-#include <plugin_VerticesExtractor/GridTableIterator.h>
-#include <core/OperatingSystem.h>
-#include <plugin_GeneOntology/KeyEncoder.h>
+#include "GeneOntology.h"
+#include "KeyEncoder.h"
+
+#include <code/VerticesExtractor/GridTableIterator.h>
+
+#include <RayPlatform/core/OperatingSystem.h>
__CreatePlugin(GeneOntology);
- /**/
-__CreateMasterModeAdapter(GeneOntology,RAY_MASTER_MODE_ONTOLOGY_MAIN); /**/
- /**/
-__CreateSlaveModeAdapter(GeneOntology,RAY_SLAVE_MODE_ONTOLOGY_MAIN); /**/
- /**/
-__CreateMessageTagAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZE_TERMS); /**/
-__CreateMessageTagAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZE_TERMS_REPLY); /**/
-__CreateMessageTagAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZATION_DONE); /**/
- /**/
+__CreateMasterModeAdapter(GeneOntology,RAY_MASTER_MODE_ONTOLOGY_MAIN);
+__CreateSlaveModeAdapter(GeneOntology,RAY_SLAVE_MODE_ONTOLOGY_MAIN);
+__CreateMessageTagAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZE_TERMS);
+__CreateMessageTagAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZE_TERMS_REPLY);
+__CreateMessageTagAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZATION_DONE);
//#define BUG_DETERMINISM
//#define DEBUG_ONTOLOGY_SYNC
@@ -338,7 +336,7 @@ void GeneOntology::call_RAY_SLAVE_MODE_ONTOLOGY_MAIN(){
if(m_rank==MASTER_RANK){
// busy wait for other to complete their tasks.
- if(m_ranksSynchronized<m_core->getMessagesHandler()->getSize()){
+ if(m_ranksSynchronized<m_size){
return;
}
@@ -1149,7 +1147,7 @@ void GeneOntology::synchronize(){
return;
}
- bool isMaster=m_core->getMessagesHandler()->getRank() == MASTER_RANK;
+ bool isMaster=m_core->getRank() == MASTER_RANK;
if(!m_synchronizedTotal && !isMaster){
@@ -1480,8 +1478,8 @@ void GeneOntology::registerPlugin(ComputeCore*core){
m_outboxAllocator=core->getOutboxAllocator();
m_inboxAllocator=core->getInboxAllocator();
- m_rank=core->getMessagesHandler()->getRank();
- m_size=core->getMessagesHandler()->getSize();
+ m_rank=core->getRank();
+ m_size=core->getSize();
m_synchronizedTotal=false;
@@ -1510,11 +1508,16 @@ void GeneOntology::resolveSymbols(ComputeCore*core){
m_colorSet=(ColorSet*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/VirtualColorManagementUnit.ray");
m_timePrinter=(TimePrinter*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/Timer.ray");
- m_searcher=(Searcher*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/plugin_Searcher.ray");
+ m_searcher=(Searcher*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/Searcher.ray");
m_slaveStarted=false;
m_started=false;
__BindPlugin(GeneOntology);
-}
+ __BindAdapter(GeneOntology,RAY_MASTER_MODE_ONTOLOGY_MAIN);
+ __BindAdapter(GeneOntology,RAY_SLAVE_MODE_ONTOLOGY_MAIN);
+ __BindAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZE_TERMS);
+ __BindAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZE_TERMS_REPLY);
+ __BindAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZATION_DONE);
+}
diff --git a/code/plugin_GeneOntology/GeneOntology.h b/code/GeneOntology/GeneOntology.h
similarity index 83%
rename from code/plugin_GeneOntology/GeneOntology.h
rename to code/GeneOntology/GeneOntology.h
index 97386ce..5b24d63 100644
--- a/code/plugin_GeneOntology/GeneOntology.h
+++ b/code/GeneOntology/GeneOntology.h
@@ -21,18 +21,19 @@
#ifndef _GeneOntology_h
#define _GeneOntology_h
-#include <core/ComputeCore.h>
-#include <plugins/CorePlugin.h>
-#include <handlers/SlaveModeHandler.h>
-#include <plugin_Searcher/ColorSet.h>
-#include <handlers/MasterModeHandler.h>
-#include <handlers/MessageTagHandler.h>
-#include <application_core/Parameters.h>
-#include <plugin_VerticesExtractor/GridTable.h>
-#include <profiling/TimePrinter.h>
-
-#include <plugin_Searcher/Searcher.h>
-#include <plugin_GeneOntology/types.h>
+#include "types.h"
+
+#include <code/Searcher/Searcher.h>
+#include <code/Searcher/ColorSet.h>
+#include <code/Mock/Parameters.h>
+#include <code/VerticesExtractor/GridTable.h>
+
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/plugins/CorePlugin.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/handlers/MasterModeHandler.h>
+#include <RayPlatform/handlers/MessageTagHandler.h>
+#include <RayPlatform/profiling/TimePrinter.h>
#include <set>
#include <stdint.h>
@@ -40,6 +41,13 @@
#include <vector>
using namespace std;
+__DeclarePlugin(GeneOntology);
+
+__DeclareMasterModeAdapter(GeneOntology,RAY_MASTER_MODE_ONTOLOGY_MAIN);
+__DeclareSlaveModeAdapter(GeneOntology,RAY_SLAVE_MODE_ONTOLOGY_MAIN);
+__DeclareMessageTagAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZE_TERMS);
+__DeclareMessageTagAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZE_TERMS_REPLY);
+__DeclareMessageTagAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZATION_DONE);
/**
* a plugin to know what is present in a sample
@@ -48,6 +56,12 @@ using namespace std;
*/
class GeneOntology: public CorePlugin{
+ __AddAdapter(GeneOntology,RAY_MASTER_MODE_ONTOLOGY_MAIN);
+ __AddAdapter(GeneOntology,RAY_SLAVE_MODE_ONTOLOGY_MAIN);
+ __AddAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZE_TERMS);
+ __AddAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZE_TERMS_REPLY);
+ __AddAdapter(GeneOntology,RAY_MPI_TAG_SYNCHRONIZATION_DONE);
+
string m_ontologyFileName;
string m_annotationFileName;
diff --git a/code/plugin_GeneOntology/KeyEncoder.cpp b/code/GeneOntology/KeyEncoder.cpp
similarity index 96%
rename from code/plugin_GeneOntology/KeyEncoder.cpp
rename to code/GeneOntology/KeyEncoder.cpp
index 537cd52..765dfd2 100644
--- a/code/plugin_GeneOntology/KeyEncoder.cpp
+++ b/code/GeneOntology/KeyEncoder.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2012 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,13 +19,14 @@
*/
-#include <plugin_GeneOntology/KeyEncoder.h>
-#include <string.h>
+#include "KeyEncoder.h"
+
+#include <code/Mock/constants.h>
+#include <string.h>
#include <iostream>
using namespace std;
-
#ifdef ASSERT
#include <assert.h>
#endif
diff --git a/code/plugin_GeneOntology/KeyEncoder.h b/code/GeneOntology/KeyEncoder.h
similarity index 89%
rename from code/plugin_GeneOntology/KeyEncoder.h
rename to code/GeneOntology/KeyEncoder.h
index 037fe9c..b3ca7f2 100644
--- a/code/plugin_GeneOntology/KeyEncoder.h
+++ b/code/GeneOntology/KeyEncoder.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2012 Sébastien Boisvert
+ Copyright (C) 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -23,10 +23,12 @@
#define _KeyEncoder
+#include "types.h"
+
+#include <code/Searcher/VirtualKmerColor.h>
+
#include <map>
#include <stdint.h>
-#include <plugin_GeneOntology/types.h>
-#include <plugin_Searcher/VirtualKmerColor.h>
using namespace std;
class KeyEncoder{
diff --git a/code/GeneOntology/Makefile b/code/GeneOntology/Makefile
new file mode 100644
index 0000000..b2d1f02
--- /dev/null
+++ b/code/GeneOntology/Makefile
@@ -0,0 +1,4 @@
+GeneOntology-y += code/GeneOntology/KeyEncoder.o
+GeneOntology-y += code/GeneOntology/GeneOntology.o
+
+obj-y += $(GeneOntology-y)
diff --git a/code/plugin_GeneOntology/types.h b/code/GeneOntology/types.h
similarity index 100%
rename from code/plugin_GeneOntology/types.h
rename to code/GeneOntology/types.h
diff --git a/code/plugin_GenomeNeighbourhood/GenomeNeighbourhood.cpp b/code/GenomeNeighbourhood/GenomeNeighbourhood.cpp
similarity index 96%
rename from code/plugin_GenomeNeighbourhood/GenomeNeighbourhood.cpp
rename to code/GenomeNeighbourhood/GenomeNeighbourhood.cpp
index 0f19003..c8aaece 100644
--- a/code/plugin_GenomeNeighbourhood/GenomeNeighbourhood.cpp
+++ b/code/GenomeNeighbourhood/GenomeNeighbourhood.cpp
@@ -25,22 +25,18 @@
//#define DEBUG_SIDE
//#define DEBUG_LEFT_PATHS
-#include <plugin_GenomeNeighbourhood/GenomeNeighbourhood.h>
-#include <sstream>
+#include "GenomeNeighbourhood.h"
+#include <sstream>
#ifdef ASSERT
#include <assert.h>
#endif
__CreatePlugin(GenomeNeighbourhood);
- /**/
-__CreateMasterModeAdapter(GenomeNeighbourhood,RAY_MASTER_MODE_NEIGHBOURHOOD); /**/
- /**/
-__CreateSlaveModeAdapter(GenomeNeighbourhood,RAY_SLAVE_MODE_NEIGHBOURHOOD); /**/
- /**/
-__CreateMessageTagAdapter(GenomeNeighbourhood,RAY_MPI_TAG_NEIGHBOURHOOD_DATA); /**/
- /**/
+__CreateMasterModeAdapter(GenomeNeighbourhood,RAY_MASTER_MODE_NEIGHBOURHOOD);
+__CreateSlaveModeAdapter(GenomeNeighbourhood,RAY_SLAVE_MODE_NEIGHBOURHOOD);
+__CreateMessageTagAdapter(GenomeNeighbourhood,RAY_MPI_TAG_NEIGHBOURHOOD_DATA);
#define FETCH_PARENTS 0x00345678
@@ -104,7 +100,7 @@ void GenomeNeighbourhood::call_RAY_MPI_TAG_NEIGHBOURHOOD_DATA(Message*message){
Message aMessage(buffer,period,message->getSource(),RAY_MPI_TAG_NEIGHBOURHOOD_DATA_REPLY,
m_parameters->getRank());
- m_core->getOutbox()->push_back(aMessage);
+ m_core->getOutbox()->push_back(&aMessage);
}
void GenomeNeighbourhood::fetchPaths(int mode){
@@ -143,7 +139,7 @@ void GenomeNeighbourhood::fetchPaths(int mode){
int bufferPosition=0;
kmer.pack(buffer,&bufferPosition);
- Rank destination=m_parameters->_vertexRank(&kmer);
+ Rank destination=m_parameters->vertexRank(&kmer);
Message aMessage(buffer,m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE),
destination,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,m_rank);
@@ -362,7 +358,7 @@ void GenomeNeighbourhood::processLinks(int mode){
int bufferPosition=0;
currentKmer.pack(buffer,&bufferPosition);
- Rank destination=m_parameters->_vertexRank(¤tKmer);
+ Rank destination=m_parameters->vertexRank(¤tKmer);
Message aMessage(buffer,m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT),
destination,RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,m_rank);
@@ -397,8 +393,8 @@ void GenomeNeighbourhood::processLinks(int mode){
assert(coverage>0);
#endif
- vector<Kmer> parents=currentKmer._getIngoingEdges(edges,m_parameters->getWordSize());
- vector<Kmer> children=currentKmer._getOutgoingEdges(edges,m_parameters->getWordSize());
+ vector<Kmer> parents=currentKmer.getIngoingEdges(edges,m_parameters->getWordSize());
+ vector<Kmer> children=currentKmer.getOutgoingEdges(edges,m_parameters->getWordSize());
#ifdef DEBUG_NEIGHBOURHOOD_COMMUNICATION
cout<<"information: "<<parents.size()<<" parents, "<<children.size()<<" children"<<endl;
@@ -482,11 +478,11 @@ void GenomeNeighbourhood::processSide(int mode){
Kmer kmer;
if(mode==FETCH_CHILDREN){
- kmer=m_contigs->at(m_contigIndex).at(contigLength-1);
+ m_contigs->at(m_contigIndex).at(contigLength-1,&kmer);
}else if(mode==FETCH_PARENTS){
- kmer=m_contigs->at(m_contigIndex).at(0);
+ m_contigs->at(m_contigIndex).at(0,&kmer);
}
createStacks(kmer);
@@ -648,7 +644,7 @@ all other cases are invalid.
void GenomeNeighbourhood::call_RAY_MASTER_MODE_NEIGHBOURHOOD(){
if(!m_started){
- m_core->getSwitchMan()->openMasterMode(m_core->getOutbox(),m_core->getMessagesHandler()->getRank());
+ m_core->getSwitchMan()->openMasterMode(m_core->getOutbox(),m_core->getRank());
m_started=true;
@@ -805,7 +801,7 @@ void GenomeNeighbourhood::call_RAY_SLAVE_MODE_NEIGHBOURHOOD(){
cout<<"Rank "<<m_rank<<": the CorePlugin GenomeNeighbourhood is disabled..."<<endl;
- m_core->getSwitchMan()->closeSlaveModeLocally(m_core->getOutbox(),m_core->getMessagesHandler()->getRank());
+ m_core->getSwitchMan()->closeSlaveModeLocally(m_core->getOutbox(),m_core->getRank());
return; /* . */
}
@@ -912,7 +908,7 @@ void GenomeNeighbourhood::call_RAY_SLAVE_MODE_NEIGHBOURHOOD(){
m_virtualCommunicator->printStatistics();
- m_core->getSwitchMan()->closeSlaveModeLocally(m_core->getOutbox(),m_core->getMessagesHandler()->getRank());
+ m_core->getSwitchMan()->closeSlaveModeLocally(m_core->getOutbox(),m_core->getRank());
}
}
@@ -935,11 +931,11 @@ void GenomeNeighbourhood::sendRightNeighbours(){
int outputPosition=0;
- buffer[outputPosition++]=m_contigNames->at(m_contigIndex);
+ buffer[outputPosition++]=m_contigNames->at(m_contigIndex).getValue();
buffer[outputPosition++]='F';
buffer[outputPosition++]=m_contigs->at(m_contigIndex).size()-1;
- buffer[outputPosition++]=m_rightNeighbours[m_neighbourIndex].getContig();
+ buffer[outputPosition++]=m_rightNeighbours[m_neighbourIndex].getContig().getValue();
buffer[outputPosition++]=m_rightNeighbours[m_neighbourIndex].getStrand();
buffer[outputPosition++]=m_rightNeighbours[m_neighbourIndex].getProgression();
@@ -995,11 +991,11 @@ void GenomeNeighbourhood::sendLeftNeighbours(){
int outputPosition=0;
- buffer[outputPosition++]=m_leftNeighbours[m_neighbourIndex].getContig();
+ buffer[outputPosition++]=m_leftNeighbours[m_neighbourIndex].getContig().getValue();
buffer[outputPosition++]=m_leftNeighbours[m_neighbourIndex].getStrand();
buffer[outputPosition++]=m_leftNeighbours[m_neighbourIndex].getProgression();
- buffer[outputPosition++]=m_contigNames->at(m_contigIndex);
+ buffer[outputPosition++]=m_contigNames->at(m_contigIndex).getValue();
buffer[outputPosition++]='F';
buffer[outputPosition++]=0;
@@ -1106,18 +1102,17 @@ void GenomeNeighbourhood::resolveSymbols(ComputeCore*core){
// fetch parallel shared objects
m_timePrinter=(TimePrinter*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/Timer.ray");
- m_contigs=(vector<vector<Kmer> >*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/ContigPaths.ray");
+ m_contigs=(vector<GraphPath>*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/ContigPaths.ray");
m_contigNames=(vector<PathHandle>*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/ContigNames.ray");
m_parameters=(Parameters*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/Parameters.ray");
m_contigLengths=(map<PathHandle,int>*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/ContigLengths.ray");
-
m_virtualCommunicator=core->getVirtualCommunicator();
m_core=core;
m_started=false;
- m_rank=core->getMessagesHandler()->getRank();
+ m_rank=core->getRank();
m_outboxAllocator=core->getOutboxAllocator();
m_workerId=0;
@@ -1133,5 +1128,9 @@ void GenomeNeighbourhood::resolveSymbols(ComputeCore*core){
m_pluginIsEnabled=true;
__BindPlugin(GenomeNeighbourhood);
-}
+ __BindAdapter(GenomeNeighbourhood,RAY_MASTER_MODE_NEIGHBOURHOOD);
+ __BindAdapter(GenomeNeighbourhood,RAY_SLAVE_MODE_NEIGHBOURHOOD);
+ __BindAdapter(GenomeNeighbourhood,RAY_MPI_TAG_NEIGHBOURHOOD_DATA);
+
+}
diff --git a/code/plugin_GenomeNeighbourhood/GenomeNeighbourhood.h b/code/GenomeNeighbourhood/GenomeNeighbourhood.h
similarity index 81%
rename from code/plugin_GenomeNeighbourhood/GenomeNeighbourhood.h
rename to code/GenomeNeighbourhood/GenomeNeighbourhood.h
index ac34512..b9e804c 100644
--- a/code/plugin_GenomeNeighbourhood/GenomeNeighbourhood.h
+++ b/code/GenomeNeighbourhood/GenomeNeighbourhood.h
@@ -25,14 +25,16 @@
#ifndef _GenomeNeighbourhood_h
#define _GenomeNeighbourhood_h
-#include <profiling/TimePrinter.h>
-#include <core/ComputeCore.h>
-#include <plugins/CorePlugin.h>
-#include <plugin_GenomeNeighbourhood/Neighbour.h>
-#include <plugin_GenomeNeighbourhood/NeighbourPair.h>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <communication/VirtualCommunicator.h>
-#include <application_core/Parameters.h>
+#include "Neighbour.h"
+#include "NeighbourPair.h"
+
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/profiling/TimePrinter.h>
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/plugins/CorePlugin.h>
#include <vector>
#include <string>
@@ -40,6 +42,11 @@
#include <stack>
using namespace std;
+__DeclarePlugin(GenomeNeighbourhood);
+
+__DeclareMasterModeAdapter(GenomeNeighbourhood,RAY_MASTER_MODE_NEIGHBOURHOOD);
+__DeclareSlaveModeAdapter(GenomeNeighbourhood,RAY_SLAVE_MODE_NEIGHBOURHOOD);
+__DeclareMessageTagAdapter(GenomeNeighbourhood,RAY_MPI_TAG_NEIGHBOURHOOD_DATA);
/**
* The plugin GenomeNeighbourhood outputs a file file
@@ -49,10 +56,13 @@ using namespace std;
* amongst other things.
*
* \author Sébastien Boisvert
- * \
- * */
+ */
class GenomeNeighbourhood: public CorePlugin{
+ __AddAdapter(GenomeNeighbourhood,RAY_MASTER_MODE_NEIGHBOURHOOD);
+ __AddAdapter(GenomeNeighbourhood,RAY_SLAVE_MODE_NEIGHBOURHOOD);
+ __AddAdapter(GenomeNeighbourhood,RAY_MPI_TAG_NEIGHBOURHOOD_DATA);
+
bool m_pluginIsEnabled;
Parameters*m_parameters;
@@ -113,9 +123,8 @@ class GenomeNeighbourhood: public CorePlugin{
MessageTag RAY_MPI_TAG_NEIGHBOURHOOD_DATA;
MessageTag RAY_MPI_TAG_NEIGHBOURHOOD_DATA_REPLY;
-
/** contig paths */
- vector<vector<Kmer> >*m_contigs;
+ vector<GraphPath>*m_contigs;
vector<PathHandle>*m_contigNames;
/** genome folks in the neighbourhood **/
diff --git a/code/GenomeNeighbourhood/Makefile b/code/GenomeNeighbourhood/Makefile
new file mode 100644
index 0000000..2b5a37d
--- /dev/null
+++ b/code/GenomeNeighbourhood/Makefile
@@ -0,0 +1,5 @@
+GenomeNeighbourhood-y += code/GenomeNeighbourhood/GenomeNeighbourhood.o
+GenomeNeighbourhood-y += code/GenomeNeighbourhood/Neighbour.o
+GenomeNeighbourhood-y += code/GenomeNeighbourhood/NeighbourPair.o
+
+obj-y += $(GenomeNeighbourhood-y)
diff --git a/code/plugin_GenomeNeighbourhood/Neighbour.cpp b/code/GenomeNeighbourhood/Neighbour.cpp
similarity index 95%
rename from code/plugin_GenomeNeighbourhood/Neighbour.cpp
rename to code/GenomeNeighbourhood/Neighbour.cpp
index 7ba2c8f..de6d1a2 100644
--- a/code/plugin_GenomeNeighbourhood/Neighbour.cpp
+++ b/code/GenomeNeighbourhood/Neighbour.cpp
@@ -19,7 +19,7 @@
*/
-#include <plugin_GenomeNeighbourhood/Neighbour.h>
+#include "Neighbour.h"
Neighbour::Neighbour(Strand dnaStrand,int depth,PathHandle contig,int progression){
m_strand=dnaStrand;
diff --git a/code/plugin_GenomeNeighbourhood/Neighbour.h b/code/GenomeNeighbourhood/Neighbour.h
similarity index 93%
rename from code/plugin_GenomeNeighbourhood/Neighbour.h
rename to code/GenomeNeighbourhood/Neighbour.h
index 5019928..3305051 100644
--- a/code/plugin_GenomeNeighbourhood/Neighbour.h
+++ b/code/GenomeNeighbourhood/Neighbour.h
@@ -22,9 +22,10 @@
#ifndef _Neighbour_h
#define _Neighbour_h
-#include <stdint.h> /** for uint64_t **/
-#include <application_core/constants.h>
+#include <code/Mock/constants.h>
+#include <code/SeedingData/PathHandle.h>
+#include <stdint.h> /** for uint64_t **/
/**
* A folk in the genome neighbourhood.
diff --git a/code/plugin_GenomeNeighbourhood/NeighbourPair.cpp b/code/GenomeNeighbourhood/NeighbourPair.cpp
similarity index 93%
rename from code/plugin_GenomeNeighbourhood/NeighbourPair.cpp
rename to code/GenomeNeighbourhood/NeighbourPair.cpp
index cd01a10..8f8cd94 100644
--- a/code/plugin_GenomeNeighbourhood/NeighbourPair.cpp
+++ b/code/GenomeNeighbourhood/NeighbourPair.cpp
@@ -19,8 +19,9 @@
*/
-#include <plugin_GenomeNeighbourhood/NeighbourPair.h>
-#include <application_core/constants.h>
+#include "NeighbourPair.h"
+
+#include <code/Mock/constants.h>
NeighbourPair::NeighbourPair(PathHandle contig1,Strand strand1,int progression1,PathHandle contig2,Strand strand2,int progression2,
int depth){
diff --git a/code/plugin_GenomeNeighbourhood/NeighbourPair.h b/code/GenomeNeighbourhood/NeighbourPair.h
similarity index 94%
rename from code/plugin_GenomeNeighbourhood/NeighbourPair.h
rename to code/GenomeNeighbourhood/NeighbourPair.h
index 9919b6b..d7cb1cf 100644
--- a/code/plugin_GenomeNeighbourhood/NeighbourPair.h
+++ b/code/GenomeNeighbourhood/NeighbourPair.h
@@ -22,9 +22,10 @@
#ifndef _NeighbourPair_h
#define _NeighbourPair_h
-#include <stdint.h> /** for uint64_t **/
-#include <application_core/constants.h>
+#include <code/Mock/constants.h>
+#include <code/SeedingData/PathHandle.h>
+#include <stdint.h> /** for uint64_t **/
/**
* A folk in the genome neighbourhood.
diff --git a/code/plugin_JoinerTaskCreator/JoinerTaskCreator.cpp b/code/JoinerTaskCreator/JoinerTaskCreator.cpp
similarity index 92%
rename from code/plugin_JoinerTaskCreator/JoinerTaskCreator.cpp
rename to code/JoinerTaskCreator/JoinerTaskCreator.cpp
index b14f9ad..2a84a24 100644
--- a/code/plugin_JoinerTaskCreator/JoinerTaskCreator.cpp
+++ b/code/JoinerTaskCreator/JoinerTaskCreator.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,28 +19,27 @@
*/
-#include <plugin_JoinerTaskCreator/JoinerTaskCreator.h>
-#include <plugin_JoinerTaskCreator/JoinerWorker.h>
-#include <core/OperatingSystem.h>
+#include "JoinerTaskCreator.h"
+#include "JoinerWorker.h"
-__CreatePlugin(JoinerTaskCreator);
+#include <RayPlatform/core/OperatingSystem.h>
- /**/
- /**/
-__CreateSlaveModeAdapter(JoinerTaskCreator,RAY_SLAVE_MODE_FINISH_FUSIONS); /**/
- /**/
- /**/
+__CreatePlugin(JoinerTaskCreator);
+__CreateSlaveModeAdapter(JoinerTaskCreator,RAY_SLAVE_MODE_FINISH_FUSIONS);
+/**
+ * mainLoop() is provided by TaskCreator
+ */
void JoinerTaskCreator::call_RAY_SLAVE_MODE_FINISH_FUSIONS(){
mainLoop();
}
void JoinerTaskCreator::constructor(VirtualProcessor*virtualProcessor,StaticVector*outbox,
RingAllocator*outboxAllocator,int*mode,Parameters*parameters,
- vector<vector<Kmer> >*paths,vector<PathHandle>*pathIdentifiers,
+ vector<GraphPath>*paths,vector<PathHandle>*pathIdentifiers,
set<PathHandle>*eliminated,VirtualCommunicator*virtualCommunicator,
- vector<vector<Kmer> >*newPaths){
+ vector<GraphPath>*newPaths){
m_newPaths=newPaths;
m_virtualCommunicator=virtualCommunicator;
@@ -101,7 +100,7 @@ void JoinerTaskCreator::finalizeMethod(){
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(sizeof(MessageUnit));
message[0]=removedPaths;
Message aMessage(message,1,MASTER_RANK,RAY_MPI_TAG_FINISH_FUSIONS_FINISHED,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
/* set the mode */
(*m_slaveMode)=RAY_SLAVE_MODE_DO_NOTHING;
@@ -134,7 +133,7 @@ bool JoinerTaskCreator::hasUnassignedTask(){
*/
Worker*JoinerTaskCreator::assignNextTask(){
- if(m_currentWorkerIdentifier % 10== 0){
+ if(m_currentWorkerIdentifier % 10000== 0){
cout<<"Rank "<<m_parameters->getRank()<<" JoinerTaskCreator assignNextTask ["<<m_currentWorkerIdentifier<<"/"<<m_paths->size()*2<<"]"<<endl;
if(m_parameters->showMemoryUsage()){
@@ -168,7 +167,7 @@ Worker*JoinerTaskCreator::assignNextTask(){
/** get the result of a worker */
void JoinerTaskCreator::processWorkerResult(Worker*worker){
- if(m_completedJobs % 10== 0){
+ if(m_completedJobs % 100== 0){
cout<<"Rank "<<m_parameters->getRank()<<" JoinerTaskCreator processWorkerResult ["<<m_completedJobs<<"/"<<m_paths->size()*2<<"]"<<endl;
if(m_parameters->showMemoryUsage()){
@@ -223,4 +222,7 @@ void JoinerTaskCreator::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_FINISH_FUSIONS_FINISHED=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_FINISH_FUSIONS_FINISHED");
__BindPlugin(JoinerTaskCreator);
+
+ __BindAdapter(JoinerTaskCreator,RAY_SLAVE_MODE_FINISH_FUSIONS);
+
}
diff --git a/code/plugin_JoinerTaskCreator/JoinerTaskCreator.h b/code/JoinerTaskCreator/JoinerTaskCreator.h
similarity index 72%
rename from code/plugin_JoinerTaskCreator/JoinerTaskCreator.h
rename to code/JoinerTaskCreator/JoinerTaskCreator.h
index 26a61e2..d705ddc 100644
--- a/code/plugin_JoinerTaskCreator/JoinerTaskCreator.h
+++ b/code/JoinerTaskCreator/JoinerTaskCreator.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,27 +22,36 @@
#ifndef _JoinerTaskCreator_H
#define _JoinerTaskCreator_H
-#include <scheduling/Worker.h>
-#include <scheduling/TaskCreator.h>
-#include <scheduling/VirtualProcessor.h>
-#include <structures/StaticVector.h>
-#include <application_core/Parameters.h>
-#include <communication/VirtualCommunicator.h>
-#include <memory/RingAllocator.h>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <core/ComputeCore.h>
+#include <code/Mock/Parameters.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/SeedingData/GraphPath.h>
+
+#include <RayPlatform/scheduling/Worker.h>
+#include <RayPlatform/scheduling/TaskCreator.h>
+#include <RayPlatform/scheduling/VirtualProcessor.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <vector>
#include <set>
using namespace std;
+__DeclarePlugin(JoinerTaskCreator);
+__DeclareSlaveModeAdapter(JoinerTaskCreator,RAY_SLAVE_MODE_FINISH_FUSIONS);
/**
* The class creates and kills workers for the fusion of
- * similar paths */
+ * similar paths
+ *
+ * \author Sébastien Boisvert
+ * */
class JoinerTaskCreator: public TaskCreator, public CorePlugin{
+ __AddAdapter(JoinerTaskCreator,RAY_SLAVE_MODE_FINISH_FUSIONS);
+
MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH;
MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
MessageTag RAY_MPI_TAG_GET_PATH_LENGTH;
@@ -58,7 +67,7 @@ class JoinerTaskCreator: public TaskCreator, public CorePlugin{
Parameters*m_parameters;
StaticVector*m_outbox;
int*m_slaveMode;
- vector<vector<Kmer> >*m_paths;
+ vector<GraphPath>*m_paths;
vector<PathHandle>*m_pathIdentifiers;
set<PathHandle>*m_eliminated;
@@ -67,7 +76,7 @@ class JoinerTaskCreator: public TaskCreator, public CorePlugin{
WorkerHandle m_currentWorkerIdentifier;
bool m_reverseStrand;
- vector<vector<Kmer> >*m_newPaths;
+ vector<GraphPath>*m_newPaths;
// fast run parameters
bool m_previouslyDone;
@@ -75,9 +84,9 @@ class JoinerTaskCreator: public TaskCreator, public CorePlugin{
public:
void constructor( VirtualProcessor*virtualProcessor,StaticVector*outbox,
- RingAllocator*outboxAllocator,int*mode,Parameters*parameters,vector<vector<Kmer> >*paths,vector<PathHandle>*pathIdentifiers,
+ RingAllocator*outboxAllocator,int*mode,Parameters*parameters,vector<GraphPath>*paths,vector<PathHandle>*pathIdentifiers,
set<PathHandle>*eliminated,VirtualCommunicator*virtualCommunicator,
- vector<vector<Kmer> >*newPaths
+ vector<GraphPath>*newPaths
);
void call_RAY_SLAVE_MODE_FINISH_FUSIONS();
diff --git a/code/plugin_JoinerTaskCreator/JoinerWorker.cpp b/code/JoinerTaskCreator/JoinerWorker.cpp
similarity index 83%
rename from code/plugin_JoinerTaskCreator/JoinerWorker.cpp
rename to code/JoinerTaskCreator/JoinerWorker.cpp
index f59fb5d..081ebd9 100644
--- a/code/plugin_JoinerTaskCreator/JoinerWorker.cpp
+++ b/code/JoinerTaskCreator/JoinerWorker.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,8 +19,8 @@
*/
+#include "JoinerWorker.h"
-#include <plugin_JoinerTaskCreator/JoinerWorker.h>
#include <iostream>
using namespace std;
@@ -64,7 +64,8 @@ void JoinerWorker::work(){
assert(m_position < (int)m_path->size());
#endif
- Kmer kmer=m_path->at(m_position);
+ Kmer kmer;
+ m_path->at(m_position,&kmer);
if(m_reverseStrand)
kmer=kmer.complementVertex(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
@@ -117,7 +118,9 @@ void JoinerWorker::work(){
}else if(m_receivedNumberOfPaths && m_pathIndex < m_numberOfPaths){
/* request a path */
if(!m_requestedPath){
- Kmer kmer=m_path->at(m_position);
+ Kmer kmer;
+ m_path->at(m_position,&kmer);
+
if(m_reverseStrand){
kmer=kmer.complementVertex(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
}
@@ -126,6 +129,7 @@ void JoinerWorker::work(){
m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
int elementsPerQuery=m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_ASK_VERTEX_PATH);
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(elementsPerQuery);
+
int outputPosition=0;
kmer.pack(message,&outputPosition);
message[outputPosition++]=m_pathIndex;
@@ -209,7 +213,7 @@ void JoinerWorker::work(){
PathHandle hitName=m_hitNames[m_hitIterator];
Rank destination=getRankFromPathUniqueId(hitName);
- message[0]=hitName;
+ message[0]=hitName.getValue();
Message aMessage(message,1,destination,
RAY_MPI_TAG_GET_PATH_LENGTH,m_parameters->getRank());
@@ -223,6 +227,7 @@ void JoinerWorker::work(){
PathHandle hitName=m_hitNames[m_hitIterator];
int length=response[0];
+
if(m_parameters->hasOption("-debug-fusions2")){
cout<<"received length, value= "<<length<<endl;
}
@@ -307,7 +312,12 @@ void JoinerWorker::work(){
if(localScore > bestSelfScore){
if(localScore >= relevantScore){
- cout<<"New best score for self (handle: "<<m_identifier<<", length: "<<m_path->size()<<") with "<<localScore<<", other handle: "<<otherIdentifier<<endl;
+
+ if(m_parameters->hasOption("-debug-fusions")){
+ cout<<"New best score for self (handle: ";
+ cout<<m_identifier<<", length: "<<m_path->size()<<") with ";
+ cout<<localScore<<", other handle: "<<otherIdentifier<<endl;
+ }
}
m_minPositionOnSelf[otherIdentifier]=start;
@@ -350,7 +360,11 @@ void JoinerWorker::work(){
if(localScore > bestOtherScore){
if(localScore >= relevantScore){
- cout<<"New best score for other (handle: "<<otherIdentifier<<", length: "<<m_hitLengths[otherIdentifier]<<") with "<<localScore<<", other handle: "<<otherIdentifier<<endl;
+ if(m_parameters->hasOption("-debug-fusions")){
+ cout<<"New best score for other (handle: "<<otherIdentifier;
+ cout<<", length: "<<m_hitLengths[otherIdentifier]<<") with ";
+ cout<<localScore<<", other handle: "<<otherIdentifier<<endl;
+ }
}
m_minPosition[otherIdentifier]=start;
@@ -522,7 +536,8 @@ Also, don't do it if the matching ratios are below 10%.
// do nothing with this.
m_isDone=true;
- cout<<"Notice: number of hits is not 1: "<<numberOfHits<<" fetched hits."<<endl;
+ if(m_parameters->hasOption("-debug-fusions"))
+ cout<<"Notice: number of hits is not 1: "<<numberOfHits<<" fetched hits."<<endl;
}
/* gather all the vertices of the hit and try to join them */
@@ -538,7 +553,7 @@ Also, don't do it if the matching ratios are below 10%.
int destination=getRankFromPathUniqueId(hitName);
- message[0]=hitName;
+ message[0]=hitName.getValue();
message[1]=m_hitPosition;
int elementsPerQuery=m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_GET_PATH_VERTEX);
@@ -553,7 +568,8 @@ Also, don't do it if the matching ratios are below 10%.
int position=0;
Kmer kmer;
kmer.unpack(&response,&position);
- m_hitVertices.push_back(kmer);
+
+ m_hitVertices.push_back(&kmer);
m_hitPosition++;
m_requestedHitVertex=false;
@@ -610,46 +626,65 @@ Also, don't do it if the matching ratios are below 10%.
* ---------------->
* */
if(selfSide==RIGHT_SIDE && otherSide == LEFT_SIDE){
- cout<<"VALID"<<endl;
+
+ if(m_parameters->hasOption("-debug-fusions"))
+ cout<<"VALID"<<endl;
- vector<Kmer> newPath;
+ GraphPath newPath;
+ newPath.setKmerLength(m_parameters->getWordSize());
/* we take directly the path */
if(!m_reverseStrand){
newPath=(*m_path);
}else{
/* we need the reverse complement path */
- vector<Kmer> rc;
+ GraphPath rc;
+ rc.setKmerLength(m_parameters->getWordSize());
+
for(int j=(*m_path).size()-1;j>=0;j--){
- rc.push_back((*m_path)[j].complementVertex(m_parameters->getWordSize(),
- m_parameters->getColorSpaceMode()));
+
+ Kmer theKmer;
+ (*m_path).at(j,&theKmer);
+ Kmer newKmer=theKmer.complementVertex(m_parameters->getWordSize(),
+ m_parameters->getColorSpaceMode());
+
+ rc.push_back(&newKmer);
}
newPath=rc;
}
/* other path is always forward strand */
for(int i=m_maxPosition[hitName]+1;i<(int)hitLength;i++){
- newPath.push_back(m_hitVertices.at(i));
+ Kmer otherKmer;
+ m_hitVertices.at(i,&otherKmer);
+ newPath.push_back(&otherKmer);
}
m_newPaths->push_back(newPath);
- cout<<"Created new path, length= "<<newPath.size()<<endl;
-
- cout<<"Received hit path data."<<endl;
- cout<<"Matches: "<<matches<<endl;
- cout<<"Self"<<endl;
- cout<<" Identifier: "<<m_identifier<<endl;
- cout<<" Strand: "<<m_reverseStrand<<endl;
- cout<<" Length: "<<m_path->size()<<endl;
- cout<<" Begin: "<<m_minPositionOnSelf[hitName]<<endl;
- cout<<" End: "<<m_maxPositionOnSelf[hitName]<<endl;
- cout<<"Hit"<<endl;
- cout<<" Identifier: "<<hitName<<endl;
- cout<<" Strand: 0"<<endl;
- cout<<" Length: "<<hitLength<<endl;
- cout<<" Begin: "<<m_minPosition[hitName]<<endl;
- cout<<" End: "<<m_maxPosition[hitName]<<endl;
+ if(m_parameters->hasOption("-debug-fusions")){
+ cout<<"Created new path, length= "<<newPath.size()<<endl;
+ }
+
+ if(m_parameters->hasOption("-debug-fusions")){
+ cout<<"Received hit path data."<<endl;
+ }
+
+ if(m_parameters->hasOption("-debug-fusions")){
+ cout<<"Matches: "<<matches<<endl;
+ cout<<"Self"<<endl;
+ cout<<" Identifier: "<<m_identifier<<endl;
+ cout<<" Strand: "<<m_reverseStrand<<endl;
+ cout<<" Length: "<<m_path->size()<<endl;
+ cout<<" Begin: "<<m_minPositionOnSelf[hitName]<<endl;
+ cout<<" End: "<<m_maxPositionOnSelf[hitName]<<endl;
+ cout<<"Hit"<<endl;
+ cout<<" Identifier: "<<hitName<<endl;
+ cout<<" Strand: 0"<<endl;
+ cout<<" Length: "<<hitLength<<endl;
+ cout<<" Begin: "<<m_minPosition[hitName]<<endl;
+ cout<<" End: "<<m_maxPosition[hitName]<<endl;
+ }
m_eliminated=true;
@@ -659,54 +694,68 @@ Also, don't do it if the matching ratios are below 10%.
* ------------>
* */
}else if(selfSide==LEFT_SIDE && otherSide == RIGHT_SIDE){
- cout<<"VALID"<<endl;
+
+ if(m_parameters->hasOption("-debug-fusions"))
+ cout<<"VALID"<<endl;
/* other path is always forward strand */
- vector<Kmer> newPath=m_hitVertices;
+ GraphPath newPath=m_hitVertices;
+ newPath.setKmerLength(m_parameters->getWordSize());
/* we push the forward path */
if(!m_reverseStrand){
for(int i=m_maxPositionOnSelf[hitName]+1;i<(int)m_path->size();i++){
- newPath.push_back(m_path->at(i));
+ Kmer aKmer;
+ m_path->at(i,&aKmer);
+ newPath.push_back(&aKmer);
}
/* we push the reverse path */
}else{
- vector<Kmer> rc;
+ GraphPath rc;
+ rc.setKmerLength(m_parameters->getWordSize());
for(int j=(*m_path).size()-1;j>=0;j--){
- rc.push_back((*m_path)[j].complementVertex(m_parameters->getWordSize(),
- m_parameters->getColorSpaceMode()));
+
+ Kmer otherKmer;
+ (*m_path).at(j,&otherKmer);
+ Kmer aKmer=otherKmer.complementVertex(m_parameters->getWordSize(),
+ m_parameters->getColorSpaceMode());
+ rc.push_back(&aKmer);
}
for(int i=m_maxPositionOnSelf[hitName]+1;i<(int)m_path->size();i++){
- newPath.push_back(rc.at(i));
+ Kmer otherKmer;
+ rc.at(i,&otherKmer);
+ newPath.push_back(&otherKmer);
}
}
m_newPaths->push_back(newPath);
- cout<<"Created new path, length= "<<newPath.size()<<endl;
-
- cout<<"Received hit path data."<<endl;
- cout<<"Matches: "<<matches<<endl;
- cout<<"Self"<<endl;
- cout<<" Identifier: "<<m_identifier<<endl;
- cout<<" Strand: "<<m_reverseStrand<<endl;
- cout<<" Length: "<<m_path->size()<<endl;
- cout<<" Begin: "<<m_minPositionOnSelf[hitName]<<endl;
- cout<<" End: "<<m_maxPositionOnSelf[hitName]<<endl;
- cout<<"Hit"<<endl;
- cout<<" Identifier: "<<hitName<<endl;
- cout<<" Strand: 0"<<endl;
- cout<<" Length: "<<hitLength<<endl;
- cout<<" Begin: "<<m_minPosition[hitName]<<endl;
- cout<<" End: "<<m_maxPosition[hitName]<<endl;
-
+ if(m_parameters->hasOption("-debug-fusions")){
+ cout<<"Created new path, length= "<<newPath.size()<<endl;
+
+ cout<<"Received hit path data."<<endl;
+ cout<<"Matches: "<<matches<<endl;
+ cout<<"Self"<<endl;
+ cout<<" Identifier: "<<m_identifier<<endl;
+ cout<<" Strand: "<<m_reverseStrand<<endl;
+ cout<<" Length: "<<m_path->size()<<endl;
+ cout<<" Begin: "<<m_minPositionOnSelf[hitName]<<endl;
+ cout<<" End: "<<m_maxPositionOnSelf[hitName]<<endl;
+ cout<<"Hit"<<endl;
+ cout<<" Identifier: "<<hitName<<endl;
+ cout<<" Strand: 0"<<endl;
+ cout<<" Length: "<<hitLength<<endl;
+ cout<<" Begin: "<<m_minPosition[hitName]<<endl;
+ cout<<" End: "<<m_maxPosition[hitName]<<endl;
+ }
m_eliminated=true;
}else{
- cout<<"INVALID"<<endl;
+ if(m_parameters->hasOption("-debug-fusions"))
+ cout<<"INVALID"<<endl;
}
m_isDone=true;
@@ -718,9 +767,9 @@ WorkerHandle JoinerWorker::getWorkerIdentifier(){
return m_workerIdentifier;
}
-void JoinerWorker::constructor(WorkerHandle number,vector<Kmer>*path,PathHandle identifier,bool reverseStrand,
+void JoinerWorker::constructor(WorkerHandle number,GraphPath*path,PathHandle identifier,bool reverseStrand,
VirtualCommunicator*virtualCommunicator,Parameters*parameters,RingAllocator*outboxAllocator,
-vector<vector<Kmer> >*newPaths,
+vector<GraphPath>*newPaths,
MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH,
MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
@@ -753,6 +802,8 @@ vector<vector<Kmer> >*newPaths,
cout<<"Spawned worker number "<<number<<endl;
cout<<" path "<<m_identifier<<" reverse "<<m_reverseStrand<<" length "<<m_path->size()<<endl;
}
+
+ m_hitVertices.setKmerLength(m_parameters->getWordSize());
}
bool JoinerWorker::isPathEliminated(){
diff --git a/code/plugin_JoinerTaskCreator/JoinerWorker.h b/code/JoinerTaskCreator/JoinerWorker.h
similarity index 84%
rename from code/plugin_JoinerTaskCreator/JoinerWorker.h
rename to code/JoinerTaskCreator/JoinerWorker.h
index d72dfe3..60e780f 100644
--- a/code/plugin_JoinerTaskCreator/JoinerWorker.h
+++ b/code/JoinerTaskCreator/JoinerWorker.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,10 +22,12 @@
#ifndef _JoinerWorker_H
#define _JoinerWorker_H
-#include <communication/VirtualCommunicator.h>
-#include <scheduling/Worker.h>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <application_core/Parameters.h>
+#include <code/SeedingData/PathHandle.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/scheduling/Worker.h>
#include <stdint.h>
#include <map>
@@ -45,7 +47,7 @@ class JoinerWorker: public Worker{
bool m_requestedNumberOfPaths;
WorkerHandle m_workerIdentifier;
bool m_isDone;
- vector<Kmer>*m_path;
+ GraphPath*m_path;
PathHandle m_identifier;
bool m_reverseStrand;
bool m_eliminated;
@@ -62,7 +64,7 @@ class JoinerWorker: public Worker{
map<PathHandle,int> m_hitLengths;
int m_hitIterator;
- vector<vector<Kmer> >*m_newPaths;
+ vector<GraphPath>*m_newPaths;
bool m_selectedHit;
int m_selectedHitIndex;
@@ -76,7 +78,7 @@ class JoinerWorker: public Worker{
map<PathHandle,vector<int> > m_selfPositions;
map<PathHandle,vector<int> > m_hitPositions;
- vector<Kmer> m_hitVertices;
+ GraphPath m_hitVertices;
int m_hitPosition;
bool m_requestedHitVertex;
@@ -84,9 +86,9 @@ class JoinerWorker: public Worker{
bool m_receivedPath;
bool m_requestedPath;
public:
- void constructor(WorkerHandle i,vector<Kmer>*path,PathHandle identifier,bool reverseStrand,
+ void constructor(WorkerHandle i,GraphPath*path,PathHandle identifier,bool reverseStrand,
VirtualCommunicator*virtualCommunicator,Parameters*parameters,RingAllocator*outboxAllocator,
-vector<vector<Kmer> >*newPaths,
+vector<GraphPath>*newPaths,
MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH,
MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
diff --git a/code/JoinerTaskCreator/Makefile b/code/JoinerTaskCreator/Makefile
new file mode 100644
index 0000000..1cb178d
--- /dev/null
+++ b/code/JoinerTaskCreator/Makefile
@@ -0,0 +1,4 @@
+JoinerTaskCreator-y += code/JoinerTaskCreator/JoinerWorker.o
+JoinerTaskCreator-y += code/JoinerTaskCreator/JoinerTaskCreator.o
+
+obj-y += $(JoinerTaskCreator-y)
diff --git a/code/plugin_KmerAcademyBuilder/BloomFilter.cpp b/code/KmerAcademyBuilder/BloomFilter.cpp
similarity index 93%
rename from code/plugin_KmerAcademyBuilder/BloomFilter.cpp
rename to code/KmerAcademyBuilder/BloomFilter.cpp
index 13e93ac..613bc08 100644
--- a/code/plugin_KmerAcademyBuilder/BloomFilter.cpp
+++ b/code/KmerAcademyBuilder/BloomFilter.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -18,9 +18,12 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_KmerAcademyBuilder/BloomFilter.h>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <memory/allocator.h>
+#include "BloomFilter.h"
+
+#include <code/KmerAcademyBuilder/Kmer.h>
+
+#include <RayPlatform/memory/allocator.h>
+
#include <iostream>
#ifdef ASSERT
#include <assert.h>
@@ -77,7 +80,6 @@ ULL means unsigned long long, it is necessary on some architectures
#ifdef ASSERT
assert(m_hashFunctions == 8);
- assert(m_bits % 64 == 0);
#endif
uint64_t requiredBytes=m_bits/8;
@@ -91,9 +93,11 @@ ULL means unsigned long long, it is necessary on some architectures
if(m_bits%64!=0)
required8Bytes++;
- m_bitmap=(uint64_t*)__Malloc(required8Bytes*sizeof(uint64_t), "RAY_MALLOC_TYPE_BLOOM_FILTER", false); /* about 62 MB of memory */
+ m_bitmap=(uint64_t*)__Malloc(required8Bytes*sizeof(uint64_t), "RAY_MALLOC_TYPE_BLOOM_FILTER", false);
cout<<"[BloomFilter] allocated "<<required8Bytes*sizeof(uint64_t)<<" bytes for table with "<<m_bits<<" bits"<<endl;
+
+#ifdef CONFIG_VERBOSE_BLOOM_FILTER
cout<<"[BloomFilter] hash numbers:";
for(int i=0;i<m_hashFunctions;i++){
@@ -102,6 +106,8 @@ ULL means unsigned long long, it is necessary on some architectures
cout<<dec<<endl;
+#endif /* CONFIG_VERBOSE_BLOOM_FILTER */
+
#ifdef ASSERT
assert(required8Bytes > 0);
assert(m_bitmap != NULL);
diff --git a/code/plugin_KmerAcademyBuilder/BloomFilter.h b/code/KmerAcademyBuilder/BloomFilter.h
similarity index 94%
rename from code/plugin_KmerAcademyBuilder/BloomFilter.h
rename to code/KmerAcademyBuilder/BloomFilter.h
index 71b1cf1..8c75191 100644
--- a/code/plugin_KmerAcademyBuilder/BloomFilter.h
+++ b/code/KmerAcademyBuilder/BloomFilter.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -21,8 +21,9 @@
#ifndef _BloomFilter_H
#define _BloomFilter_H
+#include "Kmer.h"
+
#include <stdint.h>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
/**
* Bloom filter implementation
diff --git a/code/plugin_KmerAcademyBuilder/Kmer.cpp b/code/KmerAcademyBuilder/Kmer.cpp
similarity index 67%
rename from code/plugin_KmerAcademyBuilder/Kmer.cpp
rename to code/KmerAcademyBuilder/Kmer.cpp
index badb233..8f2001f 100644
--- a/code/plugin_KmerAcademyBuilder/Kmer.cpp
+++ b/code/KmerAcademyBuilder/Kmer.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,14 +19,22 @@
*/
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <stdio.h>
-#include <assert.h>
+#include "Kmer.h"
+
+#include <code/Mock/common_functions.h>
+
+#include <RayPlatform/cryptography/crypto.h>
+
#include <string>
#include <fstream>
-#include <cryptography/crypto.h>
+#include <iostream>
+#include <ostream>
using namespace std;
+#include <string.h>
+#include <stdio.h>
+#include <assert.h>
+
bool Kmer::operator<(const Kmer&b)const{
for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
if(m_u64[i]<b.m_u64[i]){
@@ -47,11 +55,12 @@ Kmer::Kmer(){
Kmer::~Kmer(){
}
-int Kmer::getNumberOfU64(){
+int Kmer::getNumberOfU64() const {
+
return KMER_U64_ARRAY_SIZE;
}
-bool Kmer::isLower(Kmer*a){
+bool Kmer::isLower(Kmer*a)const{
for(int i=0;i<getNumberOfU64();i++){
if(getU64(i)<a->getU64(i)){
return true;
@@ -62,7 +71,7 @@ bool Kmer::isLower(Kmer*a){
return false;
}
-bool Kmer::isEqual(Kmer*a){
+bool Kmer::isEqual(Kmer*a)const{
for(int i=0;i<getNumberOfU64();i++){
if(getU64(i)!=a->getU64(i)){
return false;
@@ -71,7 +80,7 @@ bool Kmer::isEqual(Kmer*a){
return true;
}
-void Kmer::print(){
+void Kmer::print()const{
for(int j=0;j<getNumberOfU64();j++){
uint64_t a=getU64(j);
for(int k=63;k>=0;k-=2){
@@ -84,21 +93,21 @@ void Kmer::print(){
printf("\n");
}
-void Kmer::pack(MessageUnit*messageBuffer,int*messagePosition){
+void Kmer::pack(MessageUnit*messageBuffer,int*messagePosition)const{
for(int i=0;i<getNumberOfU64();i++){
messageBuffer[*messagePosition]=getU64(i);
(*messagePosition)++;
}
}
-void Kmer::unpack(MessageUnit*messageBuffer,int*messagePosition){
+void Kmer::unpack(const MessageUnit*messageBuffer,int*messagePosition){
for(int i=0;i<getNumberOfU64();i++){
setU64(i,messageBuffer[*messagePosition]);
(*messagePosition)++;
}
}
-void Kmer::unpack(vector<MessageUnit>*messageBuffer,int*messagePosition){
+void Kmer::unpack(const vector<MessageUnit>*messageBuffer,int*messagePosition){
for(int i=0;i<getNumberOfU64();i++){
setU64(i,(*messageBuffer)[*messagePosition]);
(*messagePosition)++;
@@ -129,11 +138,11 @@ bool Kmer::operator!=(const Kmer&b) const{
return false;
}
-char Kmer::getLastSymbol(int m_wordSize,bool color){
+char Kmer::getLastSymbol(int m_wordSize,bool color)const{
return codeToChar(getSecondSegmentLastCode(m_wordSize),color);
}
-uint8_t Kmer::getSecondSegmentLastCode(int w){
+uint8_t Kmer::getSecondSegmentLastCode(int w)const{
int bitPosition=2*w;
int chunkId=bitPosition/64;
int bitPositionInChunk=bitPosition%64;
@@ -143,7 +152,7 @@ uint8_t Kmer::getSecondSegmentLastCode(int w){
return (uint8_t)chunk;
}
-uint8_t Kmer::getFirstSegmentFirstCode(int w){
+uint8_t Kmer::getFirstSegmentFirstCode(int w)const{
// ATCAGTTGCAGTACTGCAATCTACG
// 0000000000000011100001100100000000000000000000000001011100100100
// 6 5 4 3 2 1 0
@@ -153,7 +162,7 @@ uint8_t Kmer::getFirstSegmentFirstCode(int w){
return a;
}
-int Kmer::vertexRank(int _size,int w,bool color){
+int Kmer::vertexRank(int _size,int w,bool color)const{
Kmer b=complementVertex(w,color);
if(isLower(&b))
b=*this;
@@ -164,7 +173,7 @@ int Kmer::vertexRank(int _size,int w,bool color){
* Get the outgoing edges
* one bit (1=yes, 0=no) per possible edge
*/
-vector<Kmer> Kmer::_getOutgoingEdges(uint8_t edges,int k){
+vector<Kmer> Kmer::getOutgoingEdges(uint8_t edges,int k)const{
vector<Kmer> b;
Kmer aTemplate;
aTemplate=*this;
@@ -208,7 +217,7 @@ vector<Kmer> Kmer::_getOutgoingEdges(uint8_t edges,int k){
* Get the ingoing edges
* one bit (1=yes, 0=no) per possible edge
*/
-vector<Kmer> Kmer::_getIngoingEdges(uint8_t edges,int k){
+vector<Kmer> Kmer::getIngoingEdges(uint8_t edges,int k)const{
vector<Kmer> b;
Kmer aTemplate;
aTemplate=*this;
@@ -272,7 +281,7 @@ vector<Kmer> Kmer::_getIngoingEdges(uint8_t edges,int k){
return b;
}
-uint64_t Kmer::hash_function_1(){
+uint64_t Kmer::hash_function_1()const{
#if KMER_U64_ARRAY_SIZE == 1
return uniform_hashing_function_1_64_64(getU64(0));
#else
@@ -286,7 +295,7 @@ uint64_t Kmer::hash_function_1(){
#endif
}
-uint64_t Kmer::hash_function_2(){
+uint64_t Kmer::hash_function_2()const{
#if KMER_U64_ARRAY_SIZE == 1
return uniform_hashing_function_2_64_64(getU64(0));
#else
@@ -300,10 +309,14 @@ uint64_t Kmer::hash_function_2(){
#endif
}
-void Kmer::convertToString(int kmerLength,bool color, char*buffer){
+void Kmer::convertToString(int kmerLength,bool color, char*buffer)const{
for(int p=0;p<kmerLength;p++){
int bitPosition=2*p;
int chunkId=p/32;
+
+#ifdef CONFIG_ASSERT
+ assert(chunkId < KMER_U64_ARRAY_SIZE);
+#endif
int bitPositionInChunk=(bitPosition%64);
uint64_t chunk=getU64(chunkId);
uint64_t j=(chunk<<(62-bitPositionInChunk))>>62; // clear the bits.
@@ -312,8 +325,8 @@ void Kmer::convertToString(int kmerLength,bool color, char*buffer){
buffer[kmerLength]='\0';
}
-string Kmer::idToWord(int wordSize,bool color){
- char a[300];
+string Kmer::idToWord(int wordSize,bool color)const{
+ char a[CONFIG_MAXKMERLENGTH+1];
convertToString(wordSize,color,a);
@@ -339,25 +352,25 @@ char codeToChar(uint8_t a,bool color){
switch(a){
case RAY_NUCLEOTIDE_A:
- return 'A';
+ return SYMBOL_A;
case RAY_NUCLEOTIDE_T:
- return 'T';
+ return SYMBOL_T;
case RAY_NUCLEOTIDE_C:
- return 'C';
+ return SYMBOL_C;
case RAY_NUCLEOTIDE_G:
- return 'G';
+ return SYMBOL_G;
}
- return 'A';
+ return SYMBOL_A;
}
-void Kmer::write(ofstream*f){
+void Kmer::write(ostream*f)const{
for(int i=0;i<getNumberOfU64();i++){
uint64_t a=getU64(i);
f->write((char*)&a,sizeof(uint64_t));
}
}
-void Kmer::read(ifstream*f){
+void Kmer::read(istream*f){
for(int i=0;i<getNumberOfU64();i++){
uint64_t a=0;
f->read((char*)&a,sizeof(uint64_t));
@@ -366,20 +379,23 @@ void Kmer::read(ifstream*f){
}
void Kmer::setU64(int i,uint64_t b){
- #ifdef ASSERT
+ #ifdef CONFIG_ASSERT
assert(i<KMER_U64_ARRAY_SIZE);
#endif
m_u64[i]=b;
}
-uint64_t Kmer::getU64(int i){
- #ifdef ASSERT
+uint64_t Kmer::getU64(int i)const{
+ #ifdef CONFIG_ASSERT
+ if(!(i < KMER_U64_ARRAY_SIZE)) {
+ cout << "Error i " << i << " KMER_U64_ARRAY_SIZE " << i << endl;
+ }
assert(i<KMER_U64_ARRAY_SIZE);
#endif
return m_u64[i];
}
-Kmer Kmer::complementVertex(int wordSize,bool colorSpace){
+Kmer Kmer::complementVertex(int wordSize,bool colorSpace)const{
Kmer output;
int bitPositionInOutput=0;
uint64_t mask=3;
@@ -403,20 +419,20 @@ Kmer Kmer::complementVertex(int wordSize,bool colorSpace){
return output;
}
-double Kmer::getGuanineCytosineProportion(int kmerLength,bool coloredMode){
- char buffer[300];
+double Kmer::getGuanineCytosineProportion(int kmerLength,bool coloredMode)const{
+ char buffer[CONFIG_MAXKMERLENGTH+1];
convertToString(kmerLength,coloredMode,buffer);
int count=0;
for(int i=0;i<kmerLength;i++){
- if(buffer[i]=='G' || buffer[i]=='C'){
+ if(buffer[i]==SYMBOL_G || buffer[i]==SYMBOL_C){
count++;
}
}
- #ifdef ASSERT
+ #ifdef CONFIG_ASSERT
assert(kmerLength!=0);
#endif
@@ -424,3 +440,97 @@ double Kmer::getGuanineCytosineProportion(int kmerLength,bool coloredMode){
return proportion;
}
+
+bool Kmer::canHaveParent(const Kmer*otherKmer,int kmerLength)const{
+ return otherKmer->canHaveChild(this,kmerLength);
+}
+
+bool Kmer::canHaveChild(const Kmer*otherKmer,int kmerLength)const{
+
+ string left=idToWord(kmerLength,false);
+ string right=otherKmer->idToWord(kmerLength,false);
+ right[kmerLength-1]='\0';
+
+ int match=0;
+
+ if(strcmp(left.c_str()+1,right.c_str()+0)!=match)
+ return false;
+ return true;
+}
+
+char Kmer::getSymbolAtPosition(int kmerLength,bool colored, int position)const{
+ int bitPosition=BITS_PER_NUCLEOTIDE*position;
+ int chunkId=position/(sizeof(uint64_t)*BITS_PER_BYTE/BITS_PER_NUCLEOTIDE);
+ int bitPositionInChunk=(bitPosition%(sizeof(uint64_t)*BITS_PER_BYTE));
+ uint64_t chunk=getU64(chunkId);
+ chunk<<=((sizeof(uint64_t)*BITS_PER_BYTE-BITS_PER_NUCLEOTIDE)-bitPositionInChunk);
+ chunk>>=(sizeof(uint64_t)*BITS_PER_BYTE-BITS_PER_NUCLEOTIDE); // clear the bits.
+ char symbol=codeToChar(chunk,colored);
+
+ return symbol;
+}
+
+int Kmer::load(const char * buffer) {
+
+ int elements = 0;
+
+ unpack((MessageUnit*) buffer, &elements);
+
+ return elements * sizeof(MessageUnit);
+}
+
+int Kmer::dump(char * buffer) const {
+
+ int elements = 0;
+
+ pack((MessageUnit*)buffer, &elements);
+
+ return elements * sizeof(MessageUnit);
+}
+
+int Kmer::getRequiredNumberOfBytes() const {
+ return getNumberOfU64() * sizeof(MessageUnit);
+}
+
+void Kmer::loadFromTextRepresentation(const char * text) {
+
+ Kmer value = wordId(text);
+
+ *this = value;
+}
+
+uint64_t Kmer::getHashValue1() const {
+
+ return hash_function_1();
+}
+
+uint64_t Kmer::getHashValue2() const {
+
+ return hash_function_2();
+}
+
+int getNumberOfNucleotides(int numberOfKmers, int kmerLength) {
+ return ( numberOfKmers==0 ) ? 0 : (numberOfKmers + kmerLength -1 );
+}
+
+uint64_t Kmer::getTwinHash1(int kmerLength, bool colorSpaceMode) const {
+
+ Kmer lowerKey;
+ getLowerKey(&lowerKey, kmerLength, colorSpaceMode);
+ uint64_t hash = lowerKey.getHashValue1();
+
+ return hash;
+}
+
+void Kmer::getLowerKey(Kmer * lower, int kmerLength, bool colorSpaceMode) const {
+
+ Kmer kmer = *this;
+
+ Kmer lowerKey = kmer.complementVertex(kmerLength, colorSpaceMode);
+
+ if(kmer < lowerKey){
+ lowerKey= kmer;
+ }
+
+ *lower = lowerKey;
+}
diff --git a/code/plugin_KmerAcademyBuilder/Kmer.h b/code/KmerAcademyBuilder/Kmer.h
similarity index 57%
rename from code/plugin_KmerAcademyBuilder/Kmer.h
rename to code/KmerAcademyBuilder/Kmer.h
index e958645..68a56c7 100644
--- a/code/plugin_KmerAcademyBuilder/Kmer.h
+++ b/code/KmerAcademyBuilder/Kmer.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,13 +22,16 @@
#ifndef _Kmer
#define _Kmer
-#include <application_core/constants.h>
+#include <code/Mock/constants.h>
+
+#include <RayPlatform/store/CarriageableItem.h>
+
+#include <RayPlatform/core/types.h>
+
#include <stdint.h>
-#include <core/types.h>
/* this header was missing, but the code compiled with clang++, gcc, intel, pgi, but not pathscale. pathscale was right */
#include <fstream>
-
#include <vector>
#ifdef ASSERT
#include <assert.h>
@@ -38,9 +41,9 @@ using namespace std;
/*
* Determine the number of uint64_t necessary to host
- * k-mers of maximum length MAXKMERLENGTH
+ * k-mers of maximum length CONFIG_MAXKMERLENGTH
*/
-#define KMER_REQUIRED_BITS (2*MAXKMERLENGTH)
+#define KMER_REQUIRED_BITS (2*CONFIG_MAXKMERLENGTH)
#define KMER_REQUIRED_BYTES (KMER_REQUIRED_BITS/8)
#define KMER_REQUIRED_BYTES_MODULO (KMER_REQUIRED_BITS%8)
#if KMER_REQUIRED_BYTES_MODULO
@@ -65,18 +68,26 @@ using namespace std;
*
* \author Sébastien Boisvert
*/
-class Kmer{
+class Kmer : public CarriageableItem {
+
+private:
/** the actual array of uint64_t */
uint64_t m_u64[KMER_U64_ARRAY_SIZE];
+
public:
Kmer();
~Kmer();
- bool isEqual(Kmer*a);
- bool isLower(Kmer*a);
- void print();
- void pack(MessageUnit *messageBuffer,int*messagePosition);
- void unpack(MessageUnit*messageBuffer,int*messagePosition);
- void unpack(vector<MessageUnit>*messageBuffer,int*messagePosition);
+ bool isEqual(Kmer*a)const;
+ bool isLower(Kmer*a)const;
+ void print()const;
+ void pack(MessageUnit *messageBuffer,int*messagePosition)const;
+ void unpack(const MessageUnit*messageBuffer,int*messagePosition);
+ void unpack(const vector<MessageUnit>*messageBuffer,int*messagePosition);
+
+ int load(const char * buffer);
+ int dump(char * buffer) const;
+ int getRequiredNumberOfBytes() const;
+
void operator=(const Kmer&b);
bool operator<(const Kmer&b)const;
bool operator!=(const Kmer&b)const;
@@ -84,52 +95,68 @@ public:
void setU64(int i,uint64_t b);
- uint64_t getU64(int i);
+ uint64_t getU64(int i)const;
+
+ int getNumberOfU64()const;
- int getNumberOfU64();
/*
* get the last letter of a uint64_t
*/
- char getLastSymbol(int w,bool color);
+ char getLastSymbol(int w,bool color)const;
+
+ char getSymbolAtPosition(int kmerLength,bool colored, int position)const;
/*
* complement a vertex, and return another one
*/
- Kmer complementVertex(int wordSize,bool colorSpace);
-
+ Kmer complementVertex(int wordSize,bool colorSpace)const;
+
/*
* use mini distant segments here.
*/
- uint8_t getFirstSegmentFirstCode(int w);
- uint8_t getSecondSegmentLastCode(int w);
- int vertexRank(int _size,int w,bool color);
+ uint8_t getFirstSegmentFirstCode(int w)const;
+ uint8_t getSecondSegmentLastCode(int w)const;
+ Rank vertexRank(int _size,int w,bool color)const;
/**
* get the outgoing Kmer objects for a Kmer a having edges and
* a k-mer length k
+ *
+ * TODO: vector<Kmer>* should be a output parameter
*/
- vector<Kmer> _getOutgoingEdges(uint8_t edges,int k);
+ vector<Kmer> getOutgoingEdges(uint8_t edges,int k)const;
/**
* get the ingoing Kmer objects for a Kmer a having edges and
* a k-mer length k
+ *
+ * TODO: vector<Kmer>* should be a output parameter
*/
- vector<Kmer> _getIngoingEdges(uint8_t edges,int k);
+ vector<Kmer> getIngoingEdges(uint8_t edges,int k)const;
/** hash 1 is used to distribute k-mers on MPI ranks */
- uint64_t hash_function_1();
+ uint64_t hash_function_1()const;
+ uint64_t getHashValue1() const;
/** hash 2 is used for double hashing in the hash tables */
- uint64_t hash_function_2();
+ uint64_t hash_function_2()const;
+ uint64_t getHashValue2() const;
/*
* transform a Kmer in a string
*/
- string idToWord(int wordSize,bool color);
+ string idToWord(int wordSize,bool color)const;
+
+ void write(ostream*f)const;
+ void read(istream*f);
- void write(ofstream*f);
- void read(ifstream*f);
+ void convertToString(int kmerLength,bool coloredMode,char*buffer)const;
+ double getGuanineCytosineProportion(int kmerLength,bool coloredMode)const;
- void convertToString(int kmerLength,bool coloredMode,char*buffer);
- double getGuanineCytosineProportion(int kmerLength,bool coloredMode);
+ bool canHaveChild(const Kmer*otherKmer,int kmerLength)const;
+ bool canHaveParent(const Kmer*otherKmer,int kmerLength)const;
+
+ void loadFromTextRepresentation(const char * text);
+ uint64_t getTwinHash1(int kmerLength, bool colorSpaceMode) const;
+ void getLowerKey(Kmer * kmer, int kmerLength, bool colorSpaceMode) const;
}ATTRIBUTE_PACKED;
@@ -138,8 +165,7 @@ public:
*/
char codeToChar(uint8_t a,bool color);
-inline int getNumberOfNucleotides(int numberOfKmers,int kmerLength){
- return ( numberOfKmers==0 ) ? 0 : (numberOfKmers + kmerLength -1 );
-}
+int getNumberOfNucleotides(int numberOfKmers,int kmerLength);
+
#endif
diff --git a/code/plugin_KmerAcademyBuilder/KmerAcademyBuilder.cpp b/code/KmerAcademyBuilder/KmerAcademyBuilder.cpp
similarity index 91%
rename from code/plugin_KmerAcademyBuilder/KmerAcademyBuilder.cpp
rename to code/KmerAcademyBuilder/KmerAcademyBuilder.cpp
index 7c33a98..cb394ae 100644
--- a/code/plugin_KmerAcademyBuilder/KmerAcademyBuilder.cpp
+++ b/code/KmerAcademyBuilder/KmerAcademyBuilder.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,26 +19,24 @@
*/
-#include <application_core/constants.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <plugin_KmerAcademyBuilder/KmerAcademyBuilder.h>
+#include "KmerAcademyBuilder.h"
+
+#include <code/Mock/constants.h>
+#include <code/Mock/common_functions.h>
+
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/structures/StaticVector.h>
+
#include <assert.h>
-#include <communication/Message.h>
#include <time.h>
-#include <structures/StaticVector.h>
-#include <application_core/common_functions.h>
#include <fstream>
-
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
__CreatePlugin(KmerAcademyBuilder);
- /**/
- /**/
-__CreateSlaveModeAdapter(KmerAcademyBuilder,RAY_SLAVE_MODE_ADD_VERTICES); /**/
- /**/
- /**/
+__CreateSlaveModeAdapter(KmerAcademyBuilder,RAY_SLAVE_MODE_ADD_VERTICES);
using namespace std;
@@ -59,7 +57,7 @@ void KmerAcademyBuilder::call_RAY_SLAVE_MODE_ADD_VERTICES(){
if(m_parameters->hasCheckpoint("GenomeGraph")){
cout<<"Rank "<<m_parameters->getRank()<<": checkpoint GenomeGraph exists, not counting k-mers."<<endl;
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_KMER_ACADEMY_DISTRIBUTED,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_finished=true;
return;
}
@@ -81,15 +79,15 @@ void KmerAcademyBuilder::call_RAY_SLAVE_MODE_ADD_VERTICES(){
return;
}
- if(m_mode_send_vertices_sequence_id%10000==0 &&m_mode_send_vertices_sequence_id_position==0
- &&m_mode_send_vertices_sequence_id<(int)m_myReads->size()){
+ if(m_mode_send_vertices_sequence_id%100000==0 &&m_mode_send_vertices_sequence_id_position==0
+ &&m_mode_send_vertices_sequence_id<(int)m_myReads->size()){
+
string reverse="";
if(m_reverseComplementVertex==true){
reverse="(reverse complement) ";
}
printf("Rank %i is counting k-mers in sequence reads %s[%i/%i]\n",m_parameters->getRank(),
reverse.c_str(),(int)m_mode_send_vertices_sequence_id+1,(int)m_myReads->size());
- fflush(stdout);
m_derivative.addX(m_mode_send_vertices_sequence_id);
m_derivative.printStatus(SLAVE_MODES[RAY_SLAVE_MODE_ADD_VERTICES],RAY_SLAVE_MODE_ADD_VERTICES);
@@ -105,11 +103,10 @@ void KmerAcademyBuilder::call_RAY_SLAVE_MODE_ADD_VERTICES(){
#endif
Message aMessage(NULL,0, MASTER_RANK,RAY_MPI_TAG_KMER_ACADEMY_DISTRIBUTED,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_finished=true;
printf("Rank %i is counting k-mers in sequence reads [%i/%i] (completed)\n",
m_parameters->getRank(),(int)m_mode_send_vertices_sequence_id,(int)m_myReads->size());
- fflush(stdout);
m_bufferedData.showStatistics(m_parameters->getRank());
@@ -136,7 +133,7 @@ void KmerAcademyBuilder::call_RAY_SLAVE_MODE_ADD_VERTICES(){
return;
}
- char memory[MAXKMERLENGTH+1];
+ char memory[CONFIG_MAXKMERLENGTH+1];
int maximumPosition=len-m_parameters->getWordSize()+1;
#ifdef ASSERT
@@ -184,8 +181,9 @@ void KmerAcademyBuilder::call_RAY_SLAVE_MODE_ADD_VERTICES(){
* only one of them.
*/
- Rank rankToFlush=kmerToSend.hash_function_1()%m_parameters->getSize();
-
+ Rank rankToFlush=kmerToSend.vertexRank(m_parameters->getSize(),m_parameters->getWordSize(),
+ m_parameters->getColorSpaceMode());
+
for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
m_bufferedData.addAt(rankToFlush,kmerToSend.getU64(i));
}
@@ -315,4 +313,6 @@ void KmerAcademyBuilder::resolveSymbols(ComputeCore*core){
__BindPlugin(KmerAcademyBuilder);
+ __BindAdapter(KmerAcademyBuilder,RAY_SLAVE_MODE_ADD_VERTICES);
+
}
diff --git a/code/plugin_KmerAcademyBuilder/KmerAcademyBuilder.h b/code/KmerAcademyBuilder/KmerAcademyBuilder.h
similarity index 77%
rename from code/plugin_KmerAcademyBuilder/KmerAcademyBuilder.h
rename to code/KmerAcademyBuilder/KmerAcademyBuilder.h
index 065b3cf..80c5939 100644
--- a/code/plugin_KmerAcademyBuilder/KmerAcademyBuilder.h
+++ b/code/KmerAcademyBuilder/KmerAcademyBuilder.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,23 +22,27 @@
#ifndef _KmerAcademyBuilder
#define _KmerAcademyBuilder
-#include <structures/StaticVector.h>
-#include <communication/BufferedData.h>
-#include <plugin_VerticesExtractor/GridTable.h>
-#include <profiling/Derivative.h>
-#include <application_core/Parameters.h>
-#include <application_core/common_functions.h>
-#include <plugin_SequencesLoader/ArrayOfReads.h>
-#include <communication/Message.h>
-#include <profiling/Profiler.h>
-#include <memory/RingAllocator.h>
-#include <plugin_SequencesLoader/Read.h>
-#include <core/ComputeCore.h>
+#include <code/VerticesExtractor/GridTable.h>
+#include <code/Mock/Parameters.h>
+#include <code/Mock/common_functions.h>
+#include <code/SequencesLoader/ArrayOfReads.h>
+#include <code/SequencesLoader/Read.h>
+
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/communication/BufferedData.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/profiling/Derivative.h>
+#include <RayPlatform/profiling/Profiler.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <set>
#include <vector>
using namespace std;
+__DeclarePlugin(KmerAcademyBuilder);
+
+__DeclareSlaveModeAdapter(KmerAcademyBuilder,RAY_SLAVE_MODE_ADD_VERTICES);
/*
* Any MPI rank has some reads to process.
@@ -50,6 +54,8 @@ using namespace std;
*/
class KmerAcademyBuilder : public CorePlugin{
+ __AddAdapter(KmerAcademyBuilder,RAY_SLAVE_MODE_ADD_VERTICES);
+
MessageTag RAY_MPI_TAG_KMER_ACADEMY_DISTRIBUTED;
MessageTag RAY_MPI_TAG_VERTICES_DATA_REPLY;
MessageTag RAY_MPI_TAG_VERTICES_DATA;
diff --git a/code/KmerAcademyBuilder/Makefile b/code/KmerAcademyBuilder/Makefile
new file mode 100644
index 0000000..543d9ad
--- /dev/null
+++ b/code/KmerAcademyBuilder/Makefile
@@ -0,0 +1,5 @@
+KmerAcademyBuilder-y += code/KmerAcademyBuilder/KmerAcademyBuilder.o
+KmerAcademyBuilder-y += code/KmerAcademyBuilder/BloomFilter.o
+KmerAcademyBuilder-y += code/KmerAcademyBuilder/Kmer.o
+
+obj-y += $(KmerAcademyBuilder-y)
diff --git a/code/plugin_Library/Library.cpp b/code/Library/Library.cpp
similarity index 83%
rename from code/plugin_Library/Library.cpp
rename to code/Library/Library.cpp
index 94fedb8..ebd0aed 100644
--- a/code/plugin_Library/Library.cpp
+++ b/code/Library/Library.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,34 +14,31 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
//#define GUILLIMIN_BUG
-#include <plugin_SequencesIndexer/ReadAnnotation.h>
-#include <plugin_Library/Library.h>
-#include <communication/mpi_tags.h>
+#include "Library.h"
+
+#include <code/SequencesIndexer/ReadAnnotation.h>
+#include <code/Mock/common_functions.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/communication/mpi_tags.h>
+#include <RayPlatform/core/OperatingSystem.h>
+
#include <sstream>
-#include <core/OperatingSystem.h>
-#include <application_core/common_functions.h>
#include <assert.h>
-#include <application_core/Parameters.h>
-
+using namespace std;
__CreatePlugin(Library);
- /**/
-__CreateMasterModeAdapter(Library,RAY_MASTER_MODE_UPDATE_DISTANCES); /**/
- /**/
-__CreateSlaveModeAdapter(Library,RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION); /**/
-__CreateSlaveModeAdapter(Library,RAY_SLAVE_MODE_SEND_LIBRARY_DISTANCES); /**/
- /**/
- /**/
-
-using namespace std;
+__CreateMasterModeAdapter(Library,RAY_MASTER_MODE_UPDATE_DISTANCES);
+__CreateSlaveModeAdapter(Library,RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION);
+__CreateSlaveModeAdapter(Library,RAY_SLAVE_MODE_SEND_LIBRARY_DISTANCES);
/* send the information to all ranks */
void Library::call_RAY_MASTER_MODE_UPDATE_DISTANCES(){
@@ -61,7 +58,7 @@ void Library::call_RAY_MASTER_MODE_UPDATE_DISTANCES(){
for(int i=0;i<m_size;i++){
Message aMessage(message,outputPosition,i,RAY_MPI_TAG_UPDATE_LIBRARY_INFORMATION,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
m_informationSent=true;
@@ -69,7 +66,7 @@ void Library::call_RAY_MASTER_MODE_UPDATE_DISTANCES(){
/** wait for a reply */
}else if(m_inbox->size()>0 && m_inbox->at(0)->getTag()==RAY_MPI_TAG_UPDATE_LIBRARY_INFORMATION_REPLY){
m_ranksThatReplied++;
-
+
/** when everyone replied, we can proceed with the next library */
if(m_ranksThatReplied==m_parameters->getSize()){
m_currentLibrary++;
@@ -88,6 +85,62 @@ void Library::call_RAY_MASTER_MODE_UPDATE_DISTANCES(){
}
}
+bool Library::hasLibraryCheckpoint(){
+ return m_parameters->hasCheckpoint("PairedLibraries");
+}
+
+void Library::readLibraryCheckpoint(){
+ ifstream f(m_parameters->getCheckpointFile("PairedLibraries").c_str());
+ cout<<"Rank "<<m_parameters->getRank()<<" is reading checkpoint PairedLibraries"<<endl;
+
+ uint32_t entries=0;
+ f.read((char*)&entries,sizeof(uint32_t));
+
+ #ifdef CONFIG_ASSERT
+ assert(m_libraryDistances.size() == 0);
+ assert(m_detectedDistances ==0);
+ #endif
+
+ while(entries--){
+
+ uint32_t library=0;
+ uint32_t distance=0;
+ uint32_t count=0;
+
+
+ f.read((char*)&library,sizeof(uint32_t));
+ f.read((char*)&distance,sizeof(uint32_t));
+ f.read((char*)&count,sizeof(uint32_t));
+
+ if(count == 0)
+ continue;
+
+#ifdef CONFIG_ASSERT
+
+ assert(count > 0);
+#endif
+
+#ifdef CONFIG_ASSERT
+ if(m_libraryDistances.count(library)>0){
+ if(m_libraryDistances[library].count(distance) !=0 ) {
+
+ cout << "Error: expected 0, got " << m_libraryDistances[library][distance] << " for library " << library;
+ cout << " and distance " << distance << endl;
+ }
+
+
+ assert(m_libraryDistances[library].count(distance)==0);
+ }
+#endif
+
+ m_libraryDistances[library][distance]=count;
+
+ m_detectedDistances+=count;
+ }
+
+ f.close();
+}
+
void Library::call_RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION(){
if(!m_initiatedIterator){
m_SEEDING_i=0;
@@ -101,6 +154,16 @@ void Library::call_RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION(){
return;
}
+/*
+ * If we have checkpoints, we read them right away and
+ * skip this code path.
+ */
+
+ if(hasLibraryCheckpoint()){
+ readLibraryCheckpoint();
+ completeSlaveMode();
+ return;
+ }
m_virtualCommunicator->resetCounters();
}
@@ -157,13 +220,13 @@ void Library::call_RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION(){
}
#endif
-
+
/* there is a strange bug that is avoided by waiting an
extra tick before doing actual stuff.
if we don't wait for m_outbox to be flushed, we get
2 messages in the outbox and something strange happens
-
- the bug does not happen on
+
+ the bug does not happen on
- Mammouth Parallel II;
- colosse
@@ -214,7 +277,7 @@ void Library::call_RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION(){
updateStates();
// add one worker to active workers
- // reason is that those already in the pool don't communicate anymore --
+ // reason is that those already in the pool don't communicate anymore --
// as for they need responses.
if(!m_virtualCommunicator->getGlobalPushedMessageStatus()&&m_activeWorkers.empty()){
// there is at least one worker to start
@@ -241,7 +304,7 @@ void Library::call_RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION(){
}
m_SEEDING_i++;
}else{
-
+
/* if there are no active workers and we failed to add
new workers above, then we need to flush something right
now
@@ -254,7 +317,7 @@ void Library::call_RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION(){
if(m_outbox->size()>0){
cout<<endl;
cout<<"Produced messages with forceFlush()"<<endl;
-
+
cout<<"Inbox: "<<m_inbox->size()<<" Outbox: "<<m_outbox->size()<<endl;
for(int p=0;p<m_outbox->size();p++){
@@ -282,10 +345,11 @@ void Library::call_RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION(){
if(m_completedJobs==(int)m_seedingData->m_SEEDING_seeds.size()){
printf("Rank %i detected %i library lengths\n",getRank(),m_detectedDistances);
- fflush(stdout);
printf("Rank %i is calculating library lengths [%i/%i] (completed)\n",getRank(),(int)m_seedingData->m_SEEDING_seeds.size(),(int)m_seedingData->m_SEEDING_seeds.size());
- fflush(stdout);
-
+
+ if(mustWriteLibraryCheckpoint())
+ writeLibraryCheckpoint();
+
completeSlaveMode();
}
@@ -312,6 +376,51 @@ void Library::call_RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION(){
#endif
}
+bool Library::mustWriteLibraryCheckpoint(){
+ return m_parameters->writeCheckpoints() && !m_parameters->hasCheckpoint("PairedLibraries");
+}
+
+void Library::writeLibraryCheckpoint(){
+ ostringstream buffer;
+ ostringstream name;
+ name << m_parameters->getCheckpointFile("PairedLibraries").c_str();
+ FILE* file = fopen(name.str().c_str(), "w");
+ uint32_t entries=0;
+
+ cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint PairedLibraries"<<endl;
+
+// count the entries
+ for(map<int,map<int,int> >::iterator i=m_libraryDistances.begin();
+ i!=m_libraryDistances.end();i++){
+
+ entries+=i->second.size();
+ }
+
+ buffer.write((char*)&entries, sizeof(uint32_t));
+
+// write the entries
+
+ for(map<int,map<int,int> >::iterator i=m_libraryDistances.begin();
+ i!=m_libraryDistances.end();i++){
+
+ int library=i->first;
+
+ for(map<int,int>::iterator j=i->second.begin();
+ j!=i->second.end();j++){
+
+ int distance=j->first;
+ int count=j->second;
+
+ buffer.write((char*)&library, sizeof(uint32_t));
+ buffer.write((char*)&distance, sizeof(uint32_t));
+ buffer.write((char*)&count, sizeof(uint32_t));
+ flushFileOperationBuffer_FILE(false, &buffer, file, CONFIG_FILE_IO_BUFFER_SIZE);
+ }
+ }
+ flushFileOperationBuffer_FILE(true, &buffer, file, CONFIG_FILE_IO_BUFFER_SIZE);
+ fclose(file);
+}
+
void Library::constructor(int m_rank,StaticVector*m_outbox,RingAllocator*m_outboxAllocator,ExtensionData*m_ed,
int m_size,
TimePrinter*m_timePrinter,int*m_mode,int*m_master_mode,
@@ -384,7 +493,7 @@ void Library::call_RAY_SLAVE_MODE_SEND_LIBRARY_DISTANCES(){
return;
}
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES_FINISHED,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
m_bufferedData.showStatistics(m_parameters->getRank());
}else if(!m_libraryIndexInitiated){
@@ -421,9 +530,9 @@ void Library::updateStates(){
m_activeWorkers.erase(workerId);
m_aliveWorkers.erase(workerId);
- if(m_completedJobs%10==0){
+
+ if(m_completedJobs%1000==0){
printf("Rank %i is calculating library lengths [%i/%i]\n",m_parameters->getRank(),m_completedJobs+1,(int)m_seedingData->m_SEEDING_seeds.size());
- fflush(stdout);
}
m_completedJobs++;
@@ -454,12 +563,11 @@ void Library::updateStates(){
void Library::completeSlaveMode(){
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION_IS_DONE,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
m_allocator.clear();
printf("Rank %i: peak number of workers: %i, maximum: %i\n",m_rank,m_maximumWorkers,m_maximumAliveWorkers);
- fflush(stdout);
m_virtualCommunicator->printStatistics();
if(m_parameters->showMemoryUsage()){
@@ -520,6 +628,10 @@ void Library::resolveSymbols(ComputeCore*core){
core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_UPDATE_DISTANCES,RAY_MASTER_MODE_TRIGGER_FUSIONS);
__BindPlugin(Library);
-}
+ __BindAdapter(Library,RAY_MASTER_MODE_UPDATE_DISTANCES);
+ __BindAdapter(Library,RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION);
+ __BindAdapter(Library,RAY_SLAVE_MODE_SEND_LIBRARY_DISTANCES);
+
+}
diff --git a/code/plugin_Library/Library.h b/code/Library/Library.h
similarity index 73%
rename from code/plugin_Library/Library.h
rename to code/Library/Library.h
index df043ba..e05b7d5 100644
--- a/code/plugin_Library/Library.h
+++ b/code/Library/Library.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,24 +22,31 @@
#ifndef _Library
#define _Library
-#include <communication/BufferedData.h>
-#include <plugin_SeedExtender/ExtensionData.h>
-#include <application_core/common_functions.h>
-#include <structures/StaticVector.h>
-#include <profiling/TimePrinter.h>
-#include <plugin_SeedExtender/ReadFetcher.h>
-#include <communication/VirtualCommunicator.h>
-#include <application_core/Parameters.h>
-#include <plugin_SeedingData/SeedingData.h>
-#include <memory/RingAllocator.h>
-#include <plugin_Library/LibraryWorker.h>
-#include <handlers/SlaveModeHandler.h>
-#include <handlers/MasterModeHandler.h>
-#include <core/ComputeCore.h>
+#include "LibraryWorker.h"
+
+#include <code/SeedExtender/ExtensionData.h>
+#include <code/SeedExtender/ReadFetcher.h>
+#include <code/Mock/common_functions.h>
+#include <code/Mock/Parameters.h>
+#include <code/SeedingData/SeedingData.h>
+
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/profiling/TimePrinter.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/communication/BufferedData.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/handlers/MasterModeHandler.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <map>
using namespace std;
+__DeclarePlugin(Library);
+
+__DeclareMasterModeAdapter(Library,RAY_MASTER_MODE_UPDATE_DISTANCES);
+__DeclareSlaveModeAdapter(Library,RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION);
+__DeclareSlaveModeAdapter(Library,RAY_SLAVE_MODE_SEND_LIBRARY_DISTANCES);
/*
* This class computes the average outer distances
@@ -51,6 +58,10 @@ using namespace std;
*/
class Library : public CorePlugin{
+ __AddAdapter(Library,RAY_MASTER_MODE_UPDATE_DISTANCES);
+ __AddAdapter(Library,RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION);
+ __AddAdapter(Library,RAY_SLAVE_MODE_SEND_LIBRARY_DISTANCES);
+
MessageTag RAY_MPI_TAG_ASK_LIBRARY_DISTANCES_FINISHED;
MessageTag RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION_IS_DONE;
MessageTag RAY_MPI_TAG_LIBRARY_DISTANCE;
@@ -113,6 +124,11 @@ class Library : public CorePlugin{
void completeSlaveMode();
+ void readLibraryCheckpoint();
+ bool hasLibraryCheckpoint();
+ void writeLibraryCheckpoint();
+ bool mustWriteLibraryCheckpoint();
+
public:
Library();
void allocateBuffers();
diff --git a/code/plugin_Library/LibraryPeakFinder.cpp b/code/Library/LibraryPeakFinder.cpp
similarity index 97%
rename from code/plugin_Library/LibraryPeakFinder.cpp
rename to code/Library/LibraryPeakFinder.cpp
index 6990ee9..3ddaa05 100644
--- a/code/plugin_Library/LibraryPeakFinder.cpp
+++ b/code/Library/LibraryPeakFinder.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -18,8 +18,10 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_Library/LibraryPeakFinder.h>
-#include <core/statistics.h>
+#include "LibraryPeakFinder.h"
+
+#include <RayPlatform/core/statistics.h>
+
#include <math.h>
#include <iostream>
#include <stdint.h>
diff --git a/code/plugin_Library/LibraryPeakFinder.h b/code/Library/LibraryPeakFinder.h
similarity index 95%
rename from code/plugin_Library/LibraryPeakFinder.h
rename to code/Library/LibraryPeakFinder.h
index eb5b515..13dc2ed 100644
--- a/code/plugin_Library/LibraryPeakFinder.h
+++ b/code/Library/LibraryPeakFinder.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
diff --git a/code/plugin_Library/LibraryWorker.cpp b/code/Library/LibraryWorker.cpp
similarity index 92%
rename from code/plugin_Library/LibraryWorker.cpp
rename to code/Library/LibraryWorker.cpp
index 3e4a581..f8257c3 100644
--- a/code/plugin_Library/LibraryWorker.cpp
+++ b/code/Library/LibraryWorker.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,13 +19,16 @@
*/
-#include <application_core/constants.h>
-#include <plugin_Library/LibraryWorker.h>
-#include <communication/Message.h>
-#include <plugin_SeedingData/SeedingData.h>
+#include "LibraryWorker.h"
+
+#include <code/Mock/constants.h>
+#include <code/SeedingData/SeedingData.h>
+
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/communication/Message.h>
+
#include <map>
#include <assert.h>
-#include <communication/VirtualCommunicator.h>
using namespace std;
bool LibraryWorker::isDone(){
@@ -81,7 +84,10 @@ void LibraryWorker::work(){
assert(m_seedingData!=NULL);
assert(m_EXTENSION_currentPosition<(int)m_seedingData->m_SEEDING_seeds[m_SEEDING_i].size());
#endif
- Kmer*vertex=m_seedingData->m_SEEDING_seeds[m_SEEDING_i].at(m_EXTENSION_currentPosition);
+
+ Kmer theKmerObject;
+ m_seedingData->m_SEEDING_seeds[m_SEEDING_i].at(m_EXTENSION_currentPosition,&theKmerObject);
+ Kmer*vertex=&theKmerObject;
m_readFetcher.constructor(vertex,m_outboxAllocator,m_inbox,m_outbox,m_parameters,m_virtualCommunicator,
m_SEEDING_i,RAY_MPI_TAG_REQUEST_VERTEX_READS);
@@ -124,7 +130,7 @@ void LibraryWorker::work(){
bool isAutomatic=m_parameters->isAutomatic(library);
if(isAutomatic){
PathHandle uniqueReadIdentifier=getPathUniqueId(buffer[1],buffer[2]);
- SplayNode<ReadHandle,LibraryElement>*node=m_database.find(uniqueReadIdentifier,false);
+ SplayNode<ReadHandle,LibraryElement>*node=m_database.find(uniqueReadIdentifier.getValue(),false);
if(node!=NULL){
LibraryElement*element=node->getValue();
int rightStrandPosition=annotation.getPositionOnStrand();
diff --git a/code/plugin_Library/LibraryWorker.h b/code/Library/LibraryWorker.h
similarity index 84%
rename from code/plugin_Library/LibraryWorker.h
rename to code/Library/LibraryWorker.h
index 0fc7f72..28f8708 100644
--- a/code/plugin_Library/LibraryWorker.h
+++ b/code/Library/LibraryWorker.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,11 +22,15 @@
#ifndef _LibraryWorker
#define _LibraryWorker
-#include <plugin_SeedingData/SeedingData.h>
-#include <plugin_SeedExtender/ExtensionData.h>
-#include <communication/VirtualCommunicator.h>
-#include <memory/RingAllocator.h>
-#include <scheduling/Worker.h>
+#include <code/SequencesLoader/ReadHandle.h>
+#include <code/SeedingData/PathHandle.h>
+#include <code/SeedingData/SeedingData.h>
+#include <code/SeedExtender/ExtensionData.h>
+
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/scheduling/Worker.h>
+
#include <map>
#include <stdint.h>
using namespace std;
@@ -56,7 +60,7 @@ class LibraryWorker : public Worker {
MyAllocator*m_allocator;
VirtualCommunicator*m_virtualCommunicator;
- PathHandle m_SEEDING_i;
+ WorkerHandle m_SEEDING_i;
RingAllocator*m_outboxAllocator;
Parameters*m_parameters;
int m_EXTENSION_currentPosition;
diff --git a/code/Library/Makefile b/code/Library/Makefile
new file mode 100644
index 0000000..ac0d366
--- /dev/null
+++ b/code/Library/Makefile
@@ -0,0 +1,5 @@
+Library-y += code/Library/LibraryPeakFinder.o
+Library-y += code/Library/LibraryWorker.o
+Library-y += code/Library/Library.o
+
+obj-y += $(Library-y)
diff --git a/code/plugin_MachineHelper/MachineHelper.cpp b/code/MachineHelper/MachineHelper.cpp
similarity index 78%
rename from code/plugin_MachineHelper/MachineHelper.cpp
rename to code/MachineHelper/MachineHelper.cpp
index f9f3c9f..459d4ac 100644
--- a/code/plugin_MachineHelper/MachineHelper.cpp
+++ b/code/MachineHelper/MachineHelper.cpp
@@ -1,6 +1,7 @@
/*
- Ray
- Copyright (C) 2012 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+ Copyright (C) 2013 Charles Joly Beauparlant
http://DeNovoAssembler.SourceForge.Net/
@@ -14,58 +15,78 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_MachineHelper/MachineHelper.h>
-#include <communication/mpi_tags.h>
-#include <communication/Message.h>
-#include <plugin_CoverageGatherer/CoverageDistribution.h>
-#include <profiling/Profiler.h>
-#include <plugin_SeedExtender/Chooser.h>
+#include "MachineHelper.h"
+
+#include <code/CoverageGatherer/CoverageDistribution.h>
+#include <code/SeedExtender/Chooser.h>
+#include <code/SeedingData/GraphPath.h>
+
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/communication/mpi_tags.h>
+#include <RayPlatform/profiling/Profiler.h>
#include <map>
#include <sstream>
+using namespace std;
-__CreatePlugin(MachineHelper);
+#ifdef ASSERT
+#include <assert.h>
+#endif
- /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_LOAD_CONFIG); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_SEND_COVERAGE_VALUES); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_WRITE_KMERS); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_LOAD_SEQUENCES); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_VERTICE_DISTRIBUTION); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_GRAPH_BUILDING); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_PURGE_NULL_EDGES); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_INDEXING); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS_WITH_ANSWERS); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_SEEDING); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_SEEDING); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_DETECTION); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_ASK_DISTANCES); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_START_UPDATING_DISTANCES); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_EXTENSIONS); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_FUSIONS); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_FIRST_FUSIONS); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_START_FUSION_CYCLE); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_ASK_EXTENSIONS); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_SCAFFOLDER); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_KILL_RANKS); /**/
-__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_KILL_ALL_MPI_RANKS); /**/
- /**/
-__CreateSlaveModeAdapter(MachineHelper,RAY_SLAVE_MODE_WRITE_KMERS); /**/
-__CreateSlaveModeAdapter(MachineHelper,RAY_SLAVE_MODE_ASSEMBLE_WAVES); /**/
-__CreateSlaveModeAdapter(MachineHelper,RAY_SLAVE_MODE_SEND_EXTENSION_DATA); /**/
-__CreateSlaveModeAdapter(MachineHelper,RAY_SLAVE_MODE_DIE); /**/
- /**/
- /**/
+__CreatePlugin(MachineHelper);
-using namespace std;
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_LOAD_CONFIG);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_SEND_COVERAGE_VALUES);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_WRITE_KMERS);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_LOAD_SEQUENCES);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_VERTICE_DISTRIBUTION);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_GRAPH_BUILDING);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_PURGE_NULL_EDGES);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_INDEXING);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS_WITH_ANSWERS);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_SEEDING);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_SEEDING);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_DETECTION);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_ASK_DISTANCES);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_START_UPDATING_DISTANCES);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_EXTENSIONS);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_FUSIONS);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_FIRST_FUSIONS);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_START_FUSION_CYCLE);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_ASK_EXTENSIONS);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_SCAFFOLDER);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_KILL_RANKS);
+__CreateMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
+
+__CreateSlaveModeAdapter(MachineHelper,RAY_SLAVE_MODE_WRITE_KMERS);
+__CreateSlaveModeAdapter(MachineHelper,RAY_SLAVE_MODE_ASSEMBLE_WAVES);
+__CreateSlaveModeAdapter(MachineHelper,RAY_SLAVE_MODE_SEND_EXTENSION_DATA);
+__CreateSlaveModeAdapter(MachineHelper,RAY_SLAVE_MODE_DIE);
+
+__CreateMessageTagAdapter(MachineHelper,RAY_MPI_TAG_NOTIFY_ERROR);
+__CreateMessageTagAdapter(MachineHelper,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS);
+__CreateMessageTagAdapter(MachineHelper,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY);
+__CreateMessageTagAdapter(MachineHelper,RAY_MPI_TAG_ASK_EXTENSION_DATA);
+__CreateMessageTagAdapter(MachineHelper,RAY_MPI_TAG_EXTENSION_DATA_END);
+
+void MachineHelper::call_RAY_MPI_TAG_EXTENSION_DATA_END(Message*message){
+ m_ranksThatWroteContigs++;
+}
+/*
+ * This is the first upcall.
+ */
void MachineHelper::call_RAY_MASTER_MODE_LOAD_CONFIG(){
+ #ifdef ASSERT
+ assert(this!=NULL);
+ #endif
+
if(m_argc==2 && m_argv[1][0]!='-'){
ifstream f(m_argv[1]);
if(!f){
@@ -73,13 +94,19 @@ void MachineHelper::call_RAY_MASTER_MODE_LOAD_CONFIG(){
m_parameters->showUsage();
(*m_aborted)=true;
f.close();
+ m_switchMan->sendToAll(m_outbox,getRank(),RAY_MPI_TAG_NOTIFY_ERROR);
m_switchMan->setMasterMode(RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
return;
}
+ }else if(m_oldDirectoryExists){
+ m_switchMan->sendToAll(m_outbox,getRank(),RAY_MPI_TAG_NOTIFY_ERROR);
+ m_switchMan->setMasterMode(RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
+ return;
}
if(m_parameters->getError()){
(*m_aborted)=true;
+ m_switchMan->sendToAll(m_outbox,getRank(),RAY_MPI_TAG_NOTIFY_ERROR);
m_switchMan->setMasterMode(RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
return;
}
@@ -90,12 +117,14 @@ void MachineHelper::call_RAY_MASTER_MODE_LOAD_CONFIG(){
for(Rank i=0;i<m_parameters->getSize();i++){
Message aMessage(message,2,i,RAY_MPI_TAG_SET_WORD_SIZE,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
m_switchMan->setMasterMode(RAY_MASTER_MODE_TEST_NETWORK);
}
+
+
void MachineHelper::constructor(int argc,char**argv,Parameters*parameters,
SwitchMan*switchMan,RingAllocator*outboxAllocator,
StaticVector*outbox,bool*aborted,
@@ -104,7 +133,7 @@ SwitchMan*switchMan,RingAllocator*outboxAllocator,
int*numberOfRanksWithCoverageData,bool*reductionOccured,
ExtensionData*ed,FusionData*fusionData,
Profiler*profiler,NetworkTest*networkTest,SeedingData*seedingData,
-TimePrinter*timePrinter,SeedExtender*seedExtender,Scaffolder*scaffolder,MessagesHandler*messagesHandler,
+TimePrinter*timePrinter,SeedExtender*seedExtender,Scaffolder*scaffolder,
StaticVector*inbox,
OpenAssemblerChooser*oa, bool*isFinalFusion, BubbleData*bubbleData,bool*alive,
int*CLEAR_n,int*DISTRIBUTE_n,int*FINISH_n,Searcher*searcher,
@@ -120,6 +149,9 @@ VirtualCommunicator*virtualCommunicator,KmerAcademyBuilder*kmerAcademyBuilder,
int*ranksDoneAttachingReads,
SequencesLoader*sl,time_t*lastTime,bool*writeKmerInitialised,Partitioner*partitioner
){
+
+ m_oldDirectoryExists=false;
+
m_sl=sl;
m_lastTime=lastTime;
m_writeKmerInitialised=writeKmerInitialised;
@@ -163,7 +195,6 @@ SequencesLoader*sl,time_t*lastTime,bool*writeKmerInitialised,Partitioner*partiti
m_timePrinter=timePrinter;
m_seedExtender=seedExtender;
m_scaffolder=scaffolder;
- m_messagesHandler=messagesHandler;
m_profiler=profiler;
m_networkTest=networkTest;
m_seedingData=seedingData;
@@ -212,15 +243,18 @@ void MachineHelper::call_RAY_MASTER_MODE_SEND_COVERAGE_VALUES (){
if(m_parameters->writeCheckpoints() && !m_parameters->hasCheckpoint("CoverageDistribution")){
cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint CoverageDistribution"<<endl;
ofstream f(m_parameters->getCheckpointFile("CoverageDistribution").c_str());
+ ostringstream buffer;
int theSize=m_coverageDistribution->size();
- f.write((char*)&theSize,sizeof(int));
+ buffer.write((char*)&theSize, sizeof(int));
for(map<CoverageDepth,LargeCount>::iterator i=m_coverageDistribution->begin();i!=m_coverageDistribution->end();i++){
CoverageDepth coverage=i->first;
LargeCount count=i->second;
- f.write((char*)&coverage,sizeof(CoverageDepth));
- f.write((char*)&count,sizeof(LargeCount));
+ buffer.write((char*)&coverage, sizeof(CoverageDepth));
+ buffer.write((char*)&count, sizeof(LargeCount));
+ flushFileOperationBuffer(false, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
}
+ flushFileOperationBuffer(true, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
f.close();
}
@@ -241,7 +275,6 @@ void MachineHelper::call_RAY_MASTER_MODE_SEND_COVERAGE_VALUES (){
m_parameters->setRepeatCoverage(distribution.getRepeatCoverage());
printf("\n");
- fflush(stdout);
cout<<endl;
cout<<"Rank "<<getRank()<<": the minimum coverage is "<<m_parameters->getMinimumCoverage()<<endl;
@@ -250,7 +283,7 @@ void MachineHelper::call_RAY_MASTER_MODE_SEND_COVERAGE_VALUES (){
LargeCount numberOfVertices=0;
LargeCount verticesWith1Coverage=0;
CoverageDepth lowestCoverage=9999;
-
+
LargeCount genomeKmers=0;
for(map<CoverageDepth,LargeCount>::iterator i=m_coverageDistribution->begin();
@@ -316,7 +349,7 @@ void MachineHelper::call_RAY_MASTER_MODE_SEND_COVERAGE_VALUES (){
for(Rank i=0;i<m_parameters->getSize();i++){
Message aMessage(buffer,3,i,RAY_MPI_TAG_SEND_COVERAGE_VALUES,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
}
@@ -325,58 +358,92 @@ int MachineHelper::getRank(){
return m_parameters->getRank();
}
-/** actually, call_RAY_MASTER_MODE_LOAD_SEQUENCES
+/** actually, call_RAY_MASTER_MODE_LOAD_SEQUENCES
* writes the AMOS file */
void MachineHelper::call_RAY_MASTER_MODE_LOAD_SEQUENCES(){
- m_timePrinter->printElapsedTime("Counting sequences to assemble");
- cout<<endl;
+ if(!m_startedToSendCounts){
+
+ m_timePrinter->printElapsedTime("Counting sequences to assemble");
+ cout<<endl;
- bool result=true;
+ bool result=true;
- if(m_parameters->useAmos()){
+ if(m_parameters->useAmos()){
/* This won't write anything if -amos was not provided */
- result=m_sl->writeSequencesToAMOSFile(getRank(),getSize(),
- m_outbox,
- m_outboxAllocator,
- &m_loadSequenceStep,
- m_bubbleData,
- m_lastTime,
- m_parameters,m_switchMan->getMasterModePointer(),m_switchMan->getSlaveModePointer());
- }
+ result=m_sl->writeSequencesToAMOSFile(getRank(),getSize(),
+ m_outbox,
+ m_outboxAllocator,
+ &m_loadSequenceStep,
+ m_bubbleData,
+ m_lastTime,
+ m_parameters,m_switchMan->getMasterModePointer(),m_switchMan->getSlaveModePointer());
+ }
- if(!result){
- (*m_aborted)=true;
- m_switchMan->setSlaveMode(RAY_SLAVE_MODE_DO_NOTHING);
- m_switchMan->setMasterMode(RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
- return;
- }
+ if(!result){
+ (*m_aborted)=true;
+ m_switchMan->setSlaveMode(RAY_SLAVE_MODE_DO_NOTHING);
+ m_switchMan->setMasterMode(RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
+ return;
+ }
- MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
- uint32_t*messageInInts=(uint32_t*)message;
- messageInInts[0]=m_parameters->getNumberOfFiles();
+ m_fileIndex=0;
+ m_theEntriesForFileWasSent=false;
- for(int i=0;i<(int)m_parameters->getNumberOfFiles();i++){
- messageInInts[1+i]=(LargeCount)m_parameters->getNumberOfSequences(i);
- }
-
- for(Rank i=0;i<getSize();i++){
- Message aMessage(message,MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit),
- i,RAY_MPI_TAG_LOAD_SEQUENCES,getRank());
- m_outbox->push_back(aMessage);
- }
+ m_startedToSendCounts=true;
- m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
+ }else if(m_fileIndex<(int)m_parameters->getNumberOfFiles()){
+
+ if(!m_theEntriesForFileWasSent){
+
+ MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+
+ int bufferSize=0;
+ message[bufferSize++]=m_fileIndex;
+ message[bufferSize++]=m_parameters->getNumberOfSequences(m_fileIndex);
+
+ for(Rank i=0;i<getSize();i++){
+ Message aMessage(message,bufferSize,
+ i,RAY_MPI_TAG_SET_FILE_ENTRIES,getRank());
+ m_outbox->push_back(&aMessage);
+ }
+
+ m_theEntriesForFileWasSent=true;
+ m_numberOfRanksThatReplied=0;
+
+ }else if(m_inbox->hasMessage(RAY_MPI_TAG_SET_FILE_ENTRIES_REPLY)){
+
+ m_numberOfRanksThatReplied++;
+
+ if(m_numberOfRanksThatReplied==m_parameters->getSize()){
+ m_fileIndex++;
+ m_theEntriesForFileWasSent=false;
+ }
+ }
+ }else{
+
+/*
+ * RAY_MPI_TAG_LOAD_SEQUENCES is handled by MessageProcessor.plugin.
+ *
+ */
+ for(Rank i=0;i<getSize();i++){
+ Message aMessage(NULL,0,i,RAY_MPI_TAG_LOAD_SEQUENCES,getRank());
+ m_outbox->push_back(&aMessage);
+ }
+
+ m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
+
+ }
}
void MachineHelper::call_RAY_MASTER_MODE_TRIGGER_VERTICE_DISTRIBUTION(){
m_timePrinter->printElapsedTime("Sequence loading");
cout<<endl;
-
+
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_START_VERTICES_DISTRIBUTION,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
}
@@ -389,7 +456,7 @@ void MachineHelper::call_RAY_MASTER_MODE_TRIGGER_GRAPH_BUILDING(){
cout<<endl;
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_BUILD_GRAPH,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
}
@@ -400,7 +467,7 @@ void MachineHelper::call_RAY_MASTER_MODE_PURGE_NULL_EDGES(){
cout<<endl;
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_PURGE_NULL_EDGES,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
}
@@ -448,7 +515,7 @@ void MachineHelper::call_RAY_MASTER_MODE_WRITE_KMERS(){
m_edgeDistribution.clear();
f.close();
cout<<"Rank "<<getRank()<<" wrote "<<edgeFile.str()<<endl;
-
+
}else if(m_coverageRank==m_numberOfRanksDone){
if(m_parameters->writeKmers()){
@@ -467,7 +534,7 @@ void MachineHelper::call_RAY_SLAVE_MODE_WRITE_KMERS(){
if(m_parameters->writeKmers()){
m_coverageGatherer->writeKmers();
}
-
+
/* send edge distribution */
GridTableIterator iterator;
iterator.constructor(m_subgraph,m_parameters->getWordSize(),m_parameters);
@@ -491,19 +558,19 @@ void MachineHelper::call_RAY_SLAVE_MODE_WRITE_KMERS(){
}
Message aMessage(buffer,outputPosition,MASTER_RANK,RAY_MPI_TAG_WRITE_KMERS_REPLY,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_switchMan->setSlaveMode(RAY_SLAVE_MODE_DO_NOTHING);
}
void MachineHelper::call_RAY_MASTER_MODE_TRIGGER_INDEXING(){
m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
-
+
m_timePrinter->printElapsedTime("Null edge purging");
cout<<endl;
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_START_INDEXING_SEQUENCES,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
}
@@ -512,7 +579,7 @@ void MachineHelper::call_RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS(){
(*m_numberOfMachinesDoneSendingVertices)=-1;
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0, i, RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_QUESTION,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
}
@@ -529,7 +596,7 @@ void MachineHelper::call_RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS_WITH_ANSWERS(){
for(m_coverageRank=0;m_coverageRank<m_parameters->getSize();m_coverageRank++){
Message aMessage(NULL,0,m_coverageRank,
RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
@@ -538,7 +605,7 @@ void MachineHelper::call_RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS_WITH_ANSWERS(){
void MachineHelper::call_RAY_MASTER_MODE_PREPARE_SEEDING(){
(*m_ranksDoneAttachingReads)=-1;
(*m_readyToSeed)=getSize();
-
+
m_switchMan->closeMasterMode();
}
@@ -546,7 +613,7 @@ void MachineHelper::call_RAY_SLAVE_MODE_ASSEMBLE_WAVES(){
// take each seed, and extend it in both direction using previously obtained information.
if(m_seedingData->m_SEEDING_i==(LargeCount)m_seedingData->m_SEEDING_seeds.size()){
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_ASSEMBLE_WAVES_DONE,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}else{
}
}
@@ -560,7 +627,7 @@ void MachineHelper::call_RAY_MASTER_MODE_TRIGGER_SEEDING(){
// tell everyone to seed now.
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_START_SEEDING,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
@@ -572,7 +639,7 @@ void MachineHelper::call_RAY_MASTER_MODE_TRIGGER_DETECTION(){
(*m_numberOfRanksDoneSeeding)=-1;
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
(*m_numberOfRanksDoneDetectingDistances)=0;
m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
@@ -583,7 +650,7 @@ void MachineHelper::call_RAY_MASTER_MODE_ASK_DISTANCES(){
(*m_numberOfRanksDoneSendingDistances)=0;
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
}
@@ -592,22 +659,23 @@ void MachineHelper::call_RAY_MASTER_MODE_START_UPDATING_DISTANCES(){
(*m_numberOfRanksDoneSendingDistances)=-1;
m_parameters->computeAverageDistances();
m_switchMan->setSlaveMode(RAY_SLAVE_MODE_DO_NOTHING);
-
+
m_switchMan->closeMasterMode();
}
void MachineHelper::call_RAY_MASTER_MODE_TRIGGER_EXTENSIONS(){
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_ASK_EXTENSION,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
}
-void MachineHelper::call_RAY_SLAVE_MODE_SEND_EXTENSION_DATA(){
+void MachineHelper::call_RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS(Message*message){
+
/* clear eliminated paths */
vector<PathHandle> newNames;
- vector<vector<Kmer> > newPaths;
+ vector<GraphPath> newPaths;
for(int i=0;i<(int)m_ed->m_EXTENSION_contigs.size();i++){
PathHandle uniqueId=m_ed->m_EXTENSION_identifiers[i];
@@ -625,75 +693,164 @@ void MachineHelper::call_RAY_SLAVE_MODE_SEND_EXTENSION_DATA(){
m_ed->m_EXTENSION_identifiers=newNames;
m_ed->m_EXTENSION_contigs=newPaths;
- cout<<"Rank "<<m_parameters->getRank()<< " is appending its fusions"<<endl;
+ m_scaffolder->setContigPaths(&(m_ed->m_EXTENSION_identifiers),&(m_ed->m_EXTENSION_contigs));
+ m_searcher->setContigs(&(m_ed->m_EXTENSION_contigs),&(m_ed->m_EXTENSION_identifiers));
+
+// <cutHere>
+
+ ostringstream testBuffer;
+ uint64_t requiredBytes=0;
+ int threshold=1024*1024*1; // 1 MiB
+
+ for(int i=0;i<(int)m_ed->m_EXTENSION_contigs.size();i++){
+ PathHandle uniqueId=m_ed->m_EXTENSION_identifiers[i];
+
+ string contig=convertToString(&(m_ed->m_EXTENSION_contigs[i]),m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
+
+ string withLineBreaks=addLineBreaks(contig,m_parameters->getColumns());
+
+ testBuffer<<">contig-"<<uniqueId<<" "<<contig.length()<<" nucleotides"<<endl<<withLineBreaks;
+
+ if(testBuffer.tellp()>=threshold){
+ requiredBytes+=testBuffer.tellp();
+ testBuffer.str("");
+ }
+ }
+
+ requiredBytes+=testBuffer.tellp();
+ testBuffer.str("");
+
+#ifdef CONFIG_DEBUG_OFFSETS
+ cout<<"[DEBUG] Rank "<<m_parameters->getRank()<<" requires "<<requiredBytes<<" bytes for storage."<<endl;
+#endif
+
+ MessageUnit*messageBuffer=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+ int bufferPosition=0;
+ messageBuffer[bufferPosition++]=requiredBytes;
+
+ Message aMessage(messageBuffer,bufferPosition,message->getSource(),
+ RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY,getRank());
+
+ m_outbox->push_back(&aMessage);
+}
+
+/**
+ * the equivalent of
+ *
+ * fopen with "r+"
+ *
+ * is
+ *
+ * fstream::open with std::ios::in and std::ios::out
+ *
+ * \see http://www.c-jump.com/CIS60/lecture09_1.htm
+ * \see http://bytes.com/topic/c/answers/127391-iostreams-equivalent-cs-fopen-r
+ */
+void MachineHelper::call_RAY_SLAVE_MODE_SEND_EXTENSION_DATA(){
+
+#ifndef CONFIG_MPI_IO
+/*
+ * Only allow one MPI rank to write at any time.
+ */
+ if(m_parameters->getRank()==0){
+ m_authorized=true;
+ }else if(m_inbox->hasMessage(RAY_MPI_TAG_SEND_AUTHORIZATION)){
+ m_authorized=true;
+ }
+
+ if(!m_authorized)
+ return;
+
+#endif
+
string output=m_parameters->getOutputFile();
- ofstream fp;
+ const char*fileNameValue=output.c_str();
- fp.open(output.c_str(),ios_base::out|ios_base::app);
+/*
+ * The const_cast thing is now required with MPI 3.0.
+ */
+ char*fileName= const_cast<char*> ( fileNameValue );
- int total=0;
+#ifdef CONFIG_DEBUG_OFFSETS
+ cout<<"[DEBUG] Rank "<<m_parameters->getRank()<< " is appending its fusions at "<<m_offsetForContigs<<endl;
+#endif
- m_scaffolder->setContigPaths(&(m_ed->m_EXTENSION_identifiers),&(m_ed->m_EXTENSION_contigs));
- m_searcher->setContigs(&(m_ed->m_EXTENSION_contigs),&(m_ed->m_EXTENSION_identifiers));
+#ifdef CONFIG_MPI_IO
+
+/*
+ * Create a view in the file for this MPI rank.
+ */
+ MPI_File fp;
+ MPI_File_open(MPI_COMM_WORLD,fileName,MPI_MODE_CREATE|MPI_MODE_RDWR,MPI_INFO_NULL,&fp);
+ MPI_Datatype elementType=MPI_BYTE;
+ MPI_Datatype fileType=MPI_BYTE;
+ char representation[]="native";
+ MPI_Offset displacement=m_offsetForContigs;
+
+ int returnValue=MPI_File_set_view(fp,displacement,elementType,fileType,representation,MPI_INFO_NULL);
+
+ if(returnValue!= MPI_SUCCESS){
+ cout<<"Error: can not create view."<<endl;
+ }
+
+#else
+
+ ofstream fp;
+ fp.open(fileName, std::ios::app);
+#endif
+
+ int total=0;
ostringstream operationBuffer;
+ bool force=false;
+
for(int i=0;i<(int)m_ed->m_EXTENSION_contigs.size();i++){
PathHandle uniqueId=m_ed->m_EXTENSION_identifiers[i];
- if(m_fusionData->m_FUSION_eliminated.count(uniqueId)>0){
- continue;
- }
-
total++;
string contig=convertToString(&(m_ed->m_EXTENSION_contigs[i]),m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
-
+
string withLineBreaks=addLineBreaks(contig,m_parameters->getColumns());
operationBuffer<<">contig-"<<uniqueId<<" "<<contig.length()<<" nucleotides"<<endl<<withLineBreaks;
- flushFileOperationBuffer(false,&operationBuffer,&fp,CONFIG_FILE_IO_BUFFER_SIZE);
+#ifdef CONFIG_MPI_IO
+ flushFileOperationBuffer_MPI_IO(force,&operationBuffer,fp,CONFIG_FILE_IO_BUFFER_SIZE);
+#else
+ flushFileOperationBuffer(force,&operationBuffer,&fp,CONFIG_FILE_IO_BUFFER_SIZE);
+#endif
}
- cout<<"Rank "<<m_parameters->getRank()<<" appended "<<total<<" elements"<<endl;
+ force=true;
- flushFileOperationBuffer(true,&operationBuffer,&fp,CONFIG_FILE_IO_BUFFER_SIZE);
+ cout<<"Rank "<<m_parameters->getRank()<<" appended "<<total<<" elements"<<endl;
+#ifdef CONFIG_MPI_IO
+ flushFileOperationBuffer_MPI_IO(force,&operationBuffer,fp,CONFIG_FILE_IO_BUFFER_SIZE);
+ MPI_File_close(&fp);
+#else
+ flushFileOperationBuffer(force,&operationBuffer,&fp,CONFIG_FILE_IO_BUFFER_SIZE);
fp.close();
-
- if(m_parameters->showMemoryUsage()){
- showMemoryUsage(getRank());
- }
-
- /** possibly write the checkpoint */
- if(m_parameters->writeCheckpoints() && !m_parameters->hasCheckpoint("ContigPaths")){
- cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint ContigPaths"<<endl;
- ofstream f(m_parameters->getCheckpointFile("ContigPaths").c_str());
- int theSize=m_ed->m_EXTENSION_contigs.size();
- f.write((char*)&theSize,sizeof(int));
-
- /* write each path with its name and vertices */
- for(int i=0;i<theSize;i++){
- PathHandle name=m_ed->m_EXTENSION_identifiers[i];
- int vertices=m_ed->m_EXTENSION_contigs[i].size();
- f.write((char*)&name,sizeof(PathHandle));
- f.write((char*)&vertices,sizeof(int));
- for(int j=0;j<vertices;j++){
- m_ed->m_EXTENSION_contigs[i][j].write(&f);
- }
- }
- f.close();
- }
+#endif
m_switchMan->setSlaveMode(RAY_SLAVE_MODE_DO_NOTHING);
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_EXTENSION_DATA_END,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
+
+/*
+ * Send the authorization code to the next MPI rank.
+ */
+ if(getRank()!=getSize()-1){
+ Message aMessage(NULL,0,getRank()+1,RAY_MPI_TAG_SEND_AUTHORIZATION,getRank());
+ m_outbox->push_back(&aMessage);
+ }
}
void MachineHelper::call_RAY_MASTER_MODE_TRIGGER_FUSIONS(){
m_timePrinter->printElapsedTime("Bidirectional extension of seeds");
cout<<endl;
-
+
m_cycleNumber=0;
m_switchMan->closeMasterMode();
@@ -704,7 +861,7 @@ void MachineHelper::call_RAY_MASTER_MODE_TRIGGER_FIRST_FUSIONS(){
(*m_reductionOccured)=true;
m_cycleStarted=false;
m_mustStop=false;
-
+
m_switchMan->closeMasterMode();
}
@@ -738,7 +895,7 @@ void MachineHelper::call_RAY_MASTER_MODE_START_FUSION_CYCLE(){
(*m_isFinalFusion)=false;
for(int i=0;i<getSize();i++){
Message aMessage(buffer,count,i,RAY_MPI_TAG_CLEAR_DIRECTIONS,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
m_currentCycleStep=1;
(*m_CLEAR_n)=0;
@@ -758,7 +915,7 @@ void MachineHelper::call_RAY_MASTER_MODE_START_FUSION_CYCLE(){
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_DISTRIBUTE_FUSIONS,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
(*m_DISTRIBUTE_n)=0;
}else if((*m_DISTRIBUTE_n) ==getSize() && !(*m_isFinalFusion) && m_currentCycleStep==2){
@@ -768,7 +925,7 @@ void MachineHelper::call_RAY_MASTER_MODE_START_FUSION_CYCLE(){
(*m_isFinalFusion)=true;
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_FINISH_FUSIONS,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
(*m_FINISH_n)=0;
}else if((*m_FINISH_n) ==getSize() && (*m_isFinalFusion) && m_currentCycleStep==3){
@@ -791,7 +948,7 @@ void MachineHelper::call_RAY_MASTER_MODE_START_FUSION_CYCLE(){
for(int i=0;i<getSize();i++){
Message aMessage(buffer,count,i,RAY_MPI_TAG_CLEAR_DIRECTIONS,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
(*m_FINISH_n)=-1;
@@ -803,10 +960,10 @@ void MachineHelper::call_RAY_MASTER_MODE_START_FUSION_CYCLE(){
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_DISTRIBUTE_FUSIONS,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
(*m_DISTRIBUTE_n)=0;
-
+
cout<<"Rank 0: starting distribution step"<<endl;
}else if((*m_DISTRIBUTE_n)==getSize() && (*m_isFinalFusion) && m_currentCycleStep==5){
//cout<<"cycleStep= "<<m_currentCycleStep<<endl;
@@ -820,7 +977,7 @@ void MachineHelper::call_RAY_MASTER_MODE_START_FUSION_CYCLE(){
m_timePrinter->printElapsedTime("Merging of redundant paths");
cout<<endl;
- m_switchMan->setMasterMode(RAY_MASTER_MODE_ASK_EXTENSIONS);
+ m_switchMan->closeMasterMode();
m_ed->m_EXTENSION_currentRankIsSet=false;
m_ed->m_EXTENSION_rank=-1;
@@ -832,9 +989,9 @@ void MachineHelper::call_RAY_MASTER_MODE_START_FUSION_CYCLE(){
(*m_DISTRIBUTE_n)=-1;
for(int i=0;i<(int)getSize();i++){// start fusion.
Message aMessage(NULL,0,i,RAY_MPI_TAG_START_FUSION,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
-
+
}else if(m_fusionData->m_FUSION_numberOfRanksDone==getSize() && (*m_isFinalFusion) && m_currentCycleStep==6){
/** always force cycle number 2 */
@@ -856,19 +1013,69 @@ void MachineHelper::call_RAY_MASTER_MODE_START_FUSION_CYCLE(){
}
}
+void MachineHelper::notifyThatOldDirectoryExists(){
+
+ m_oldDirectoryExists=true;
+}
+
+void MachineHelper::call_RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY(Message*message){
+
+ Rank source=message->getSource();
+ MessageUnit*buffer=message->getBuffer();
+ uint64_t bytes=buffer[0];
+
+ m_rankStorage[source]=bytes;
+
+#ifdef CONFIG_DEBUG_OFFSETS
+ cout << "[DEBUG] Rank " << getRank() << " rank " << source << " needs " << bytes << " bytes" << endl;
+#endif
+
+ m_ranksThatComputedStorage++;
+
+ if(m_ranksThatComputedStorage==getSize()){
+
+/*
+ * Write the file in parallel (or not).
+ */
+
+ uint64_t offset=0;
+
+ for(int rank=0;rank<getSize();rank++){
+
+ MessageUnit*buffer=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+ buffer[0]=offset;
+
+ Message aMessage(buffer,1,rank,RAY_MPI_TAG_ASK_EXTENSION_DATA,getRank());
+ m_outbox->push_back(&aMessage);
+
+ offset+=m_rankStorage[rank];
+ }
+ }
+}
+
void MachineHelper::call_RAY_MASTER_MODE_ASK_EXTENSIONS(){
// ask ranks to send their extensions.
if(!m_ed->m_EXTENSION_currentRankIsSet){
- m_ed->m_EXTENSION_currentRankIsSet=true;
- m_ed->m_EXTENSION_currentRankIsStarted=false;
- m_ed->m_EXTENSION_rank++;
+
+ m_ranksThatWroteContigs=0;
+ m_ranksThatComputedStorage=0;
+ m_switchMan->sendToAll(m_outbox,getRank(),RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS);
+
+ m_rankStorage.resize(getSize());
+
+ for(int i = 0; i < getSize() ; i++) {
+ m_rankStorage[i] = 0;
+ }
m_seedExtender->closePathFile();
- }
- if(m_ed->m_EXTENSION_rank==getSize()){
+ m_ed->m_EXTENSION_currentRankIsSet=true;
+
+ }else if(m_ranksThatWroteContigs==getSize()){
+
m_timePrinter->printElapsedTime("Generation of contigs");
+
if(m_parameters->useAmos()){
m_switchMan->setMasterMode(RAY_MASTER_MODE_AMOS);
@@ -884,21 +1091,26 @@ void MachineHelper::call_RAY_MASTER_MODE_ASK_EXTENSIONS(){
m_scaffolder->m_numberOfRanksFinished=0;
}
-
- }else if(!m_ed->m_EXTENSION_currentRankIsStarted){
- m_ed->m_EXTENSION_currentRankIsStarted=true;
- Message aMessage(NULL,0,m_ed->m_EXTENSION_rank,RAY_MPI_TAG_ASK_EXTENSION_DATA,getRank());
- m_outbox->push_back(aMessage);
- m_ed->m_EXTENSION_currentRankIsDone=false;
- }else if(m_ed->m_EXTENSION_currentRankIsDone){
- m_ed->m_EXTENSION_currentRankIsSet=false;
}
}
+void MachineHelper::call_RAY_MPI_TAG_ASK_EXTENSION_DATA(Message*message){
+ (m_seedingData->m_SEEDING_i)=0;
+ (m_ed->m_EXTENSION_currentPosition)=0;
+
+ MessageUnit*buffer=message->getBuffer();
+
+ m_offsetForContigs = buffer[0];
+
+#ifdef CONFIG_DEBUG_OFFSETS
+ cout << "[DEBUG] Rank " << getRank() << " call_RAY_MPI_TAG_ASK_EXTENSION_DATA received offset: " << m_offsetForContigs << endl;
+#endif
+}
+
void MachineHelper::call_RAY_MASTER_MODE_SCAFFOLDER(){
for(int i=0;i<getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_START_SCAFFOLDER,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
m_switchMan->setMasterMode(RAY_MASTER_MODE_DO_NOTHING);
}
@@ -913,24 +1125,17 @@ void MachineHelper::call_RAY_SLAVE_MODE_DIE(){
/* write the network test data if not already written */
m_networkTest->writeData();
- /** write message-passing interface file */
- ostringstream file;
- file<<m_parameters->getPrefix()<<"MessagePassingInterface.txt";
-
- string fileInString=file.str();
- m_messagesHandler->appendStatistics(fileInString.c_str());
-
/** actually die */
(*m_alive)=false;
- /** tell master that the rank died
+ /** tell master that the rank died
* obviously, this message won't be recorded in the MessagePassingInterface file...
- * Because of that, MessagesHandler will do it for us.
+ * Because of that, the middleware will do it for us.
* */
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_GOOD_JOB_SEE_YOU_SOON_REPLY,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
- /** do nothing while dying
+ /** do nothing while dying
* the aging process takes a while -- 1024 cycles.
* after that, it is death itself.
* */
@@ -945,22 +1150,11 @@ void MachineHelper::call_RAY_MASTER_MODE_KILL_ALL_MPI_RANKS(){
m_initialisedKiller=true;
m_machineRank=m_parameters->getSize()-1;
- /** empty the file if it exists */
- ostringstream file;
- file<<m_parameters->getPrefix()<<"MessagePassingInterface.txt";
-
- FILE*fp=fopen(file.str().c_str(),"w+");
- if(fp==NULL){
- cout<<"Error: cannot create file "<<file<<endl;
- }
- fprintf(fp,"# Source\tDestination\tTag\tCount\n");
- fclose(fp);
-
// activate the relay checker
m_numberOfRanksDone=0;
for(Rank i=0;i<m_parameters->getSize();i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
// another rank activated its relay checker
@@ -975,7 +1169,7 @@ void MachineHelper::call_RAY_MASTER_MODE_KILL_ALL_MPI_RANKS(){
* a message.
* For the other ones, we wait for the response of the previous.
*/
- }else if(m_machineRank==m_parameters->getSize()-1 ||
+ }else if(m_machineRank==m_parameters->getSize()-1 ||
(m_inbox->size()>0 && (*m_inbox)[0]->getTag()==RAY_MPI_TAG_GOOD_JOB_SEE_YOU_SOON_REPLY)){
/**
@@ -988,17 +1182,49 @@ void MachineHelper::call_RAY_MASTER_MODE_KILL_ALL_MPI_RANKS(){
/** send a killer message */
Message aMessage(NULL,0,m_machineRank,RAY_MPI_TAG_GOOD_JOB_SEE_YOU_SOON,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
/** change the next to kill */
m_machineRank--;
}
}
+void MachineHelper::call_RAY_MPI_TAG_NOTIFY_ERROR(Message*message){
+ (*m_aborted)=true;
+}
+
int MachineHelper::getSize(){
return m_parameters->getSize();
}
+void MachineHelper::performAssemblyWorkflow(ComputeCore*core) {
+ core->setFirstMasterMode(m_plugin,RAY_MASTER_MODE_LOAD_CONFIG);
+
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_LOAD_CONFIG, RAY_MASTER_MODE_TEST_NETWORK);
+
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_PREPARE_SEEDING,RAY_MASTER_MODE_TRIGGER_SEEDING);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TRIGGER_SEEDING,RAY_MASTER_MODE_START_UPDATING_DISTANCES);
+
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_LOAD_SEQUENCES,RAY_MASTER_MODE_TRIGGER_VERTICE_DISTRIBUTION);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TRIGGER_VERTICE_DISTRIBUTION,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS_WITH_ANSWERS);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS_WITH_ANSWERS,RAY_MASTER_MODE_SEND_COVERAGE_VALUES);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_SEND_COVERAGE_VALUES,RAY_MASTER_MODE_TRIGGER_GRAPH_BUILDING);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TRIGGER_GRAPH_BUILDING,RAY_MASTER_MODE_PURGE_NULL_EDGES);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_PURGE_NULL_EDGES,RAY_MASTER_MODE_WRITE_KMERS);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_WRITE_KMERS,RAY_MASTER_MODE_TRIGGER_INDEXING);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TRIGGER_INDEXING,RAY_MASTER_MODE_PREPARE_SEEDING);
+
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_START_UPDATING_DISTANCES,RAY_MASTER_MODE_UPDATE_DISTANCES);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TRIGGER_FUSIONS,RAY_MASTER_MODE_TRIGGER_FIRST_FUSIONS);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TRIGGER_FIRST_FUSIONS,RAY_MASTER_MODE_START_FUSION_CYCLE);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_START_FUSION_CYCLE,RAY_MASTER_MODE_EVALUATE_PATHS);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_ASK_EXTENSIONS,RAY_MASTER_MODE_SCAFFOLDER);
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_SCAFFOLDER,RAY_MASTER_MODE_WRITE_SCAFFOLDS);
+
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_KILL_RANKS,RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
+}
+
void MachineHelper::registerPlugin(ComputeCore*core){
PluginHandle plugin=core->allocatePluginHandle();
m_plugin=plugin;
@@ -1126,6 +1352,39 @@ void MachineHelper::registerPlugin(ComputeCore*core){
RAY_MPI_TAG_GOOD_JOB_SEE_YOU_SOON_REPLY=core->allocateMessageTagHandle(plugin);
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_GOOD_JOB_SEE_YOU_SOON_REPLY,"RAY_MPI_TAG_GOOD_JOB_SEE_YOU_SOON_REPLY");
+ core->setObjectSymbol(m_plugin,&(m_ed->m_EXTENSION_contigs),"/RayAssembler/ObjectStore/ContigPaths.ray");
+ core->setObjectSymbol(m_plugin,&(m_ed->m_EXTENSION_identifiers),"/RayAssembler/ObjectStore/ContigNames.ray");
+
+ RAY_MPI_TAG_NOTIFY_ERROR=core->allocateMessageTagHandle(m_plugin);
+ core->setMessageTagObjectHandler(m_plugin,RAY_MPI_TAG_NOTIFY_ERROR,__GetAdapter(MachineHelper,RAY_MPI_TAG_NOTIFY_ERROR));
+ core->setMessageTagSymbol(m_plugin,RAY_MPI_TAG_NOTIFY_ERROR,"RAY_MPI_TAG_NOTIFY_ERROR");
+
+ RAY_MPI_TAG_SEND_AUTHORIZATION=core->allocateMessageTagHandle(m_plugin);
+ core->setMessageTagSymbol(m_plugin,RAY_MPI_TAG_SEND_AUTHORIZATION,"RAY_MPI_TAG_SEND_AUTHORIZATION");
+
+ RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS=core->allocateMessageTagHandle(m_plugin);
+ core->setMessageTagObjectHandler(m_plugin,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS,
+ __GetAdapter(MachineHelper,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS));
+ core->setMessageTagSymbol(m_plugin,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS,
+ "RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS");
+
+ RAY_MPI_TAG_EXTENSION_DATA_END=core->allocateMessageTagHandle(plugin);
+ core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_EXTENSION_DATA_END,
+ __GetAdapter(MachineHelper,RAY_MPI_TAG_EXTENSION_DATA_END));
+ core->setMessageTagSymbol(plugin,RAY_MPI_TAG_EXTENSION_DATA_END,"RAY_MPI_TAG_EXTENSION_DATA_END");
+
+ RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY=core->allocateMessageTagHandle(m_plugin);
+ core->setMessageTagObjectHandler(m_plugin,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY,
+ __GetAdapter(MachineHelper,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY));
+ core->setMessageTagSymbol(m_plugin,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY,
+ "RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY");
+
+ RAY_MPI_TAG_ASK_EXTENSION_DATA=core->allocateMessageTagHandle(plugin);
+ core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_ASK_EXTENSION_DATA, __GetAdapter(MachineHelper,RAY_MPI_TAG_ASK_EXTENSION_DATA));
+ core->setMessageTagSymbol(plugin,RAY_MPI_TAG_ASK_EXTENSION_DATA,"RAY_MPI_TAG_ASK_EXTENSION_DATA");
+
+ void*address=&(m_fusionData->m_FUSION_identifier_map);
+ core->setObjectSymbol(m_plugin,address,"/RayAssembler/ObjectStore/ContigNameIndex.ray");
}
void MachineHelper::resolveSymbols(ComputeCore*core){
@@ -1189,6 +1448,8 @@ void MachineHelper::resolveSymbols(ComputeCore*core){
RAY_MASTER_MODE_UPDATE_DISTANCES=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_UPDATE_DISTANCES");
RAY_MASTER_MODE_WRITE_KMERS=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_WRITE_KMERS");
RAY_MASTER_MODE_WRITE_SCAFFOLDS=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_WRITE_SCAFFOLDS");
+ RAY_MASTER_MODE_STEP_A=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_STEP_A");
+ RAY_MASTER_MODE_EVALUATE_PATHS=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_EVALUATE_PATHS");
RAY_MPI_TAG_FINISH_FUSIONS=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_FINISH_FUSIONS");
RAY_MPI_TAG_GET_CONTIG_CHUNK=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_CONTIG_CHUNK");
@@ -1273,31 +1534,18 @@ void MachineHelper::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_DISTRIBUTE_FUSIONS=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_DISTRIBUTE_FUSIONS");
RAY_MPI_TAG_EXTENSION_DATA_END=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_EXTENSION_DATA_END");
- core->setFirstMasterMode(m_plugin,RAY_MASTER_MODE_LOAD_CONFIG);
-
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_LOAD_CONFIG, RAY_MASTER_MODE_TEST_NETWORK);
-
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_PREPARE_SEEDING,RAY_MASTER_MODE_TRIGGER_SEEDING);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TRIGGER_SEEDING,RAY_MASTER_MODE_START_UPDATING_DISTANCES);
-
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_LOAD_SEQUENCES,RAY_MASTER_MODE_TRIGGER_VERTICE_DISTRIBUTION);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TRIGGER_VERTICE_DISTRIBUTION,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS_WITH_ANSWERS);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS_WITH_ANSWERS,RAY_MASTER_MODE_SEND_COVERAGE_VALUES);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_SEND_COVERAGE_VALUES,RAY_MASTER_MODE_TRIGGER_GRAPH_BUILDING);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TRIGGER_GRAPH_BUILDING,RAY_MASTER_MODE_PURGE_NULL_EDGES);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_PURGE_NULL_EDGES,RAY_MASTER_MODE_WRITE_KMERS);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_WRITE_KMERS,RAY_MASTER_MODE_TRIGGER_INDEXING);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TRIGGER_INDEXING,RAY_MASTER_MODE_PREPARE_SEEDING);
+ RAY_MPI_TAG_SET_FILE_ENTRIES=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_SET_FILE_ENTRIES");
+ RAY_MPI_TAG_SET_FILE_ENTRIES_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_SET_FILE_ENTRIES_REPLY");
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_START_UPDATING_DISTANCES,RAY_MASTER_MODE_UPDATE_DISTANCES);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TRIGGER_FUSIONS,RAY_MASTER_MODE_TRIGGER_FIRST_FUSIONS);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TRIGGER_FIRST_FUSIONS,RAY_MASTER_MODE_START_FUSION_CYCLE);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_START_FUSION_CYCLE,RAY_MASTER_MODE_ASK_EXTENSIONS);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_ASK_EXTENSIONS,RAY_MASTER_MODE_SCAFFOLDER);
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_SCAFFOLDER,RAY_MASTER_MODE_WRITE_SCAFFOLDS);
+ RAY_MPI_TAG_ASK_EXTENSION_DATA=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_ASK_EXTENSION_DATA");
- core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_KILL_RANKS,RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
+ if (m_parameters->hasOption("-example")) {
+ cout << "************** Example Mode **************" << endl;
+ core->setFirstMasterMode(m_plugin, RAY_MASTER_MODE_STEP_A);
+ }
+ else {
+ performAssemblyWorkflow(core);
+ }
RAY_MPI_TAG_GOOD_JOB_SEE_YOU_SOON=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GOOD_JOB_SEE_YOU_SOON");
RAY_MPI_TAG_GOOD_JOB_SEE_YOU_SOON_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GOOD_JOB_SEE_YOU_SOON_REPLY");
@@ -1307,8 +1555,46 @@ void MachineHelper::resolveSymbols(ComputeCore*core){
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_GOOD_JOB_SEE_YOU_SOON, RAY_SLAVE_MODE_DIE);
- core->setObjectSymbol(m_plugin,&(m_ed->m_EXTENSION_contigs),"/RayAssembler/ObjectStore/ContigPaths.ray");
- core->setObjectSymbol(m_plugin,&(m_ed->m_EXTENSION_identifiers),"/RayAssembler/ObjectStore/ContigNames.ray");
+ core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_ASK_EXTENSION_DATA, RAY_SLAVE_MODE_SEND_EXTENSION_DATA);
__BindPlugin(MachineHelper);
+
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_LOAD_CONFIG);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_SEND_COVERAGE_VALUES);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_WRITE_KMERS);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_LOAD_SEQUENCES);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_VERTICE_DISTRIBUTION);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_GRAPH_BUILDING);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_PURGE_NULL_EDGES);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_INDEXING);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS_WITH_ANSWERS);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_SEEDING);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_SEEDING);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_DETECTION);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_ASK_DISTANCES);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_START_UPDATING_DISTANCES);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_EXTENSIONS);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_FUSIONS);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_FIRST_FUSIONS);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_START_FUSION_CYCLE);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_ASK_EXTENSIONS);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_SCAFFOLDER);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_KILL_RANKS);
+ __BindAdapter(MachineHelper,RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
+ __BindAdapter(MachineHelper,RAY_SLAVE_MODE_WRITE_KMERS);
+ __BindAdapter(MachineHelper,RAY_SLAVE_MODE_ASSEMBLE_WAVES);
+ __BindAdapter(MachineHelper,RAY_SLAVE_MODE_SEND_EXTENSION_DATA);
+ __BindAdapter(MachineHelper,RAY_SLAVE_MODE_DIE);
+
+ __BindAdapter(MachineHelper,RAY_MPI_TAG_NOTIFY_ERROR);
+ __BindAdapter(MachineHelper,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS);
+ __BindAdapter(MachineHelper,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY);
+ __BindAdapter(MachineHelper,RAY_MPI_TAG_ASK_EXTENSION_DATA);
+ __BindAdapter(MachineHelper,RAY_MPI_TAG_EXTENSION_DATA_END);
+
+ m_startedToSendCounts=false;
+ m_authorized=false;
}
+
+
diff --git a/code/plugin_MachineHelper/MachineHelper.h b/code/MachineHelper/MachineHelper.h
similarity index 61%
rename from code/plugin_MachineHelper/MachineHelper.h
rename to code/MachineHelper/MachineHelper.h
index 1e55906..5c02866 100644
--- a/code/plugin_MachineHelper/MachineHelper.h
+++ b/code/MachineHelper/MachineHelper.h
@@ -1,6 +1,7 @@
/*
- Ray
- Copyright (C) 2012 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+ Copyright (C) 2013 Charles Joly Beauparlant
http://DeNovoAssembler.SourceForge.Net/
@@ -21,47 +22,121 @@
#ifndef _MachineHelper_h
#define _MachineHelper_h
-#include <application_core/Parameters.h>
-#include <memory/RingAllocator.h>
-#include <structures/StaticVector.h>
-#include <scheduling/SwitchMan.h>
-#include <handlers/MasterModeHandler.h>
-#include <handlers/SlaveModeHandler.h>
-#include <plugin_SeedExtender/ExtensionData.h>
-#include <plugin_FusionData/FusionData.h>
-#include <profiling/Profiler.h>
-#include <plugin_NetworkTest/NetworkTest.h>
-#include <plugin_Partitioner/Partitioner.h>
-#include <plugin_SequencesLoader/SequencesLoader.h>
-#include <plugin_SeedingData/SeedingData.h>
-#include <communication/MessagesHandler.h>
-#include <plugin_Scaffolder/Scaffolder.h>
-#include <plugin_SeedExtender/SeedExtender.h>
-#include <profiling/TimePrinter.h>
-#include <plugin_SeedExtender/OpenAssemblerChooser.h>
-#include <plugin_SeedExtender/BubbleData.h>
-#include <plugin_Searcher/Searcher.h>
-#include <plugin_SequencesLoader/ArrayOfReads.h>
-#include <plugin_VerticesExtractor/VerticesExtractor.h>
-#include <plugin_EdgePurger/EdgePurger.h>
-#include <plugin_VerticesExtractor/GridTable.h>
-#include <plugin_KmerAcademyBuilder/KmerAcademyBuilder.h>
-#include <communication/VirtualCommunicator.h>
-#include <plugin_CoverageGatherer/CoverageGatherer.h>
-#include <plugin_SequencesIndexer/SequencesIndexer.h>
-#include <core/ComputeCore.h>
+#include <code/Mock/Parameters.h>
+#include <code/FusionData/FusionData.h>
+#include <code/NetworkTest/NetworkTest.h>
+#include <code/Example/Example.h>
+#include <code/Partitioner/Partitioner.h>
+#include <code/SequencesLoader/SequencesLoader.h>
+#include <code/SeedingData/SeedingData.h>
+#include <code/Scaffolder/Scaffolder.h>
+#include <code/SeedExtender/SeedExtender.h>
+#include <code/SeedExtender/ExtensionData.h>
+#include <code/SeedExtender/OpenAssemblerChooser.h>
+#include <code/SeedExtender/BubbleData.h>
+#include <code/Searcher/Searcher.h>
+#include <code/SequencesLoader/ArrayOfReads.h>
+#include <code/VerticesExtractor/VerticesExtractor.h>
+#include <code/EdgePurger/EdgePurger.h>
+#include <code/VerticesExtractor/GridTable.h>
+#include <code/KmerAcademyBuilder/KmerAcademyBuilder.h>
+#include <code/CoverageGatherer/CoverageGatherer.h>
+#include <code/SequencesIndexer/SequencesIndexer.h>
+
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/scheduling/SwitchMan.h>
+#include <RayPlatform/handlers/MasterModeHandler.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/profiling/TimePrinter.h>
+#include <RayPlatform/profiling/Profiler.h>
#include <stdint.h>
#include <map>
using namespace std;
-
-/** this file contains __legacy code__
+__DeclarePlugin(MachineHelper);
+
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_LOAD_CONFIG);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_SEND_COVERAGE_VALUES);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_WRITE_KMERS);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_LOAD_SEQUENCES);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_VERTICE_DISTRIBUTION);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_GRAPH_BUILDING);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_PURGE_NULL_EDGES);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_INDEXING);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS_WITH_ANSWERS);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_SEEDING);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_SEEDING);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_DETECTION);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_ASK_DISTANCES);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_START_UPDATING_DISTANCES);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_EXTENSIONS);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_FUSIONS);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_FIRST_FUSIONS);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_START_FUSION_CYCLE);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_ASK_EXTENSIONS);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_SCAFFOLDER);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_KILL_RANKS);
+__DeclareMasterModeAdapter(MachineHelper,RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
+
+__DeclareSlaveModeAdapter(MachineHelper,RAY_SLAVE_MODE_WRITE_KMERS);
+__DeclareSlaveModeAdapter(MachineHelper,RAY_SLAVE_MODE_ASSEMBLE_WAVES);
+__DeclareSlaveModeAdapter(MachineHelper,RAY_SLAVE_MODE_SEND_EXTENSION_DATA);
+__DeclareSlaveModeAdapter(MachineHelper,RAY_SLAVE_MODE_DIE);
+
+__DeclareMessageTagAdapter(MachineHelper,RAY_MPI_TAG_NOTIFY_ERROR);
+__DeclareMessageTagAdapter(MachineHelper,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS);
+__DeclareMessageTagAdapter(MachineHelper,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY);
+__DeclareMessageTagAdapter(MachineHelper,RAY_MPI_TAG_ASK_EXTENSION_DATA);
+__DeclareMessageTagAdapter(MachineHelper,RAY_MPI_TAG_EXTENSION_DATA_END);
+
+/**
+ * This file contains __legacy code__
* Old handlers are here.
* TODO: move them elsewhere ?
* \author Sébastien Boisvert */
class MachineHelper: public CorePlugin{
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_LOAD_CONFIG);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_SEND_COVERAGE_VALUES);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_WRITE_KMERS);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_LOAD_SEQUENCES);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_VERTICE_DISTRIBUTION);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_GRAPH_BUILDING);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_PURGE_NULL_EDGES);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_INDEXING);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_DISTRIBUTIONS_WITH_ANSWERS);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_PREPARE_SEEDING);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_SEEDING);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_DETECTION);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_ASK_DISTANCES);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_START_UPDATING_DISTANCES);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_EXTENSIONS);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_FUSIONS);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_TRIGGER_FIRST_FUSIONS);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_START_FUSION_CYCLE);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_ASK_EXTENSIONS);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_SCAFFOLDER);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_KILL_RANKS);
+ __AddAdapter(MachineHelper,RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
+
+ __AddAdapter(MachineHelper,RAY_SLAVE_MODE_WRITE_KMERS);
+ __AddAdapter(MachineHelper,RAY_SLAVE_MODE_ASSEMBLE_WAVES);
+ __AddAdapter(MachineHelper,RAY_SLAVE_MODE_SEND_EXTENSION_DATA);
+ __AddAdapter(MachineHelper,RAY_SLAVE_MODE_DIE);
+
+ __AddAdapter(MachineHelper,RAY_MPI_TAG_NOTIFY_ERROR);
+ __AddAdapter(MachineHelper,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS);
+ __AddAdapter(MachineHelper,RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY);
+ __AddAdapter(MachineHelper,RAY_MPI_TAG_ASK_EXTENSION_DATA);
+ __AddAdapter(MachineHelper,RAY_MPI_TAG_EXTENSION_DATA_END);
+
+ MessageTag RAY_MPI_TAG_NOTIFY_ERROR;
MessageTag RAY_MPI_TAG_FINISH_FUSIONS;
MessageTag RAY_MPI_TAG_GET_CONTIG_CHUNK;
MessageTag RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY;
@@ -145,6 +220,27 @@ class MachineHelper: public CorePlugin{
MessageTag RAY_MPI_TAG_DISTRIBUTE_FUSIONS;
MessageTag RAY_MPI_TAG_EXTENSION_DATA_END;
+ MessageTag RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS;
+ MessageTag RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY;
+
+ int m_ranksThatComputedStorage;
+ vector<uint64_t> m_rankStorage;
+ uint64_t m_offsetForContigs;
+ int m_ranksThatWroteContigs;
+ bool m_authorized;
+
+/*
+ * Stuff for sending entries in files.
+ */
+
+ int m_numberOfRanksThatReplied;
+ bool m_theEntriesForFileWasSent;
+ int m_fileIndex;
+ bool m_startedToSendCounts;
+ MessageTag RAY_MPI_TAG_SET_FILE_ENTRIES;
+ MessageTag RAY_MPI_TAG_SET_FILE_ENTRIES_REPLY;
+ MessageTag RAY_MPI_TAG_SEND_AUTHORIZATION;
+
MasterMode RAY_MASTER_MODE_ADD_COLORS;
MasterMode RAY_MASTER_MODE_AMOS;
MasterMode RAY_MASTER_MODE_ASK_DISTANCES;
@@ -178,6 +274,8 @@ class MachineHelper: public CorePlugin{
MasterMode RAY_MASTER_MODE_UPDATE_DISTANCES;
MasterMode RAY_MASTER_MODE_WRITE_KMERS;
MasterMode RAY_MASTER_MODE_WRITE_SCAFFOLDS;
+ MasterMode RAY_MASTER_MODE_STEP_A;
+ MasterMode RAY_MASTER_MODE_EVALUATE_PATHS;
SlaveMode RAY_SLAVE_MODE_EXTENSION;
SlaveMode RAY_SLAVE_MODE_ADD_COLORS;
@@ -206,7 +304,7 @@ class MachineHelper: public CorePlugin{
SlaveMode RAY_SLAVE_MODE_TEST_NETWORK;
SlaveMode RAY_SLAVE_MODE_WRITE_KMERS;
-
+ bool m_oldDirectoryExists;
SequencesLoader*m_sl;
time_t*m_lastTime;
@@ -246,7 +344,6 @@ class MachineHelper: public CorePlugin{
TimePrinter*m_timePrinter;
SeedExtender*m_seedExtender;
Scaffolder*m_scaffolder;
- MessagesHandler*m_messagesHandler;
bool m_coverageInitialised;
int m_currentCycleStep;
@@ -287,7 +384,7 @@ class MachineHelper: public CorePlugin{
int getRank();
int getSize();
-
+ void performAssemblyWorkflow(ComputeCore*core);
public:
void constructor(int argc,char**argv,Parameters*parameters,
@@ -297,7 +394,7 @@ public:
int*numberOfMachinesDoneSendingCoverage,int*numberOfRanksWithCoverageData,
bool*reductionOccured,ExtensionData*ed,FusionData*fusionData,
Profiler*p,NetworkTest*nt,SeedingData*sd,
-TimePrinter*timePrinter,SeedExtender*seedExtender,Scaffolder*scaffolder,MessagesHandler*messagesHandler,
+TimePrinter*timePrinter,SeedExtender*seedExtender,Scaffolder*scaffolder,
StaticVector*inbox, OpenAssemblerChooser*oa, bool*isFinalFusion, BubbleData*bubbleData, bool*alive,
int*CLEAR_n,int*DISTRIBUTE_n,int*FINISH_n,Searcher*searcher,
int*numberOfRanksDoneSeeding, int*numberOfRanksDoneDetectingDistances, int*numberOfRanksDoneSendingDistances,
@@ -341,6 +438,14 @@ SequencesLoader*sl,time_t*lastTime,bool*writeKmerInitialised,Partitioner*partiti
void call_RAY_SLAVE_MODE_SEND_EXTENSION_DATA();
void call_RAY_SLAVE_MODE_DIE();
+ void call_RAY_MPI_TAG_NOTIFY_ERROR(Message*message);
+ void call_RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS(Message*message);
+ void call_RAY_MPI_TAG_COMPUTE_REQUIRED_SPACE_FOR_EXTENSIONS_REPLY(Message*message);
+ void call_RAY_MPI_TAG_ASK_EXTENSION_DATA(Message*message);
+ void call_RAY_MPI_TAG_EXTENSION_DATA_END(Message*message);
+
+ void notifyThatOldDirectoryExists();
+
void registerPlugin(ComputeCore*core);
void resolveSymbols(ComputeCore*core);
};
diff --git a/code/MachineHelper/Makefile b/code/MachineHelper/Makefile
new file mode 100644
index 0000000..c8b350b
--- /dev/null
+++ b/code/MachineHelper/Makefile
@@ -0,0 +1,3 @@
+MachineHelper-y += code/MachineHelper/MachineHelper.o
+
+obj-y += $(MachineHelper-y)
diff --git a/code/Makefile b/code/Makefile
deleted file mode 100644
index 397491e..0000000
--- a/code/Makefile
+++ /dev/null
@@ -1,35 +0,0 @@
-# author Sébastien Boisvert
-#
-# based on http://www.ravnborg.org/kbuild/makefiles.html
-#
-# the code is distributed in a small Ray core that is built on top
-# of the Ray platform.
-#
-# Then, plugins (interface CorePlugin in the Ray Platform)
-# are simply added onto the core (class ComputeCore in the
-# Ray Platform).
-
-MPICXX = mpicxx
-LD = ld
-AR = ar
-RM = rm
-CXXFLAGS= -O3 -Wall -ansi
-
-TARGET=TheRayGenomeAssembler.a
-
-all: $(TARGET)
-
-include application_core/Makefile
-include plugin_*/Makefile
-
-$(TARGET): $(obj-y)
- $(AR) rcs $@ $^
-
-# inference rule
-%.o: %.cpp
- $(MPICXX) $(CXXFLAGS) -I ../RayPlatform -I. -c -o $@ $<
-
-clean:
- $(RM) -f ray_core.o $(PLUGINS-y) $(obj-y) $(TARGET)
-
-
diff --git a/code/MessageProcessor/Makefile b/code/MessageProcessor/Makefile
new file mode 100644
index 0000000..24802f2
--- /dev/null
+++ b/code/MessageProcessor/Makefile
@@ -0,0 +1,3 @@
+MessageProcessor-y += code/MessageProcessor/MessageProcessor.o
+
+obj-y += $(MessageProcessor-y)
diff --git a/code/plugin_MessageProcessor/MessageProcessor.cpp b/code/MessageProcessor/MessageProcessor.cpp
similarity index 88%
rename from code/plugin_MessageProcessor/MessageProcessor.cpp
rename to code/MessageProcessor/MessageProcessor.cpp
index 0c7278d..da74e9c 100644
--- a/code/plugin_MessageProcessor/MessageProcessor.cpp
+++ b/code/MessageProcessor/MessageProcessor.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,160 +14,147 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
//#define GUILLIMIN_BUG
-#include <application_core/constants.h>
-#include <string.h>
-#include <core/OperatingSystem.h>
+#include "MessageProcessor.h"
+
+#include <code/Mock/constants.h>
+#include <code/Mock/common_functions.h>
+#include <code/Mock/Parameters.h>
+#include <code/FusionData/FusionData.h>
+#include <code/SequencesLoader/Read.h>
+#include <code/SequencesIndexer/ReadAnnotation.h>
+#include <code/SeedExtender/Direction.h>
+
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/structures/SplayTree.h>
+#include <RayPlatform/structures/SplayNode.h>
+#include <RayPlatform/structures/SplayTreeIterator.h>
+
#include <assert.h>
-#include <plugin_SequencesLoader/Read.h>
-#include <plugin_MessageProcessor/MessageProcessor.h>
-#include <structures/StaticVector.h>
-#include <application_core/common_functions.h>
-#include <plugin_SequencesIndexer/ReadAnnotation.h>
-#include <structures/SplayTree.h>
-#include <plugin_SeedExtender/Direction.h>
-#include <structures/SplayNode.h>
-#include <structures/SplayTreeIterator.h>
-#include <plugin_FusionData/FusionData.h>
-#include <application_core/Parameters.h>
-#include <core/ComputeCore.h>
+#include <string.h>
__CreatePlugin(MessageProcessor);
- /**/
- /**/
- /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_LOAD_SEQUENCES); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_CONTIG_INFO); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SCAFFOLDING_LINKS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_READ_MARKERS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_READ_MATE); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_READS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SET_WORD_SIZE); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_READS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_INFO); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_READS_FROM_LIST); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_START_INDEXING_SEQUENCES); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEQUENCES_READY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTICES_DATA); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_PURGE_NULL_EDGES); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTICES_DISTRIBUTED); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_OUT_EDGES_DATA_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_OUT_EDGES_DATA); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_START_VERTICES_DISTRIBUTION); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_IN_EDGES_DATA_REPLY); /**/
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_CONTIG_INFO);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SCAFFOLDING_LINKS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_READ_MARKERS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_READ_MATE);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_READS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SET_WORD_SIZE);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_READS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_INFO);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_READS_FROM_LIST);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_START_INDEXING_SEQUENCES);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEQUENCES_READY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTICES_DATA);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_PURGE_NULL_EDGES);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTICES_DISTRIBUTED);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_OUT_EDGES_DATA_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_OUT_EDGES_DATA);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_START_VERTICES_DISTRIBUTION);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_IN_EDGES_DATA_REPLY);
__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_IN_EDGES_DATA); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_QUESTION); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_TEST_NETWORK_MESSAGE); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_COVERAGE_DATA); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_COVERAGE_END); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEND_COVERAGE_VALUES); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_READY_TO_SEED); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_START_SEEDING); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_COVERAGE_AND_MARK); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_EDGES); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_EDGES_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEEDING_IS_OVER); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_SEED_LENGTHS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEND_SEED_LENGTHS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_IS_DONE); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_EXTENSION); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_EXTENSION_DATA); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA_END); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ATTACH_SEQUENCE); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ATTACH_SEQUENCE_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_LENGTH); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_LENGTH_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_WITH_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASSEMBLE_WAVES); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASSEMBLE_WAVES_DONE); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_START_FUSION); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_FUSION_DONE); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_LENGTH); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_LENGTH_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY_END); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATH); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATH_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_HAS_PAIRED_READ); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_HAS_PAIRED_READ_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PAIRED_READ); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PAIRED_READ_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_CLEAR_DIRECTIONS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_CLEAR_DIRECTIONS_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_FINISH_FUSIONS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_FINISH_FUSIONS_FINISHED); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_START); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ELIMINATE_PATH); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_VERTEX); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_VERTEX_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_WRITE_AMOS); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_WRITE_AMOS_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION_IS_DONE); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_LIBRARY_DISTANCE_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_LIBRARY_DISTANCE); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES_FINISHED); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_UPDATE_LIBRARY_INFORMATION); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_KMER_ACADEMY_DISTRIBUTED); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEND_COVERAGE_VALUES_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_READ_SEQUENCE); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_READ_SEQUENCE_REPLY); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_I_FINISHED_SCAFFOLDING); /**/
-__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_CONTIG_CHUNK); /**/
- /**/
-
-
-void MessageProcessor::call_RAY_MPI_TAG_LOAD_SEQUENCES(Message*message){
- uint32_t*incoming=(uint32_t*)message->getBuffer();
- for(int i=0;i<(int)incoming[0];i++){
- if(m_parameters->hasOption("-debug-partitioner"))
- cout<<"Rank "<<m_parameters->getRank()<<" RAY_MPI_TAG_LOAD_SEQUENCES File "<<i<<" "<<incoming[i+1]<<endl;
- m_parameters->setNumberOfSequences(i,incoming[1+i]);
- }
-}
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_QUESTION);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_COVERAGE_DATA);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_COVERAGE_END);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEND_COVERAGE_VALUES);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_READY_TO_SEED);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_START_SEEDING);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_COVERAGE_AND_MARK);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_EDGES);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_EDGES_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEEDING_IS_OVER);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_SEED_LENGTHS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEND_SEED_LENGTHS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_IS_DONE);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_EXTENSION);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ATTACH_SEQUENCE);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ATTACH_SEQUENCE_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_LENGTH);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_LENGTH_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_WITH_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASSEMBLE_WAVES);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASSEMBLE_WAVES_DONE);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_START_FUSION);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_FUSION_DONE);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_LENGTH);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_LENGTH_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY_END);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATH);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATH_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_HAS_PAIRED_READ);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_HAS_PAIRED_READ_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PAIRED_READ);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PAIRED_READ_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_CLEAR_DIRECTIONS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_CLEAR_DIRECTIONS_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_FINISH_FUSIONS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_FINISH_FUSIONS_FINISHED);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_START);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ELIMINATE_PATH);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_VERTEX);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_VERTEX_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_WRITE_AMOS);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_WRITE_AMOS_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION_IS_DONE);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_LIBRARY_DISTANCE_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_LIBRARY_DISTANCE);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES_FINISHED);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_UPDATE_LIBRARY_INFORMATION);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_KMER_ACADEMY_DISTRIBUTED);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEND_COVERAGE_VALUES_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_READ_SEQUENCE);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_READ_SEQUENCE_REPLY);
+__CreateMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_I_FINISHED_SCAFFOLDING);
+
+__CreateMessageTagAdapter(MessageProcessor, RAY_MESSAGE_TAG_PUSH_SEEDS);
+__CreateMessageTagAdapter(MessageProcessor, RAY_MESSAGE_TAG_PUSH_SEEDS_REPLY);
void MessageProcessor::call_RAY_MPI_TAG_CONTIG_INFO(Message*message){
MessageUnit*incoming=(MessageUnit*)message->getBuffer();
m_scaffolder->addMasterContig(incoming[0],incoming[1]);
-
+
MessageUnit*outgoingMessage=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
Message aMessage(outgoingMessage,message->getCount(),
message->getSource(),RAY_MPI_TAG_CONTIG_INFO_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_SCAFFOLDING_LINKS(Message*message){
@@ -201,14 +188,14 @@ void MessageProcessor::call_RAY_MPI_TAG_SCAFFOLDING_LINKS(Message*message){
}
//cout<<__func__<<" "<<leftContig<<" "<<leftStrand<<" "<<rightContig<<" "<<rightStrand<<" "<<average<<" "<<number<<endl;
-
+
SummarizedLink link(leftContig,leftStrand,rightContig,rightStrand,average,number,standardDeviation);
m_scaffolder->addMasterLink(&link);
MessageUnit*outgoingMessage=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
Message aMessage(outgoingMessage,message->getCount(),
message->getSource(),RAY_MPI_TAG_SCAFFOLDING_LINKS_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_GET_READ_MARKERS(Message*message){
@@ -241,7 +228,7 @@ void MessageProcessor::call_RAY_MPI_TAG_GET_READ_MARKERS(Message*message){
}
Message aMessage(outgoingMessage,outputPosition,message->getSource(),RAY_MPI_TAG_GET_READ_MARKERS_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_GET_READ_MATE(Message*message){
@@ -269,7 +256,7 @@ void MessageProcessor::call_RAY_MPI_TAG_GET_READ_MATE(Message*message){
}
Message aMessage(outgoingMessage,j,message->getSource(),RAY_MPI_TAG_GET_READ_MATE_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
/*
@@ -332,7 +319,7 @@ void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_READS(Message*message){
#ifdef ASSERT
// check the padding
-
+
int start=KMER_U64_ARRAY_SIZE+1;
for(int iterator=start;iterator<period;iterator++){
@@ -361,7 +348,7 @@ void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_READS(Message*message){
if(ptr==NULL){
ptr=m_subgraph->getReads(&vertex);
}
-
+
bool gotOne=false;
while(ptr!=NULL&&!gotOne){
@@ -413,7 +400,7 @@ void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_READS(Message*message){
#endif
Message aMessage(outgoingMessage,message->getCount(),message->getSource(),RAY_MPI_TAG_REQUEST_VERTEX_READS_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
#ifdef GUILLIMIN_BUG
if(printBug){
@@ -426,9 +413,10 @@ void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_READS(Message*message){
}
void MessageProcessor::call_RAY_MPI_TAG_SET_WORD_SIZE(Message*message){
+ /*
void*buffer=message->getBuffer();
MessageUnit*incoming=(MessageUnit*)buffer;
- (*m_wordSize)=incoming[0];
+ */
}
/*
@@ -476,7 +464,7 @@ void MessageProcessor::call_RAY_MPI_TAG_VERTEX_READS(Message*message){
}
outgoingMessage[outputPosition++]=(MessageUnit)e;
Message aMessage(outgoingMessage,outputPosition,message->getSource(),RAY_MPI_TAG_VERTEX_READS_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_VERTEX_INFO(Message*message){
@@ -506,14 +494,6 @@ void MessageProcessor::call_RAY_MPI_TAG_VERTEX_INFO(Message*message){
Rank origin=getRankFromPathUniqueId(wave);
-/*
- int progression=incoming[bufferPosition++];
- // add direction in the graph
- Direction*d=(Direction*)m_directionsAllocator->allocate(sizeof(Direction));
- d->constructor(wave,progression,lower);
- m_subgraph->addDirection(&vertex,d);
-*/
-
m_subgraph->find(&vertex)->assemble(origin);
MessageUnit*outgoingMessage=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
@@ -540,7 +520,7 @@ void MessageProcessor::call_RAY_MPI_TAG_VERTEX_INFO(Message*message){
outgoingMessage[3]=(MessageUnit)e;
outgoingMessage[4]=processed;
Message aMessage(outgoingMessage,pos,message->getSource(),RAY_MPI_TAG_VERTEX_INFO_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT(Message*message){
@@ -562,9 +542,9 @@ void MessageProcessor::call_RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT(Message*message
outgoingMessage[i+1]=node->getCoverage(&vertex);
}
}
-
+
Message aMessage(outgoingMessage,count,message->getSource(),RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
/*
@@ -607,7 +587,7 @@ void MessageProcessor::call_RAY_MPI_TAG_VERTEX_READS_FROM_LIST(Message*message){
}
outgoingMessage[0]=processed;
Message aMessage(outgoingMessage,1+processed*4,message->getSource(),RAY_MPI_TAG_VERTEX_READS_FROM_LIST_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_START_INDEXING_SEQUENCES(Message*message){
@@ -678,22 +658,22 @@ void MessageProcessor::call_RAY_MPI_TAG_START_INDEXING_SEQUENCES(Message*message
if(m_parameters->writeCheckpoints() && !m_parameters->hasCheckpoint("GenomeGraph")){
/* announce the user that we are writing a checkpoint */
cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint GenomeGraph"<<endl;
- cout.flush();
ofstream f(m_parameters->getCheckpointFile("GenomeGraph").c_str());
+ ostringstream buffer;
GridTableIterator iterator;
iterator.constructor(m_subgraph,m_parameters->getWordSize(),m_parameters);
LargeCount theSize=m_subgraph->size();
- f.write((char*)&theSize,sizeof(LargeCount));
-
+ buffer.write((char*)&theSize, sizeof(LargeCount));
while(iterator.hasNext()){
Vertex*node=iterator.next();
Kmer key=*(iterator.getKey());
- node->write(&key,&f,m_parameters->getWordSize());
+ node->write(&key, &buffer, m_parameters->getWordSize());
+ flushFileOperationBuffer(false, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
}
-
+ flushFileOperationBuffer(true, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
f.close();
}
@@ -719,23 +699,25 @@ void MessageProcessor::call_RAY_MPI_TAG_VERTICES_DATA(Message*message){
int pos=i;
kmerObject.unpack(incoming,&pos);
-/* make sure that the payload
+/* make sure that the payload
* is for this process and not another one...
*/
#ifdef ASSERT
- Rank rankToFlush=kmerObject.hash_function_1()%m_parameters->getSize();
+ Rank rankToFlush=kmerObject.vertexRank(m_parameters->getSize(),m_parameters->getWordSize(),
+ m_parameters->getColorSpaceMode());
+
assert(rankToFlush==m_rank);
#endif
//bool isTheLowerKmer=false;
Kmer lowerKmer=kmerObject;
-/*
- * TODO: remove call to reverseComplement, this if should never be
+/*
+ * TODO: remove call to reverseComplement, this if should never be
* picked up because only the lowest k-mers are sent
- * I am not sure it would work but if it does that would reduces
- * the number of sent messages
- *
+ * I am not sure it would work but if it does that would reduces
+ * the number of sent messages
+ *
* *** Anyway, the message was delivered anyway already.
*/
Kmer reverseComplement=kmerObject.complementVertex(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
@@ -771,13 +753,12 @@ void MessageProcessor::call_RAY_MPI_TAG_VERTICES_DATA(Message*message){
if((*m_last_value)!=(int)m_subgraph->size() && (int)m_subgraph->size()%100000==0){
(*m_last_value)=m_subgraph->size();
printf("Rank %i has %i vertices\n",m_rank,(int)m_subgraph->size());
- fflush(stdout);
if(m_parameters->showMemoryUsage()){
showMemoryUsage(m_rank);
}
}
-
+
/*
* We have a go. We insert the k-mer in the distributed
@@ -790,13 +771,13 @@ void MessageProcessor::call_RAY_MPI_TAG_VERTICES_DATA(Message*message){
#endif
/*
- * Initialize the k-mer coverage
+ * Initialize the k-mer coverage
* It starts at 0 if the Bloom filter
* is disabled, 1 otherwise.
*/
if(m_subgraph->inserted()){
- tmp->constructor();
-
+ tmp->constructor();
+
CoverageDepth startingValue=0;
/*
@@ -817,11 +798,14 @@ void MessageProcessor::call_RAY_MPI_TAG_VERTICES_DATA(Message*message){
*/
CoverageDepth oldCoverage=tmp->getCoverage(&kmerObject);
CoverageDepth newCoverage=oldCoverage+1;
- tmp->setCoverage(&kmerObject,newCoverage);
+
+ // avoid integer overflow on data type CoverageDepth
+ if(newCoverage > oldCoverage)
+ tmp->setCoverage(&kmerObject,newCoverage);
}
Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_VERTICES_DATA_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_PURGE_NULL_EDGES(Message*message){
@@ -829,7 +813,7 @@ void MessageProcessor::call_RAY_MPI_TAG_PURGE_NULL_EDGES(Message*message){
#ifdef ASSERT
GridTableIterator iterator;
- iterator.constructor(m_subgraph,*m_wordSize,m_parameters);
+ iterator.constructor(m_subgraph, m_parameters->getWordSize(),m_parameters);
LargeCount n=0;
@@ -847,15 +831,13 @@ void MessageProcessor::call_RAY_MPI_TAG_PURGE_NULL_EDGES(Message*message){
printf("Rank %i has %i vertices (completed)\n",m_rank,(int)m_subgraph->size());
- fflush(stdout);
-
+
#if 0
m_subgraph->printStatistics();
#endif
if(m_parameters->showMemoryUsage()){
showMemoryUsage(m_rank);
- fflush(stdout);
}
*m_mode=RAY_SLAVE_MODE_PURGE_NULL_EDGES;
@@ -894,18 +876,18 @@ void MessageProcessor::call_RAY_MPI_TAG_OUT_EDGES_DATA(Message*message){
continue; /* NULL because coverage is too low */
}
- node->addOutgoingEdge(&prefix,&suffix,(*m_wordSize));
+ node->addOutgoingEdge(&prefix,&suffix, m_parameters->getWordSize());
}
Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_OUT_EDGES_DATA_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_START_VERTICES_DISTRIBUTION(Message*message){
// with 4 000 000 kmers, the ratio is objects/bits is 16.
// with 0.000574
- m_bloomBits=__BLOOM_DEFAULT_BITS;
+ m_bloomBits=m_verticesExtractor->getDefaultNumberOfBitsForBloomFilter();
if(m_parameters->hasConfigurationOption("-bloom-filter-bits",1))
m_bloomBits=m_parameters->getConfigurationInteger("-bloom-filter-bits",0);
@@ -953,13 +935,14 @@ void MessageProcessor::call_RAY_MPI_TAG_IN_EDGES_DATA(Message*message){
continue;
}
- node->addIngoingEdge(&suffix,&prefix,(*m_wordSize));
+ node->addIngoingEdge(&suffix,&prefix, m_parameters->getWordSize());
/*
* Make sure that the edge was added.
*/
+//#define ASSERT_123
#ifdef ASSERT
- vector<Kmer>inEdges=node->getIngoingEdges(&suffix,m_parameters->getWordSize());
+ vector<Kmer> inEdges=node->getIngoingEdges(&suffix,m_parameters->getWordSize());
bool found=false;
for(int j=0;j<(int)inEdges.size();j++){
if(inEdges[j]==prefix){
@@ -967,19 +950,28 @@ void MessageProcessor::call_RAY_MPI_TAG_IN_EDGES_DATA(Message*message){
break;
}
}
+ if(!found) {
+ cout << "Error: can not find prefix." << endl;
+ cout << "ingoing edges: " << inEdges.size() << endl;
+ cout << "i= " << i << " count= " << count << endl;
+ cout << prefix.idToWord(m_parameters->getWordSize(),
+ m_parameters->getColorSpaceMode()) << " -> ";
+ cout << suffix.idToWord(m_parameters->getWordSize(),
+ m_parameters->getColorSpaceMode()) << endl;
+ }
assert(found);
#endif
}
Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_IN_EDGES_DATA_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_QUESTION(Message*message){
int source=message->getSource();
Message aMessage(NULL, 0, source, RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER(Message*message){
@@ -989,12 +981,6 @@ void MessageProcessor::call_RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER(Mes
}
}
-/* we reply with an empty message */
-void MessageProcessor::call_RAY_MPI_TAG_TEST_NETWORK_MESSAGE(Message*message){
- Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_TEST_NETWORK_MESSAGE_REPLY,m_rank);
- m_outbox->push_back(aMessage);
-}
-
void MessageProcessor::call_RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION(Message*message){
uint64_t kmersInBloomFilter=0;
@@ -1002,7 +988,7 @@ void MessageProcessor::call_RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION(Message*me
if(m_bloomBits>0){
uint64_t setBits=m_bloomFilter.getNumberOfSetBits();
uint64_t bits=m_bloomFilter.getNumberOfBits();
-
+
#ifdef ASSERT
assert(bits>0);
assert(bits==m_bloomBits);
@@ -1023,7 +1009,7 @@ void MessageProcessor::call_RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION(Message*me
m_bloomFilter.destructor();
cout<<"Rank "<<m_rank<<" destroyed its Bloom filter"<<endl;
-
+
}
// complete incremental resizing, if any
@@ -1052,7 +1038,6 @@ void MessageProcessor::call_RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION(Message*me
if(m_parameters->showMemoryUsage()){
showMemoryUsage(m_rank);
- fflush(stdout);
}
(*m_mode_send_coverage_iterator)=0;
@@ -1072,7 +1057,7 @@ void MessageProcessor::call_RAY_MPI_TAG_COVERAGE_DATA(Message*message){
}
Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_COVERAGE_DATA_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_COVERAGE_END(Message*message){
@@ -1091,7 +1076,7 @@ void MessageProcessor::call_RAY_MPI_TAG_SEND_COVERAGE_VALUES(Message*message){
m_oa->constructor();
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_SEND_COVERAGE_VALUES_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_READY_TO_SEED(Message*message){
@@ -1104,7 +1089,7 @@ void MessageProcessor::call_RAY_MPI_TAG_READY_TO_SEED(Message*message){
// TODO: move checkpointing code in checkpointing.
void MessageProcessor::call_RAY_MPI_TAG_START_SEEDING(Message*message){
/* read checkpoints ReadOffsets and OptimalMarkers */
-
+
if(m_parameters->hasCheckpoint("OptimalMarkers") && m_parameters->hasCheckpoint("ReadOffsets")){
cout<<"Rank "<<m_parameters->getRank()<<" is reading checkpoint ReadOffsets"<<endl;
ifstream f(m_parameters->getCheckpointFile("ReadOffsets").c_str());
@@ -1142,7 +1127,7 @@ void MessageProcessor::call_RAY_MPI_TAG_START_SEEDING(Message*message){
marker->read(&f2,isLower);
Vertex*node=m_subgraph->find(&kmer);
-
+
#ifdef ASSERT
if(node==NULL){
cout<<"Not found: "<<kmer.idToWord(m_parameters->getWordSize(),
@@ -1165,28 +1150,33 @@ void MessageProcessor::call_RAY_MPI_TAG_START_SEEDING(Message*message){
if(m_parameters->writeCheckpoints() && !m_parameters->hasCheckpoint("OptimalMarkers")){
cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint ReadOffsets"<<endl;
ofstream f(m_parameters->getCheckpointFile("ReadOffsets").c_str());
+ ostringstream buffer;
+
LargeCount count=m_myReads->size();
- f.write((char*)&count,sizeof(LargeCount));
+ buffer.write((char*)&count, sizeof(LargeCount));
+
for(int i=0;i<(int)m_myReads->size();i++){
- m_myReads->at(i)->writeOffsets(&f);
+ m_myReads->at(i)->writeOffsets(&buffer);
+ flushFileOperationBuffer(false, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
}
+ flushFileOperationBuffer(true, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
f.close();
-
+
cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint OptimalMarkers"<<endl;
ofstream f2(m_parameters->getCheckpointFile("OptimalMarkers").c_str());
-
GridTableIterator iterator;
iterator.constructor(m_subgraph,m_parameters->getWordSize(),m_parameters);
-
+
count=m_subgraph->size();
- f2.write((char*)&count,sizeof(LargeCount));
+ buffer.write((char*)&count, sizeof(LargeCount));
while(iterator.hasNext()){
Vertex*node=iterator.next();
Kmer key=*(iterator.getKey());
- node->writeAnnotations(&key,&f2,m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
+ node->writeAnnotations(&key,&buffer,m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
+ flushFileOperationBuffer(false, &buffer, &f2, CONFIG_FILE_IO_BUFFER_SIZE);
}
-
+ flushFileOperationBuffer(true, &buffer, &f2, CONFIG_FILE_IO_BUFFER_SIZE);
f2.close();
}
@@ -1228,7 +1218,7 @@ void MessageProcessor::call_RAY_MPI_TAG_GET_COVERAGE_AND_MARK(Message*message){
m_subgraph->addDirection(&vertex,e);
Message aMessage(message2,2,message->getSource(),RAY_MPI_TAG_GET_COVERAGE_AND_MARK_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE(Message*message){
@@ -1262,7 +1252,7 @@ void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE(Message*message)
}
Message aMessage(message2,count,source,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_REPLY(Message*message){
@@ -1290,7 +1280,7 @@ void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES(Message*me
Vertex*node=m_subgraph->find(&vertex);
vector<Kmer> outgoingEdges;
if(node!=NULL)
- outgoingEdges=node->getOutgoingEdges(&vertex,*m_wordSize);
+ outgoingEdges=node->getOutgoingEdges(&vertex, m_parameters->getWordSize());
int outputPosition=(1+4*KMER_U64_ARRAY_SIZE)*i;
message2[outputPosition++]=outgoingEdges.size();
for(int j=0;j<(int)outgoingEdges.size();j++){
@@ -1298,7 +1288,7 @@ void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES(Message*me
}
}
Message aMessage(message2,(count/KMER_U64_ARRAY_SIZE)*(1+4*KMER_U64_ARRAY_SIZE),source,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_EDGES(Message*message){
@@ -1318,9 +1308,9 @@ void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_EDGES(Message*message){
assert(node!=NULL);
#endif
- vector<Kmer> outgoingEdges=node->getOutgoingEdges(&vertex,*m_wordSize);
- vector<Kmer> ingoingEdges=node->getIngoingEdges(&vertex,*m_wordSize);
-
+ vector<Kmer> outgoingEdges=node->getOutgoingEdges(&vertex, m_parameters->getWordSize());
+ vector<Kmer> ingoingEdges=node->getIngoingEdges(&vertex, m_parameters->getWordSize());
+
message2[outputPosition++]=outgoingEdges.size();
for(int i=0;i<(int)outgoingEdges.size();i++){
outgoingEdges[i].pack(message2,&outputPosition);
@@ -1330,7 +1320,7 @@ void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_EDGES(Message*message){
ingoingEdges[i].pack(message2,&outputPosition);
}
Message aMessage(message2,outputPosition,source,RAY_MPI_TAG_REQUEST_VERTEX_EDGES_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_EDGES_REPLY(Message*message){
@@ -1381,7 +1371,7 @@ void MessageProcessor::call_RAY_MPI_TAG_SEEDING_IS_OVER(Message*message){
(*m_master_mode)=RAY_MASTER_MODE_DO_NOTHING;
for(int i=0;i<m_size;i++){
Message aMessage(NULL,0,i,RAY_MPI_TAG_REQUEST_SEED_LENGTHS,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
}
}
@@ -1399,9 +1389,9 @@ void MessageProcessor::call_RAY_MPI_TAG_SEND_SEED_LENGTHS(Message*message){
int number=incoming[i+1];
m_seedingData->m_masterSeedLengths[seedLength]+=number;
}
-
+
Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_SEND_SEED_LENGTHS_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS(Message*message){
@@ -1409,15 +1399,14 @@ void MessageProcessor::call_RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS(Message*mes
if((*m_numberOfRanksDoneSeeding)==m_size){
(*m_numberOfRanksDoneSeeding)=0;
- (*m_master_mode)=RAY_MASTER_MODE_TRIGGER_DETECTION;
- m_seedingData->writeSeedStatistics();
+ (*m_master_mode)=RAY_MASTER_MODE_REGISTER_SEEDS;
}
}
void MessageProcessor::call_RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER(Message*message){
Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS(Message*message){
@@ -1425,7 +1414,7 @@ void MessageProcessor::call_RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS(Message*m
MessageUnit*dummy=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
Message aMessage(dummy,5,source,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY(Message*message){
@@ -1456,11 +1445,13 @@ void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES(Message*mes
Vertex*node=m_subgraph->find(&vertex);
#ifdef ASSERT
if(node==NULL){
- cout<<"Rank="<<m_rank<<" "<<vertex.idToWord(*m_wordSize,m_parameters->getColorSpaceMode())<<" does not exist."<<endl;
+ cout<<"Rank="<<m_rank<<" "<<vertex.idToWord(m_parameters->getWordSize(),
+ m_parameters->getColorSpaceMode())<<" does not exist."<<endl;
}
assert(node!=NULL);
- #endif
- vector<Kmer> ingoingEdges=node->getIngoingEdges(&vertex,*m_wordSize);
+ #endif
+ vector<Kmer> ingoingEdges=node->getIngoingEdges(&vertex,
+ m_parameters->getWordSize());
int outputPosition=i*5;
message2[outputPosition++]=ingoingEdges.size();
for(int j=0;j<(int)ingoingEdges.size();j++){
@@ -1468,7 +1459,7 @@ void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES(Message*mes
}
}
Message aMessage(message2,count*5,source,RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES_REPLY(Message*message){
@@ -1509,12 +1500,6 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_EXTENSION(Message*message){
}
}
-
-void MessageProcessor::call_RAY_MPI_TAG_ASK_EXTENSION_DATA(Message*message){
- (m_seedingData->m_SEEDING_i)=0;
- (m_ed->m_EXTENSION_currentPosition)=0;
-}
-
void MessageProcessor::call_RAY_MPI_TAG_EXTENSION_DATA_REPLY(Message*message){
(*m_ready)=true;
}
@@ -1528,11 +1513,7 @@ void MessageProcessor::call_RAY_MPI_TAG_EXTENSION_DATA(Message*message){
}
Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_EXTENSION_DATA_REPLY,m_rank);
- m_outbox->push_back(aMessage);
-}
-
-void MessageProcessor::call_RAY_MPI_TAG_EXTENSION_DATA_END(Message*message){
- (m_ed->m_EXTENSION_currentRankIsDone)=true;
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_ATTACH_SEQUENCE(Message*message){
@@ -1562,7 +1543,7 @@ void MessageProcessor::call_RAY_MPI_TAG_ATTACH_SEQUENCE(Message*message){
if(coverage==1){
continue;
}
-
+
ReadAnnotation*e=(ReadAnnotation*)m_si->getAllocator()->allocate(sizeof(ReadAnnotation));
#ifdef ASSERT
@@ -1574,7 +1555,7 @@ void MessageProcessor::call_RAY_MPI_TAG_ATTACH_SEQUENCE(Message*message){
}
MessageUnit*message2=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
Message aMessage(message2,count,message->getSource(),RAY_MPI_TAG_ATTACH_SEQUENCE_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_ATTACH_SEQUENCE_REPLY(Message*message){
@@ -1586,12 +1567,13 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION(Message*mess
Rank source=message->getSource();
MessageUnit*incoming=(MessageUnit*)buffer;
Strand strand=incoming[2];
- Kmer vertex=(*m_myReads)[incoming[0]]->getVertex(incoming[1],(*m_wordSize),strand,m_parameters->getColorSpaceMode());
+ Kmer vertex=(*m_myReads)[incoming[0]]->getVertex(incoming[1], m_parameters->getWordSize(),
+ strand,m_parameters->getColorSpaceMode());
MessageUnit*message2=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(MessageUnit));
int bufferPosition=0;
vertex.pack(message2,&bufferPosition);
Message aMessage(message2,bufferPosition,source,RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION_REPLY(Message*message){
@@ -1614,7 +1596,7 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_READ_LENGTH(Message*message){
assert(read!=NULL);
#endif
int length=read->length();
-
+
MessageUnit*message2=(MessageUnit*)m_outboxAllocator->allocate(3*sizeof(MessageUnit));
int pos=0;
message2[pos++]=length;
@@ -1622,7 +1604,7 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_READ_LENGTH(Message*message){
message2[pos++]=read->getReverseOffset();
Message aMessage(message2,pos,source,RAY_MPI_TAG_ASK_READ_LENGTH_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_ASK_READ_LENGTH_REPLY(Message*message){
@@ -1632,16 +1614,39 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_READ_LENGTH_REPLY(Message*message){
(m_ed->m_EXTENSION_receivedLength)=incoming[0];
}
+void MessageProcessor::call_RAY_MESSAGE_TAG_PUSH_SEEDS(Message*message){
+
+#ifdef DEBUG_ISSUE_136
+ cout<< " inside call_RAY_MESSAGE_TAG_PUSH_SEEDS source= " << message->getSource() << endl;
+#endif
+
+// : activate this call
+ call_RAY_MPI_TAG_SAVE_WAVE_PROGRESSION(message);
+
+ Message aMessage(NULL,0,message->getSource(),RAY_MESSAGE_TAG_PUSH_SEEDS_REPLY,m_rank);
+ m_outbox->push_back(&aMessage);
+}
+
+void MessageProcessor::call_RAY_MESSAGE_TAG_PUSH_SEEDS_REPLY(Message*message){
+}
+
void MessageProcessor::call_RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_WITH_REPLY(Message*message){
call_RAY_MPI_TAG_SAVE_WAVE_PROGRESSION(message);
Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_SAVE_WAVE_PROGRESSION(Message*message){
void*buffer=message->getBuffer();
MessageUnit*incoming=(MessageUnit*)buffer;
int count=message->getCount();
+
+#ifdef DEBUG_ISSUE_136
+ cout << " call_RAY_MPI_TAG_SAVE_WAVE_PROGRESSION count: " << count ;
+ cout << " elements in payload: " << count / ( KMER_U64_ARRAY_SIZE + 2 );
+ cout << endl;
+#endif
+
for(int i=0;i<count;i+=(KMER_U64_ARRAY_SIZE+2)){
Kmer vertex;
int pos=i;
@@ -1651,6 +1656,11 @@ void MessageProcessor::call_RAY_MPI_TAG_SAVE_WAVE_PROGRESSION(Message*message){
#ifdef ASSERT
Vertex*node=m_subgraph->find(&vertex);
+ if(node==NULL){
+ cout<<"Error: vertex does not exist: "<<vertex.idToWord(m_parameters->getWordSize(),
+ m_parameters->getColorSpaceMode())<<endl;
+ }
+
assert(node!=NULL);
Vertex copy=*node;
@@ -1658,7 +1668,7 @@ void MessageProcessor::call_RAY_MPI_TAG_SAVE_WAVE_PROGRESSION(Message*message){
#endif
PathHandle wave=incoming[pos++];
-
+
#ifdef ASSERT
if(getRankFromPathUniqueId(wave)>=m_size){
cout<<"Invalid rank: "<<getRankFromPathUniqueId(wave)<<" maximum is "<<m_size-1<<endl;
@@ -1671,6 +1681,21 @@ void MessageProcessor::call_RAY_MPI_TAG_SAVE_WAVE_PROGRESSION(Message*message){
e->constructor(wave,progression,lower);
m_subgraph->addDirection(&vertex,e);
+
+#ifdef DEBUG_ISSUE_136
+ if(
+ vertex.idToWord(m_parameters->getWordSize(), m_parameters->getColorSpaceMode()) == "GCTGAATGGCGTTACCGGCCTGGTTGAGTATCACGAGCATTTCAATCGCTTTTAATCTCCGGGGTTTGCAGACTGCTTAACAGCTCTGCAA"
+ || vertex.idToWord(m_parameters->getWordSize(), m_parameters->getColorSpaceMode()) == "TTGCAGAGCTGTTAAGCAGTCTGCAAACCCCGGAGATTAAAAGCGATTGAAATGCTCGTGATACTCAACCAGGCCGGTAACGCCATTCAGC"
+ || rc.idToWord(m_parameters->getWordSize(), m_parameters->getColorSpaceMode()) == "GCTGAATGGCGTTACCGGCCTGGTTGAGTATCACGAGCATTTCAATCGCTTTTAATCTCCGGGGTTTGCAGACTGCTTAACAGCTCTGCAA"
+ || rc.idToWord(m_parameters->getWordSize(), m_parameters->getColorSpaceMode()) == "TTGCAGAGCTGTTAAGCAGTCTGCAAACCCCGGAGATTAAAAGCGATTGAAATGCTCGTGATACTCAACCAGGCCGGTAACGCCATTCAGC"
+ ){
+ cout << "[DEBUG] Adding annotation for key, region: "<<wave << " location: "<<progression << endl;
+ }
+
+ cout << " Adding annotation <annotation><region>"<<wave << "</region>";
+ cout << "<location>" << progression << "</location></annotation>" << endl;
+#endif
+
}
}
@@ -1692,12 +1717,11 @@ void MessageProcessor::call_RAY_MPI_TAG_START_FUSION(Message*message){
}
void MessageProcessor::call_RAY_MPI_TAG_FUSION_DONE(Message*message){
-
+
MessageUnit*incoming=message->getBuffer();
bool reductionOccured=incoming[0];
if(reductionOccured){
- //cout<<"Reduction occured from RAY_MPI_TAG_FUSION_DONE!"<<endl;
(*m_nextReductionOccured)=true;
}
@@ -1724,8 +1748,8 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE(Message*message){
#ifdef ASSERT
Vertex*node=m_subgraph->find(&vertex);
if(node==NULL){
- cout<<"Source="<<message->getSource()<<" Destination="<<m_rank<<" "<<vertex.idToWord(*m_wordSize,m_parameters->getColorSpaceMode())<<" does not exist, aborting"<<endl;
- cout.flush();
+ cout<<"Source="<<message->getSource()<<" Destination="<<m_rank<<" "<<vertex.idToWord(m_parameters->getWordSize(),
+ m_parameters->getColorSpaceMode())<<" does not exist, aborting"<<endl;
}
assert(node!=NULL);
@@ -1739,7 +1763,7 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE(Message*message){
}
Message aMessage(message2,count,source,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE_REPLY(Message*message){
@@ -1775,7 +1799,7 @@ void MessageProcessor::call_RAY_MPI_TAG_GET_PATH_LENGTH(Message*message){
}
Message aMessage(message2,count,source,RAY_MPI_TAG_GET_PATH_LENGTH_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION(Message*message){
@@ -1803,14 +1827,14 @@ void MessageProcessor::call_RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION(Message*messa
message2[i+0]=coverage;
message2[i+1]=(paths.size()==1);
if(paths.size()==1){
- message2[i+2]=paths[0].getWave();
+ message2[i+2]=paths[0].getWave().getValue();
message2[i+3]=paths[0].getProgression();
}
message2[i+4]=edges;
}
Message aMessage(message2,count,source,RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_GET_PATH_LENGTH_REPLY(Message*message){
@@ -1824,7 +1848,7 @@ void MessageProcessor::call_RAY_MPI_TAG_GET_PATH_LENGTH_REPLY(Message*message){
input: Kmer ; index
output: Kmer ; index ; list
-Receives a k-mer and a first index, and returns a message containing
+Receives a k-mer and a first index, and returns a message containing
the k-mer, the new first index, and a list of path identifiers in the graph.
*/
@@ -1851,7 +1875,7 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_VERTEX_PATHS(Message*message){
assert(firstPathId<(int)paths.size());
#endif
- PathHandle pathId=paths[firstPathId].getWave();
+ PathHandle pathId=paths[firstPathId].getWave().getValue();
int progression=paths[firstPathId].getProgression();
#ifdef ASSERT
@@ -1861,7 +1885,7 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_VERTEX_PATHS(Message*message){
assert(getRankFromPathUniqueId(pathId)<m_size);
#endif
- message2[outputPosition++]=pathId;
+ message2[outputPosition++]=pathId.getValue();
message2[outputPosition++]=progression;
firstPathId++;
}
@@ -1875,10 +1899,10 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_VERTEX_PATHS(Message*message){
if(firstPathId==(int)paths.size()){
Message aMessage(message2,outputPosition,source,RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY_END,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}else{
Message aMessage(message2,outputPosition,source,RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
}
@@ -1908,7 +1932,7 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY(Message*message){
vertex.pack(message2,&outputPosition);
message2[outputPosition++]=incoming[origin];
Message aMessage(message2,outputPosition,message->getSource(),RAY_MPI_TAG_ASK_VERTEX_PATHS,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY_END(Message*message){
@@ -1969,7 +1993,7 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_VERTEX_PATH(Message*message){
Direction d=paths[indexInArray];
kmer.pack(message2,&outputPosition);
- message2[outputPosition++]=d.getWave();
+ message2[outputPosition++]=d.getWave().getValue();
#ifdef ASSERT
Rank rank=getRankFromPathUniqueId(d.getWave());
@@ -1983,7 +2007,7 @@ void MessageProcessor::call_RAY_MPI_TAG_ASK_VERTEX_PATH(Message*message){
}
Message aMessage(message2,outputPosition,source,RAY_MPI_TAG_ASK_VERTEX_PATH_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_ASK_VERTEX_PATH_REPLY(Message*message){
@@ -2019,7 +2043,7 @@ void MessageProcessor::call_RAY_MPI_TAG_HAS_PAIRED_READ(Message*message){
message2[i]=(*m_myReads)[index]->hasPairedRead();
}
Message aMessage(message2,count,source,RAY_MPI_TAG_HAS_PAIRED_READ_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_HAS_PAIRED_READ_REPLY(Message*message){
@@ -2056,7 +2080,7 @@ void MessageProcessor::call_RAY_MPI_TAG_GET_PAIRED_READ(Message*message){
message2[2]=t->getLibrary();
Message aMessage(message2,3,source,RAY_MPI_TAG_GET_PAIRED_READ_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_GET_PAIRED_READ_REPLY(Message*message){
@@ -2082,7 +2106,7 @@ void MessageProcessor::call_RAY_MPI_TAG_CLEAR_DIRECTIONS(Message*message){
// clear graph
GridTableIterator iterator;
- iterator.constructor(m_subgraph,*m_wordSize,m_parameters);
+ iterator.constructor(m_subgraph, m_parameters->getWordSize(), m_parameters);
#ifdef ASSERT
LargeCount cleared=0;
@@ -2097,7 +2121,7 @@ void MessageProcessor::call_RAY_MPI_TAG_CLEAR_DIRECTIONS(Message*message){
cleared++;
Vertex*node=m_subgraph->find(&key);
- assert(node->m_directions == NULL);
+ assert(node->getFirstDirection() == NULL);
#endif
}
@@ -2120,7 +2144,7 @@ void MessageProcessor::call_RAY_MPI_TAG_CLEAR_DIRECTIONS(Message*message){
m_fusionData->m_FINISH_newFusions.clear();
- vector<vector<Kmer> > fusions;
+ vector<GraphPath> fusions;
for(int i=0;i<(int)(m_ed->m_EXTENSION_contigs).size();i++){
bool eliminated=false;
@@ -2139,10 +2163,15 @@ void MessageProcessor::call_RAY_MPI_TAG_CLEAR_DIRECTIONS(Message*message){
continue;
}
- vector<Kmer> rc;
+ GraphPath rc;
+ rc.setKmerLength(m_parameters->getWordSize());
for(int j=(m_ed->m_EXTENSION_contigs)[i].size()-1;j>=0;j--){
- rc.push_back(m_ed->m_EXTENSION_contigs[i][j].complementVertex(*m_wordSize,
- m_parameters->getColorSpaceMode()));
+
+ Kmer otherKmer;
+ m_ed->m_EXTENSION_contigs[i].at(j,&otherKmer);
+ Kmer kmer=otherKmer.complementVertex(m_parameters->getWordSize(),
+ m_parameters->getColorSpaceMode());
+ rc.push_back(&kmer);
}
fusions.push_back(rc);
}
@@ -2184,7 +2213,7 @@ void MessageProcessor::call_RAY_MPI_TAG_CLEAR_DIRECTIONS(Message*message){
cout<<"Rank "<<m_parameters->getRank()<<" cleared, "<<m_ed->m_EXTENSION_contigs.size()<<" paths"<<endl;
Message aMessage(NULL,0,source,RAY_MPI_TAG_CLEAR_DIRECTIONS_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_CLEAR_DIRECTIONS_REPLY(Message*message){
@@ -2220,7 +2249,7 @@ void MessageProcessor::call_RAY_MPI_TAG_DISTRIBUTE_FUSIONS(Message*message){
void MessageProcessor::call_RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY(Message*message){
Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED(Message*message){
@@ -2270,13 +2299,15 @@ void MessageProcessor::call_RAY_MPI_TAG_GET_PATH_VERTEX(Message*message){
assert(position<(int)(m_ed->m_EXTENSION_contigs)[m_fusionData->m_FUSION_identifier_map[id]].size());
#endif
- Kmer a=(m_ed->m_EXTENSION_contigs)[m_fusionData->m_FUSION_identifier_map[id]][position];
+ Kmer object;
+ m_ed->m_EXTENSION_contigs[m_fusionData->m_FUSION_identifier_map[id]].at(position,&object);
+ Kmer*a=&object;
int pos=i;
- a.pack(messageBytes,&pos);
+ a->pack(messageBytes,&pos);
}
Message aMessage(messageBytes,count,source,RAY_MPI_TAG_GET_PATH_VERTEX_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_GET_PATH_VERTEX_REPLY(Message*message){
@@ -2298,21 +2329,6 @@ void MessageProcessor::call_RAY_MPI_TAG_WRITE_AMOS_REPLY(Message*message){
}
void MessageProcessor::call_RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION(Message*message){
- /* write the Seeds checkpoint */
- if(m_parameters->writeCheckpoints() && !m_parameters->hasCheckpoint("Seeds")){
- ofstream f(m_parameters->getCheckpointFile("Seeds").c_str());
- cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint Seeds"<<endl;
- int count=m_seedingData->m_SEEDING_seeds.size();
- f.write((char*)&count,sizeof(int));
- for(int i=0;i<(int)m_seedingData->m_SEEDING_seeds.size();i++){
- int length=m_seedingData->m_SEEDING_seeds[i].size();
- f.write((char*)&length,sizeof(int));
- for(int j=0;j<(int)m_seedingData->m_SEEDING_seeds[i].size();j++){
- m_seedingData->m_SEEDING_seeds[i].at(j)->write(&f);
- }
- }
- f.close();
- }
(m_seedingData->m_SEEDING_i)=0;
(m_ed->m_EXTENSION_currentPosition)=0;
@@ -2339,7 +2355,7 @@ void MessageProcessor::call_RAY_MPI_TAG_LIBRARY_DISTANCE(Message*message){
}
Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_LIBRARY_DISTANCE_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_ASK_LIBRARY_DISTANCES(Message*message){
@@ -2376,7 +2392,7 @@ void MessageProcessor::call_RAY_MPI_TAG_UPDATE_LIBRARY_INFORMATION(Message*messa
}
Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_UPDATE_LIBRARY_INFORMATION_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_KMER_ACADEMY_DISTRIBUTED(Message*message){
@@ -2427,7 +2443,7 @@ void MessageProcessor::call_RAY_MPI_TAG_REQUEST_READ_SEQUENCE(Message*message){
char*dest=(char*)(messageBytes+5);
memcpy(dest,m_myReads->at(index)->getRawSequence(),m_myReads->at(index)->getRequiredBytes());
Message aMessage(messageBytes,toAllocate/sizeof(MessageUnit),source,RAY_MPI_TAG_REQUEST_READ_SEQUENCE_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void MessageProcessor::call_RAY_MPI_TAG_REQUEST_READ_SEQUENCE_REPLY(Message*message){
@@ -2455,31 +2471,6 @@ void MessageProcessor::call_RAY_MPI_TAG_I_FINISHED_SCAFFOLDING(Message*message){
}
}
-void MessageProcessor::call_RAY_MPI_TAG_GET_CONTIG_CHUNK(Message*message){
- MessageUnit*incoming=(MessageUnit*)message->getBuffer();
- PathHandle contigId=incoming[0];
- int position=incoming[1];
- int index=m_fusionData->m_FUSION_identifier_map[contigId];
- int length=m_ed->m_EXTENSION_contigs[index].size();
- MessageUnit*messageContent=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
- int outputPosition=0;
- int origin=outputPosition;
- outputPosition++;
- int count=0;
-
- while(position<length
- && (outputPosition+KMER_U64_ARRAY_SIZE)<(int)(MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit))){
- m_ed->m_EXTENSION_contigs[index][position++].pack(messageContent,&outputPosition);
- count++;
- }
- messageContent[origin]=count;
- Message aMessage(messageContent,
- m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_GET_CONTIG_CHUNK),
- message->getSource(),RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY,
- m_parameters->getRank());
- m_outbox->push_back(aMessage);
-}
-
void MessageProcessor::setScaffolder(Scaffolder*a){
m_scaffolder=a;
}
@@ -2499,7 +2490,6 @@ SequencesLoader*sequencesLoader,ExtensionData*ed,
int rank,
int*m_numberOfMachinesDoneSendingEdges,
FusionData*m_fusionData,
- int*m_wordSize,
ArrayOfReads*m_myReads,
int size,
RingAllocator*m_inboxAllocator,
@@ -2517,7 +2507,6 @@ SequencesLoader*sequencesLoader,ExtensionData*ed,
int*m_readyToSeed,
int*m_FINISH_n,
bool*m_nextReductionOccured,
- MyAllocator*m_directionsAllocator,
int*m_mode_send_coverage_iterator,
map<CoverageDepth,LargeCount>*m_coverageDistribution,
int*m_sequence_ready_machines,
@@ -2551,7 +2540,6 @@ SequencesIndexer*m_si){
m_rank=rank;
this->m_numberOfMachinesDoneSendingEdges=m_numberOfMachinesDoneSendingEdges;
this->m_fusionData=m_fusionData;
- this->m_wordSize=m_wordSize;
this->m_myReads=m_myReads;
m_size=size;
this->m_inboxAllocator=m_inboxAllocator;
@@ -2570,7 +2558,6 @@ SequencesIndexer*m_si){
this->m_readyToSeed=m_readyToSeed;
this->m_FINISH_n=m_FINISH_n;
this->m_nextReductionOccured=m_nextReductionOccured;
- this->m_directionsAllocator=m_directionsAllocator;
this->m_mode_send_coverage_iterator=m_mode_send_coverage_iterator;
this->m_coverageDistribution=m_coverageDistribution;
this->m_sequence_ready_machines=m_sequence_ready_machines;
@@ -2609,6 +2596,8 @@ void MessageProcessor::setSwitchMan(SwitchMan*a){
}
void MessageProcessor::registerPlugin(ComputeCore*core){
+
+ m_core = core;
PluginHandle plugin=core->allocatePluginHandle();
m_plugin=plugin;
@@ -2617,10 +2606,6 @@ void MessageProcessor::registerPlugin(ComputeCore*core){
core->setPluginAuthors(plugin,"Sébastien Boisvert");
core->setPluginLicense(plugin,"GNU General Public License version 3");
- RAY_MPI_TAG_LOAD_SEQUENCES=core->allocateMessageTagHandle(plugin);
- core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_LOAD_SEQUENCES, __GetAdapter(MessageProcessor,RAY_MPI_TAG_LOAD_SEQUENCES));
- core->setMessageTagSymbol(plugin,RAY_MPI_TAG_LOAD_SEQUENCES,"RAY_MPI_TAG_LOAD_SEQUENCES");
-
RAY_MPI_TAG_CONTIG_INFO=core->allocateMessageTagHandle(plugin);
core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_CONTIG_INFO, __GetAdapter(MessageProcessor,RAY_MPI_TAG_CONTIG_INFO));
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_CONTIG_INFO,"RAY_MPI_TAG_CONTIG_INFO");
@@ -2712,10 +2697,6 @@ void MessageProcessor::registerPlugin(ComputeCore*core){
core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER, __GetAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER));
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER,"RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER");
- RAY_MPI_TAG_TEST_NETWORK_MESSAGE=core->allocateMessageTagHandle(plugin);
- core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_TEST_NETWORK_MESSAGE, __GetAdapter(MessageProcessor,RAY_MPI_TAG_TEST_NETWORK_MESSAGE));
- core->setMessageTagSymbol(plugin,RAY_MPI_TAG_TEST_NETWORK_MESSAGE,"RAY_MPI_TAG_TEST_NETWORK_MESSAGE");
-
RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION=core->allocateMessageTagHandle(plugin);
core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION, __GetAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION));
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION,"RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION");
@@ -2813,10 +2794,6 @@ void MessageProcessor::registerPlugin(ComputeCore*core){
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_ASK_EXTENSION,"RAY_MPI_TAG_ASK_EXTENSION");
- RAY_MPI_TAG_ASK_EXTENSION_DATA=core->allocateMessageTagHandle(plugin);
- core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_ASK_EXTENSION_DATA, __GetAdapter(MessageProcessor,RAY_MPI_TAG_ASK_EXTENSION_DATA));
- core->setMessageTagSymbol(plugin,RAY_MPI_TAG_ASK_EXTENSION_DATA,"RAY_MPI_TAG_ASK_EXTENSION_DATA");
-
RAY_MPI_TAG_EXTENSION_DATA_REPLY=core->allocateMessageTagHandle(plugin);
core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_EXTENSION_DATA_REPLY, __GetAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA_REPLY));
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_EXTENSION_DATA_REPLY,"RAY_MPI_TAG_EXTENSION_DATA_REPLY");
@@ -2825,10 +2802,6 @@ void MessageProcessor::registerPlugin(ComputeCore*core){
core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_EXTENSION_DATA, __GetAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA));
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_EXTENSION_DATA,"RAY_MPI_TAG_EXTENSION_DATA");
- RAY_MPI_TAG_EXTENSION_DATA_END=core->allocateMessageTagHandle(plugin);
- core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_EXTENSION_DATA_END, __GetAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA_END));
- core->setMessageTagSymbol(plugin,RAY_MPI_TAG_EXTENSION_DATA_END,"RAY_MPI_TAG_EXTENSION_DATA_END");
-
RAY_MPI_TAG_ATTACH_SEQUENCE=core->allocateMessageTagHandle(plugin);
core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_ATTACH_SEQUENCE, __GetAdapter(MessageProcessor,RAY_MPI_TAG_ATTACH_SEQUENCE));
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_ATTACH_SEQUENCE,"RAY_MPI_TAG_ATTACH_SEQUENCE");
@@ -3038,14 +3011,15 @@ void MessageProcessor::registerPlugin(ComputeCore*core){
core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_I_FINISHED_SCAFFOLDING, __GetAdapter(MessageProcessor,RAY_MPI_TAG_I_FINISHED_SCAFFOLDING));
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_I_FINISHED_SCAFFOLDING,"RAY_MPI_TAG_I_FINISHED_SCAFFOLDING");
- RAY_MPI_TAG_GET_CONTIG_CHUNK=core->allocateMessageTagHandle(plugin);
- core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_GET_CONTIG_CHUNK, __GetAdapter(MessageProcessor,RAY_MPI_TAG_GET_CONTIG_CHUNK));
- core->setMessageTagSymbol(plugin,RAY_MPI_TAG_GET_CONTIG_CHUNK,"RAY_MPI_TAG_GET_CONTIG_CHUNK");
+ __ConfigureMessageTagHandler(MessageProcessor, RAY_MESSAGE_TAG_PUSH_SEEDS);
+ __ConfigureMessageTagHandler(MessageProcessor, RAY_MESSAGE_TAG_PUSH_SEEDS_REPLY);
}
-
void MessageProcessor::resolveSymbols(ComputeCore*core){
+
+ m_core=core;
+
RAY_SLAVE_MODE_PURGE_NULL_EDGES=core->getSlaveModeFromSymbol(m_plugin,"RAY_SLAVE_MODE_PURGE_NULL_EDGES");
RAY_SLAVE_MODE_AMOS=core->getSlaveModeFromSymbol(m_plugin,"RAY_SLAVE_MODE_AMOS");
@@ -3074,8 +3048,7 @@ void MessageProcessor::resolveSymbols(ComputeCore*core){
RAY_MASTER_MODE_TRIGGER_FUSIONS=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_TRIGGER_FUSIONS");
RAY_MASTER_MODE_TRIGGER_SEEDING=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_TRIGGER_SEEDING");
RAY_MASTER_MODE_WRITE_SCAFFOLDS=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_WRITE_SCAFFOLDS");
-
-
+ RAY_MASTER_MODE_REGISTER_SEEDS=m_core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_REGISTER_SEEDS");
RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION");
RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER");
@@ -3138,15 +3111,12 @@ void MessageProcessor::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY_REPLY");
RAY_MPI_TAG_ELIMINATE_PATH=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_ELIMINATE_PATH");
RAY_MPI_TAG_EXTENSION_DATA=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_EXTENSION_DATA");
- RAY_MPI_TAG_EXTENSION_DATA_END=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_EXTENSION_DATA_END");
RAY_MPI_TAG_EXTENSION_DATA_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_EXTENSION_DATA_REPLY");
RAY_MPI_TAG_EXTENSION_IS_DONE=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_EXTENSION_IS_DONE");
RAY_MPI_TAG_EXTENSION_START=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_EXTENSION_START");
RAY_MPI_TAG_FINISH_FUSIONS=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_FINISH_FUSIONS");
RAY_MPI_TAG_FINISH_FUSIONS_FINISHED=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_FINISH_FUSIONS_FINISHED");
RAY_MPI_TAG_FUSION_DONE=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_FUSION_DONE");
- RAY_MPI_TAG_GET_CONTIG_CHUNK=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_CONTIG_CHUNK");
- RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY");
RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION");
RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION_REPLY");
RAY_MPI_TAG_GET_COVERAGE_AND_MARK=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_COVERAGE_AND_MARK");
@@ -3172,7 +3142,6 @@ void MessageProcessor::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_KMER_ACADEMY_DISTRIBUTED=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_KMER_ACADEMY_DISTRIBUTED");
RAY_MPI_TAG_LIBRARY_DISTANCE=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_LIBRARY_DISTANCE");
RAY_MPI_TAG_LIBRARY_DISTANCE_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_LIBRARY_DISTANCE_REPLY");
- RAY_MPI_TAG_LOAD_SEQUENCES=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_LOAD_SEQUENCES");
RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS");
RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY");
RAY_MPI_TAG_OUT_EDGES_DATA=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_OUT_EDGES_DATA");
@@ -3181,7 +3150,6 @@ void MessageProcessor::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER");
RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER_REPLY");
RAY_MPI_TAG_ASK_EXTENSION=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_ASK_EXTENSION");
- RAY_MPI_TAG_ASK_EXTENSION_DATA=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_ASK_EXTENSION_DATA");
RAY_MPI_TAG_ASK_LIBRARY_DISTANCES=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_ASK_LIBRARY_DISTANCES");
RAY_MPI_TAG_ASK_LIBRARY_DISTANCES_FINISHED=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_ASK_LIBRARY_DISTANCES_FINISHED");
RAY_MPI_TAG_ASK_READ_LENGTH=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_ASK_READ_LENGTH");
@@ -3204,7 +3172,6 @@ void MessageProcessor::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_CLEAR_DIRECTIONS=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_CLEAR_DIRECTIONS");
RAY_MPI_TAG_CLEAR_DIRECTIONS_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_CLEAR_DIRECTIONS_REPLY");
- core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_LOAD_SEQUENCES, RAY_SLAVE_MODE_LOAD_SEQUENCES);
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_WRITE_AMOS, RAY_SLAVE_MODE_AMOS);
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_START_INDEXING_SEQUENCES, RAY_SLAVE_MODE_INDEX_SEQUENCES );
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_START_VERTICES_DISTRIBUTION, RAY_SLAVE_MODE_ADD_VERTICES);
@@ -3212,7 +3179,6 @@ void MessageProcessor::resolveSymbols(ComputeCore*core){
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_START_SEEDING, RAY_SLAVE_MODE_START_SEEDING);
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_REQUEST_SEED_LENGTHS, RAY_SLAVE_MODE_SEND_SEED_LENGTHS);
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_ASK_EXTENSION, RAY_SLAVE_MODE_EXTENSION );
- core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_ASK_EXTENSION_DATA, RAY_SLAVE_MODE_SEND_EXTENSION_DATA);
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_ASSEMBLE_WAVES, RAY_SLAVE_MODE_ASSEMBLE_WAVES);
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_START_FUSION, RAY_SLAVE_MODE_FUSION);
@@ -3222,8 +3188,6 @@ void MessageProcessor::resolveSymbols(ComputeCore*core){
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION, RAY_SLAVE_MODE_AUTOMATIC_DISTANCE_DETECTION);
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES, RAY_SLAVE_MODE_SEND_LIBRARY_DISTANCES);
-
- core->setMessageTagReplyMessageTag(m_plugin, RAY_MPI_TAG_GET_CONTIG_CHUNK, RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY );
core->setMessageTagReplyMessageTag(m_plugin, RAY_MPI_TAG_REQUEST_VERTEX_READS, RAY_MPI_TAG_REQUEST_VERTEX_READS_REPLY );
core->setMessageTagReplyMessageTag(m_plugin, RAY_MPI_TAG_GET_READ_MATE, RAY_MPI_TAG_GET_READ_MATE_REPLY );
core->setMessageTagReplyMessageTag(m_plugin, RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE, RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_REPLY );
@@ -3242,9 +3206,7 @@ void MessageProcessor::resolveSymbols(ComputeCore*core){
core->setMessageTagReplyMessageTag(m_plugin, RAY_MPI_TAG_VERTEX_INFO, RAY_MPI_TAG_VERTEX_INFO_REPLY );
core->setMessageTagReplyMessageTag(m_plugin, RAY_MPI_TAG_REQUEST_READ_SEQUENCE, RAY_MPI_TAG_REQUEST_READ_SEQUENCE_REPLY );
core->setMessageTagReplyMessageTag(m_plugin, RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES, RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES_REPLY );
- core->setMessageTagReplyMessageTag(m_plugin, RAY_MPI_TAG_TEST_NETWORK_MESSAGE, RAY_MPI_TAG_TEST_NETWORK_MESSAGE_REPLY );
- core->setMessageTagSize(m_plugin, RAY_MPI_TAG_GET_CONTIG_CHUNK, MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit) );
core->setMessageTagSize(m_plugin, RAY_MPI_TAG_REQUEST_VERTEX_READS, max(5,KMER_U64_ARRAY_SIZE+1) );
core->setMessageTagSize(m_plugin, RAY_MPI_TAG_GET_READ_MATE, 4 );
core->setMessageTagSize(m_plugin, RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE, KMER_U64_ARRAY_SIZE );
@@ -3260,8 +3222,112 @@ void MessageProcessor::resolveSymbols(ComputeCore*core){
core->setMessageTagSize(m_plugin, RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE, KMER_U64_ARRAY_SIZE );
core->setMessageTagSize(m_plugin, RAY_MPI_TAG_ASK_VERTEX_PATH, (KMER_U64_ARRAY_SIZE + 2) );
core->setMessageTagSize(m_plugin, RAY_MPI_TAG_GET_PATH_VERTEX, max(2,KMER_U64_ARRAY_SIZE) );
+ core->setMessageTagSize(m_plugin, RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_WITH_REPLY, KMER_U64_ARRAY_SIZE+2 );
+ core->setMessageTagSize(m_plugin, RAY_MESSAGE_TAG_PUSH_SEEDS, KMER_U64_ARRAY_SIZE+2 );
__BindPlugin(MessageProcessor);
-}
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_CONTIG_INFO);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_SCAFFOLDING_LINKS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_READ_MARKERS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_READ_MATE);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_READS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_SET_WORD_SIZE);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_READS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_INFO);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_READS_FROM_LIST);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_START_INDEXING_SEQUENCES);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_SEQUENCES_READY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_VERTICES_DATA);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_PURGE_NULL_EDGES);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_VERTICES_DISTRIBUTED);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_OUT_EDGES_DATA_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_OUT_EDGES_DATA);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_START_VERTICES_DISTRIBUTION);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_IN_EDGES_DATA_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_IN_EDGES_DATA); /**/
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_QUESTION);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_COVERAGE_DATA);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_COVERAGE_END);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_SEND_COVERAGE_VALUES);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_READY_TO_SEED);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_START_SEEDING);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_COVERAGE_AND_MARK);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_EDGES);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_EDGES_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_SEEDING_IS_OVER);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_SEED_LENGTHS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_SEND_SEED_LENGTHS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_IS_DONE);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_EXTENSION);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ATTACH_SEQUENCE);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ATTACH_SEQUENCE_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_LENGTH);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_LENGTH_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_WITH_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASSEMBLE_WAVES);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASSEMBLE_WAVES_DONE);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_START_FUSION);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_FUSION_DONE);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_LENGTH);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_LENGTH_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY_END);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATH);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATH_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_HAS_PAIRED_READ);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_HAS_PAIRED_READ_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_PAIRED_READ);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_PAIRED_READ_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_CLEAR_DIRECTIONS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_CLEAR_DIRECTIONS_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_FINISH_FUSIONS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_FINISH_FUSIONS_FINISHED);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_START);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ELIMINATE_PATH);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_VERTEX);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_VERTEX_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_WRITE_AMOS);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_WRITE_AMOS_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION_IS_DONE);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_LIBRARY_DISTANCE_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_LIBRARY_DISTANCE);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES_FINISHED);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_UPDATE_LIBRARY_INFORMATION);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_KMER_ACADEMY_DISTRIBUTED);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_SEND_COVERAGE_VALUES_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_READ_SEQUENCE);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_READ_SEQUENCE_REPLY);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_I_FINISHED_SCAFFOLDING);
+
+ m_directionsAllocator = (MyAllocator*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/directionMemoryPool.ray");
+}
diff --git a/code/plugin_MessageProcessor/MessageProcessor.h b/code/MessageProcessor/MessageProcessor.h
similarity index 51%
rename from code/plugin_MessageProcessor/MessageProcessor.h
rename to code/MessageProcessor/MessageProcessor.h
index 330e343..e64165c 100644
--- a/code/plugin_MessageProcessor/MessageProcessor.h
+++ b/code/MessageProcessor/MessageProcessor.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,59 +19,260 @@
*/
-#ifndef _MessageProcessor
-#define _MessageProcessor
-
+#ifndef _MessageProcessor_h
+#define _MessageProcessor_h
+
+#include <code/SeedExtender/OpenAssemblerChooser.h>
+#include <code/SeedExtender/SeedExtender.h>
+#include <code/SequencesLoader/ArrayOfReads.h>
+#include <code/SequencesLoader/SequencesLoader.h>
+#include <code/SequencesIndexer/ReadAnnotation.h>
+#include <code/SequencesIndexer/SequencesIndexer.h>
+#include <code/KmerAcademyBuilder/BloomFilter.h>
+#include <code/Library/Library.h>
+#include <code/SeedingData/SeedingData.h>
+#include <code/FusionData/FusionData.h>
+#include <code/VerticesExtractor/VerticesExtractor.h>
+#include <code/VerticesExtractor/Vertex.h>
+#include <code/VerticesExtractor/GridTable.h>
+#include <code/Mock/Parameters.h>
+#include <code/Scaffolder/Scaffolder.h>
+
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/memory/MyAllocator.h>
+#include <RayPlatform/structures/SplayTree.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/communication/BufferedData.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/communication/MessageRouter.h>
+#include <RayPlatform/scheduling/SwitchMan.h>
+#include <RayPlatform/plugins/CorePlugin.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <vector>
using namespace std;
-#include <memory/RingAllocator.h>
-#include <memory/MyAllocator.h>
-
-#include <plugin_SeedExtender/OpenAssemblerChooser.h>
-
-#include <plugin_SequencesLoader/ArrayOfReads.h>
-#include <structures/SplayTree.h>
-#include <structures/StaticVector.h>
-#include <plugin_SequencesIndexer/ReadAnnotation.h>
-#include <plugin_VerticesExtractor/Vertex.h>
-#include <plugin_KmerAcademyBuilder/BloomFilter.h>
-
-#include <plugin_Library/Library.h>
-#include <plugin_SequencesIndexer/SequencesIndexer.h>
-#include <plugin_SeedingData/SeedingData.h>
-#include <plugin_SeedExtender/SeedExtender.h>
-#include <plugin_SequencesLoader/SequencesLoader.h>
-#include <plugin_FusionData/FusionData.h>
-#include <plugin_VerticesExtractor/VerticesExtractor.h>
-
-#include <plugin_VerticesExtractor/GridTable.h>
-#include <application_core/Parameters.h>
-#include <plugin_Scaffolder/Scaffolder.h>
-
-#include <communication/Message.h>
-#include <communication/BufferedData.h>
-#include <communication/VirtualCommunicator.h>
-#include <communication/MessageRouter.h>
-
-#include <scheduling/SwitchMan.h>
-#include <plugins/CorePlugin.h>
-
-
-#include <core/ComputeCore.h>
-
-
+__DeclarePlugin(MessageProcessor);
+
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_CONTIG_INFO);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SCAFFOLDING_LINKS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_READ_MARKERS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_READ_MATE);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_READS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SET_WORD_SIZE);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_READS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_INFO);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_READS_FROM_LIST);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_START_INDEXING_SEQUENCES);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEQUENCES_READY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTICES_DATA);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_PURGE_NULL_EDGES);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_VERTICES_DISTRIBUTED);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_OUT_EDGES_DATA_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_OUT_EDGES_DATA);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_START_VERTICES_DISTRIBUTION);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_IN_EDGES_DATA_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_IN_EDGES_DATA); /**/
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_QUESTION);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_COVERAGE_DATA);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_COVERAGE_END);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEND_COVERAGE_VALUES);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_READY_TO_SEED);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_START_SEEDING);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_COVERAGE_AND_MARK);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_EDGES);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_EDGES_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEEDING_IS_OVER);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_SEED_LENGTHS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEND_SEED_LENGTHS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_IS_DONE);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_EXTENSION);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ATTACH_SEQUENCE);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ATTACH_SEQUENCE_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_LENGTH);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_LENGTH_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_WITH_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASSEMBLE_WAVES);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASSEMBLE_WAVES_DONE);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_START_FUSION);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_FUSION_DONE);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_LENGTH);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_LENGTH_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY_END);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATH);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATH_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_HAS_PAIRED_READ);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_HAS_PAIRED_READ_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PAIRED_READ);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PAIRED_READ_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_CLEAR_DIRECTIONS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_CLEAR_DIRECTIONS_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_FINISH_FUSIONS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_FINISH_FUSIONS_FINISHED);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_START);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ELIMINATE_PATH);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_VERTEX);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_VERTEX_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_WRITE_AMOS);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_WRITE_AMOS_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION_IS_DONE);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_LIBRARY_DISTANCE_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_LIBRARY_DISTANCE);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES_FINISHED);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_UPDATE_LIBRARY_INFORMATION);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_KMER_ACADEMY_DISTRIBUTED);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_SEND_COVERAGE_VALUES_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_READ_SEQUENCE);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_READ_SEQUENCE_REPLY);
+__DeclareMessageTagAdapter(MessageProcessor,RAY_MPI_TAG_I_FINISHED_SCAFFOLDING);
+
+__DeclareMessageTagAdapter(MessageProcessor, RAY_MESSAGE_TAG_PUSH_SEEDS);
+__DeclareMessageTagAdapter(MessageProcessor, RAY_MESSAGE_TAG_PUSH_SEEDS_REPLY);
/**
* MessageProcessor receives all the messages of a MPI rank
* Message objects may also be checked using the Message inbox (m_inbox)
*
* Sometimes, a message will generate a reply (_REPLY)
+ *
* \author Sébastien Boisvert
*/
class MessageProcessor : public CorePlugin {
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_CONTIG_INFO);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_SCAFFOLDING_LINKS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_GET_READ_MARKERS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_GET_READ_MATE);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_READS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_SET_WORD_SIZE);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_READS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_INFO);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_VERTEX_READS_FROM_LIST);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_START_INDEXING_SEQUENCES);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_SEQUENCES_READY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_VERTICES_DATA);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_PURGE_NULL_EDGES);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_VERTICES_DISTRIBUTED);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_OUT_EDGES_DATA_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_OUT_EDGES_DATA);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_START_VERTICES_DISTRIBUTION);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_IN_EDGES_DATA_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_IN_EDGES_DATA); /**/
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_QUESTION);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_COVERAGE_DATA);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_COVERAGE_END);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_SEND_COVERAGE_VALUES);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_READY_TO_SEED);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_START_SEEDING);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_GET_COVERAGE_AND_MARK);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_EDGES);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_EDGES_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_SEEDING_IS_OVER);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_SEED_LENGTHS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_SEND_SEED_LENGTHS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_IS_DONE);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_EXTENSION);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_DATA);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ATTACH_SEQUENCE);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ATTACH_SEQUENCE_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_LENGTH);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_READ_LENGTH_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_WITH_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_SAVE_WAVE_PROGRESSION_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASSEMBLE_WAVES);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASSEMBLE_WAVES_DONE);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_START_FUSION);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_FUSION_DONE);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_LENGTH);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_LENGTH_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATHS_REPLY_END);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATH);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_VERTEX_PATH_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_HAS_PAIRED_READ);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_HAS_PAIRED_READ_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_GET_PAIRED_READ);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_GET_PAIRED_READ_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_CLEAR_DIRECTIONS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_CLEAR_DIRECTIONS_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_FINISH_FUSIONS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_FINISH_FUSIONS_FINISHED);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_EXTENSION_START);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ELIMINATE_PATH);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_VERTEX);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_GET_PATH_VERTEX_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_WRITE_AMOS);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_WRITE_AMOS_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_AUTOMATIC_DISTANCE_DETECTION_IS_DONE);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_LIBRARY_DISTANCE_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_LIBRARY_DISTANCE);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_ASK_LIBRARY_DISTANCES_FINISHED);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_UPDATE_LIBRARY_INFORMATION);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_KMER_ACADEMY_DISTRIBUTED);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_SEND_COVERAGE_VALUES_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_READ_SEQUENCE);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_REQUEST_READ_SEQUENCE_REPLY);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_I_FINISHED_SCAFFOLDING);
+
+ __AddAdapter(MessageProcessor, RAY_MESSAGE_TAG_PUSH_SEEDS);
+ __AddAdapter(MessageProcessor, RAY_MESSAGE_TAG_PUSH_SEEDS_REPLY);
+
uint64_t m_bloomBits;
MessageTag RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION;
@@ -135,15 +336,12 @@ class MessageProcessor : public CorePlugin {
MessageTag RAY_MPI_TAG_DISTRIBUTE_FUSIONS_FINISHED_REPLY_REPLY;
MessageTag RAY_MPI_TAG_ELIMINATE_PATH;
MessageTag RAY_MPI_TAG_EXTENSION_DATA;
- MessageTag RAY_MPI_TAG_EXTENSION_DATA_END;
MessageTag RAY_MPI_TAG_EXTENSION_DATA_REPLY;
MessageTag RAY_MPI_TAG_EXTENSION_IS_DONE;
MessageTag RAY_MPI_TAG_EXTENSION_START;
MessageTag RAY_MPI_TAG_FINISH_FUSIONS;
MessageTag RAY_MPI_TAG_FINISH_FUSIONS_FINISHED;
MessageTag RAY_MPI_TAG_FUSION_DONE;
- MessageTag RAY_MPI_TAG_GET_CONTIG_CHUNK;
- MessageTag RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY;
MessageTag RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION;
MessageTag RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION_REPLY;
MessageTag RAY_MPI_TAG_GET_COVERAGE_AND_MARK;
@@ -171,7 +369,6 @@ class MessageProcessor : public CorePlugin {
MessageTag RAY_MPI_TAG_KMER_ACADEMY_DISTRIBUTED;
MessageTag RAY_MPI_TAG_LIBRARY_DISTANCE;
MessageTag RAY_MPI_TAG_LIBRARY_DISTANCE_REPLY;
- MessageTag RAY_MPI_TAG_LOAD_SEQUENCES;
MessageTag RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS;
MessageTag RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY;
MessageTag RAY_MPI_TAG_OUT_EDGES_DATA;
@@ -180,7 +377,6 @@ class MessageProcessor : public CorePlugin {
MessageTag RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER;
MessageTag RAY_MPI_TAG_ACTIVATE_RELAY_CHECKER_REPLY;
MessageTag RAY_MPI_TAG_ASK_EXTENSION;
- MessageTag RAY_MPI_TAG_ASK_EXTENSION_DATA;
MessageTag RAY_MPI_TAG_ASK_LIBRARY_DISTANCES;
MessageTag RAY_MPI_TAG_ASK_LIBRARY_DISTANCES_FINISHED;
MessageTag RAY_MPI_TAG_ASK_READ_LENGTH;
@@ -203,6 +399,9 @@ class MessageProcessor : public CorePlugin {
MessageTag RAY_MPI_TAG_CLEAR_DIRECTIONS;
MessageTag RAY_MPI_TAG_CLEAR_DIRECTIONS_REPLY;
+ MessageTag RAY_MESSAGE_TAG_PUSH_SEEDS;
+ MessageTag RAY_MESSAGE_TAG_PUSH_SEEDS_REPLY;
+
MasterMode RAY_MASTER_MODE_ASK_DISTANCES;
MasterMode RAY_MASTER_MODE_DO_NOTHING;
MasterMode RAY_MASTER_MODE_PREPARE_SEEDING;
@@ -214,6 +413,8 @@ class MessageProcessor : public CorePlugin {
MasterMode RAY_MASTER_MODE_TRIGGER_SEEDING;
MasterMode RAY_MASTER_MODE_WRITE_SCAFFOLDS;
+ MasterMode RAY_MASTER_MODE_REGISTER_SEEDS;
+
SlaveMode RAY_SLAVE_MODE_PURGE_NULL_EDGES;
SlaveMode RAY_SLAVE_MODE_AMOS;
@@ -273,7 +474,6 @@ class MessageProcessor : public CorePlugin {
int m_rank;
int*m_numberOfMachinesDoneSendingEdges;
FusionData*m_fusionData;
- int*m_wordSize;
ArrayOfReads*m_myReads;
int m_size;
SequencesIndexer*m_si;
@@ -312,7 +512,6 @@ class MessageProcessor : public CorePlugin {
int*m_numberOfRanksWithCoverageData;
SeedExtender*seedExtender;
-
public:
void constructor(
@@ -331,7 +530,6 @@ ExtensionData*ed,
int rank,
int*m_numberOfMachinesDoneSendingEdges,
FusionData*m_fusionData,
- int*m_wordSize,
ArrayOfReads*m_myReads,
int size,
RingAllocator*m_inboxAllocator,
@@ -349,7 +547,6 @@ ExtensionData*ed,
int*m_readyToSeed,
int*m_FINISH_n,
bool*m_nextReductionOccured,
- MyAllocator*m_directionsAllocator,
int*m_mode_send_coverage_iterator,
map<CoverageDepth,LargeCount>*m_coverageDistribution,
int*m_sequence_ready_machines,
@@ -378,7 +575,6 @@ SequencesIndexer*m_si
// list of declarations
- void call_RAY_MPI_TAG_LOAD_SEQUENCES(Message*message);
void call_RAY_MPI_TAG_CONTIG_INFO(Message*message);
void call_RAY_MPI_TAG_SCAFFOLDING_LINKS(Message*message);
void call_RAY_MPI_TAG_GET_READ_MARKERS(Message*message);
@@ -401,7 +597,6 @@ SequencesIndexer*m_si
void call_RAY_MPI_TAG_IN_EDGES_DATA(Message*message);
void call_RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_QUESTION(Message*message);
void call_RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION_ANSWER(Message*message);
- void call_RAY_MPI_TAG_TEST_NETWORK_MESSAGE(Message*message);
void call_RAY_MPI_TAG_PREPARE_COVERAGE_DISTRIBUTION(Message*message);
void call_RAY_MPI_TAG_COVERAGE_DATA(Message*message);
void call_RAY_MPI_TAG_COVERAGE_END(Message*message);
@@ -426,10 +621,8 @@ SequencesIndexer*m_si
void call_RAY_MPI_TAG_REQUEST_VERTEX_INGOING_EDGES_REPLY(Message*message);
void call_RAY_MPI_TAG_EXTENSION_IS_DONE(Message*message);
void call_RAY_MPI_TAG_ASK_EXTENSION(Message*message);
- void call_RAY_MPI_TAG_ASK_EXTENSION_DATA(Message*message);
void call_RAY_MPI_TAG_EXTENSION_DATA_REPLY(Message*message);
void call_RAY_MPI_TAG_EXTENSION_DATA(Message*message);
- void call_RAY_MPI_TAG_EXTENSION_DATA_END(Message*message);
void call_RAY_MPI_TAG_ATTACH_SEQUENCE(Message*message);
void call_RAY_MPI_TAG_ATTACH_SEQUENCE_REPLY(Message*message);
void call_RAY_MPI_TAG_ASK_READ_VERTEX_AT_POSITION(Message*message);
@@ -483,12 +676,12 @@ SequencesIndexer*m_si
void call_RAY_MPI_TAG_REQUEST_READ_SEQUENCE(Message*message);
void call_RAY_MPI_TAG_REQUEST_READ_SEQUENCE_REPLY(Message*message);
void call_RAY_MPI_TAG_I_FINISHED_SCAFFOLDING(Message*message);
- void call_RAY_MPI_TAG_GET_CONTIG_CHUNK(Message*message);
+ void call_RAY_MESSAGE_TAG_PUSH_SEEDS(Message*message);
+ void call_RAY_MESSAGE_TAG_PUSH_SEEDS_REPLY(Message*message);
+
void registerPlugin(ComputeCore*core);
void resolveSymbols(ComputeCore*core);
};
#endif
-
-
diff --git a/code/Mock/Makefile b/code/Mock/Makefile
new file mode 100644
index 0000000..b80771a
--- /dev/null
+++ b/code/Mock/Makefile
@@ -0,0 +1,5 @@
+Mock-y += code/Mock/Parameters.o
+Mock-y += code/Mock/common_functions.o
+Mock-y += code/Mock/Mock.o
+
+obj-y += $(Mock-y)
diff --git a/code/Mock/Mock.cpp b/code/Mock/Mock.cpp
new file mode 100644
index 0000000..2bad1af
--- /dev/null
+++ b/code/Mock/Mock.cpp
@@ -0,0 +1,47 @@
+/*
+ Ray
+ Copyright (C) 2012 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+*/
+
+#include "Mock.h"
+#include "Parameters.h"
+#include <code/Surveyor/Mother.h>
+
+void Mock::registerPlugin(ComputeCore*core){
+ m_plugin=core->allocatePluginHandle();
+
+ core->setPluginName(m_plugin,"Mock");
+ core->setPluginDescription(m_plugin,"This plugin does nothing at all.");
+ core->setPluginAuthors(m_plugin,"Sébastien Boisvert");
+ core->setPluginLicense(m_plugin,"GNU General Public License version 3");
+}
+
+void Mock::resolveSymbols(ComputeCore*core){
+ /* nothing to resolve... */
+
+ m_core = core;
+ Parameters * parameters=(Parameters*)m_core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/Parameters.ray");
+
+ if(parameters->hasOption("-run-surveyor")) {
+ Mother * mother = new Mother();
+
+ mother->setParameters(parameters);
+ m_core->spawnActor(mother);
+ }
+}
+
diff --git a/RayPlatform/cryptography/crypto.h b/code/Mock/Mock.h
similarity index 66%
rename from RayPlatform/cryptography/crypto.h
rename to code/Mock/Mock.h
index 1a7b33b..baa250f 100644
--- a/RayPlatform/cryptography/crypto.h
+++ b/code/Mock/Mock.h
@@ -1,6 +1,6 @@
/*
- RayPlatform
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Ray
+ Copyright (C) 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -16,17 +16,24 @@
You have received a copy of the GNU General Public License
along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
-
*/
-#ifndef _crypto
-#define _crypto
+#ifndef _Mock_h
+#define _Mock_h
-#include<stdint.h>
+#include <RayPlatform/core/ComputeCore.h>
-uint64_t uniform_hashing_function_1_64_64(uint64_t key);
-uint64_t uniform_hashing_function_2_64_64(uint64_t key);
+/**
+ * This plugin is a mock, it does nothing.
+ *
+ * \author Sébastien Boisvert
+ **/
+class Mock: public CorePlugin {
-uint32_t computeCyclicRedundancyCode32(uint8_t*bytes,uint32_t numberOfBytes);
+public:
+ void registerPlugin(ComputeCore*core);
+ void resolveSymbols(ComputeCore*core);
+};
#endif
+
diff --git a/code/application_core/Parameters.cpp b/code/Mock/Parameters.cpp
similarity index 86%
rename from code/application_core/Parameters.cpp
rename to code/Mock/Parameters.cpp
index 7e81e7f..97970dd 100644
--- a/code/application_core/Parameters.cpp
+++ b/code/Mock/Parameters.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -23,22 +23,27 @@
* TODO: add option -minimumScaffoldLength
*/
-#include <core/OperatingSystem.h>
-#include<application_core/common_functions.h>
-#include<application_core/Parameters.h>
-#include <plugin_Library/LibraryPeakFinder.h>
-#include<plugin_SequencesLoader/Read.h>
-#include<plugin_SequencesLoader/Loader.h>
-#include <memory/MyAllocator.h>
-
-#include<string>
-#include<sstream>
-#include<iostream>
-#include<vector>
-#include<cstdlib>
-#include<fstream>
-#include<assert.h>
-#include<math.h>
+#include "Parameters.h"
+#include "common_functions.h"
+
+#include <code/Library/LibraryPeakFinder.h>
+#include <code/SequencesLoader/Read.h>
+#include <code/SequencesLoader/Loader.h>
+#include <code/SequencesLoader/ReadHandle.h>
+#include <code/SequencesLoader/SequenceFileDetector.h>
+
+#include <RayPlatform/memory/MyAllocator.h>
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/core/types.h> /* for CONFIG_MINI_RANKS */
+
+#include <string>
+#include <sstream>
+#include <iostream>
+#include <vector>
+#include <cstdlib>
+#include <fstream>
+#include <assert.h>
+#include <math.h>
using namespace std;
void Parameters::getIndexes(int count,vector<int>*out){
@@ -89,9 +94,6 @@ Parameters::Parameters(){
/** use the new NovaEngine (TM) */
m_options.insert("-use-NovaEngine");
- /** the default is to not use a default value */
- m_degree=2;
-
m_checkpointDirectory="Checkpoints";
m_hasCheckpointDirectory=false;
@@ -114,10 +116,6 @@ bool Parameters::runProfiler(){
return m_profiler;
}
-bool Parameters::debugSeeds(){
- return m_debugSeeds;
-}
-
int Parameters::getWordSize(){
return m_wordSize;
}
@@ -125,7 +123,7 @@ int Parameters::getWordSize(){
void Parameters::loadCommandsFromFile(char*file){
// first, load the configuration into a stream
-
+
ostringstream content;
#define __BLOCK_SIZE 4096
@@ -197,7 +195,7 @@ void Parameters::loadCommandsFromFile(char*file){
m_commands.push_back(token);
}
-
+
}
void Parameters::loadCommandsFromArguments(int argc,char**argv){
@@ -209,6 +207,8 @@ void Parameters::loadCommandsFromArguments(int argc,char**argv){
/* parse commands */
void Parameters::parseCommands(){
+ //cout << "DEBUG Parameters::parseCommands" << endl;
+
m_initiated=true;
set<string> commands;
@@ -249,7 +249,7 @@ void Parameters::parseCommands(){
pairedReadsCommands.insert("LoadPairedEndReads");
pairedReadsCommands.insert("-LoadPairedEndReads");
pairedReadsCommands.insert("--LoadPairedEndReads");
-
+
set<string> interleavedCommands;
interleavedCommands.insert("-i");
@@ -268,9 +268,10 @@ void Parameters::parseCommands(){
set<string> outputFileCommands;
outputFileCommands.insert("-o");
+ outputFileCommands.insert("-output");
outputFileCommands.insert("-OutputFile");
outputFileCommands.insert("--OutputFile");
-
+
set<string> memoryMappedFileCommands;
memoryMappedFileCommands.insert("-MemoryPrefix");
@@ -293,10 +294,6 @@ void Parameters::parseCommands(){
debugBubbles.insert("-debug-bubbles");
debugBubbles.insert("--debug-bubbles");
- set<string> debugSeeds;
- debugSeeds.insert("-debug-seeds");
- debugSeeds.insert("--debug-seeds");
-
set<string> runProfiler;
runProfiler.insert("-run-profiler");
runProfiler.insert("--run-profiler");
@@ -342,6 +339,9 @@ void Parameters::parseCommands(){
set<string> maximumSeedCoverage;
maximumSeedCoverage.insert("-use-maximum-seed-coverage");
+ set<string> detectSequenceFiles;
+ detectSequenceFiles.insert("-detect-sequence-files");
+
vector<set<string> > toAdd;
toAdd.push_back(checkpoints);
toAdd.push_back(coloringOneColor);
@@ -363,15 +363,15 @@ void Parameters::parseCommands(){
toAdd.push_back(reduceMemoryUsage);
toAdd.push_back(memoryMappedFileCommands);
toAdd.push_back(connectionType);
- toAdd.push_back(showMemory);
+ toAdd.push_back(showMemory);
toAdd.push_back(debugBubbles);
- toAdd.push_back(debugSeeds);
toAdd.push_back(runProfiler);
toAdd.push_back(showContext);
toAdd.push_back(showMalloc);
toAdd.push_back(writeKmers);
toAdd.push_back(colorSpaceMode);
toAdd.push_back(maximumSeedCoverage);
+ toAdd.push_back(detectSequenceFiles);
for(int i=0;i<(int)toAdd.size();i++){
for(set<string>::iterator j=toAdd[i].begin();j!=toAdd[i].end();j++){
@@ -383,8 +383,57 @@ void Parameters::parseCommands(){
bool providedMemoryPrefix=false;
+ // expand user command
+
+ vector<string> & extraCommands = m_commands;
+
+ //cout << "DEBUG adding extraCommands" << endl;
+
+ for(int i=0;i<(int)m_commands.size();i++){
+ string token=m_commands[i];
+
+ if(detectSequenceFiles.count(token) > 0) {
+
+ i++;
+ int items = m_commands.size() - i;
+
+ if(items < 1) {
+ if(m_rank == MASTER_RANK)
+ cout << "Error: " << token << " needs 1 item, you provided " << items << endl;
+
+ m_error = true;
+ return;
+ }
+ string directory = m_commands[i];
+
+ SequenceFileDetector sequenceFileDetector;
+ sequenceFileDetector.detectSequenceFiles(directory);
+
+ for(int i = 0 ; i < (int)sequenceFileDetector.getLeftFiles().size() ; ++i) {
+
+ string & leftFile = sequenceFileDetector.getLeftFiles()[i];
+ string & rightFile = sequenceFileDetector.getRightFiles()[i];
+
+ extraCommands.push_back("LoadPairedEndReads");
+ extraCommands.push_back(leftFile);
+ extraCommands.push_back(rightFile);
+ }
+
+ for(int i = 0; i < (int)sequenceFileDetector.getSingleFiles().size() ; ++i) {
+
+ string & singleFile = sequenceFileDetector.getSingleFiles()[i];
+
+ extraCommands.push_back("LoadSingleEndReads");
+ extraCommands.push_back(singleFile);
+ }
+ }
+ }
+
+ // now parse the commands.
+
for(int i=0;i<(int)m_commands.size();i++){
string token=m_commands[i];
+
if(singleReadsCommands.count(token)>0){
i++;
int items=m_commands.size()-i;
@@ -406,6 +455,8 @@ void Parameters::parseCommands(){
cout<<"-s (single sequences)"<<endl;
cout<<" Sequences: "<<token<<endl;
}
+
+
}else if(memoryMappedFileCommands.count(token)>0){
i++;
int items=m_commands.size()-i;
@@ -441,6 +492,7 @@ void Parameters::parseCommands(){
m_hasCheckpointDirectory=true;
cout<<"Rank "<<m_rank<<" checkpoint directory: "<<m_checkpointDirectory;
+ cout << endl;
if(m_rank==MASTER_RANK){
@@ -590,7 +642,7 @@ void Parameters::parseCommands(){
m_singleEndReadsFile.push_back(left);
i++;
token=m_commands[i];
-
+
// add right file
string right=token;
int rightFile=m_singleEndReadsFile.size();
@@ -622,7 +674,7 @@ void Parameters::parseCommands(){
if(!isValidInteger(m_commands[i+1].c_str())
|| !isValidInteger(m_commands[i+2].c_str())){
-
+
if(m_rank==MASTER_RANK){
cout<<"Warning: invalid integer values for distances, you provided: -p ";
cout<<left<<" "<<right<<" "<<m_commands[i+1];
@@ -692,7 +744,7 @@ void Parameters::parseCommands(){
m_reducerIsActivated=true;
m_reducerPeriod=1000000;
-
+
if(items==1){
m_reducerPeriod=atoi(m_commands[i+1].c_str());
}
@@ -738,7 +790,7 @@ void Parameters::parseCommands(){
token=m_commands[i];
m_peakCoverage=atoi(token.c_str());
m_providedPeakCoverage=true;
-
+
}else if(connectionType.count(token)>0){
i++;
int items=m_commands.size()-i;
@@ -769,7 +821,7 @@ void Parameters::parseCommands(){
}else if(phylogeny.count(token)>0){
- cout<<"Enabling CorePlugin 'PhylogenyViewer'"<<endl;
+ cout<<"Enabling CorePlugin 'TaxonomyViewer'"<<endl;
i++;
int items=m_commands.size()-i;
@@ -841,13 +893,13 @@ void Parameters::parseCommands(){
if(m_wordSize<15){
m_wordSize=15;
}
- if(m_wordSize>MAXKMERLENGTH){
+ if(m_wordSize>CONFIG_MAXKMERLENGTH){
if(m_rank==MASTER_RANK){
cout<<endl;
- cout<<"Rank "<<MASTER_RANK<<": Warning, k > MAXKMERLENGTH"<<endl;
- cout<<"Rank "<<MASTER_RANK<<": Change MAXKMERLENGTH in the Makefile and recompile Ray."<<endl;
+ cout<<"Rank "<<MASTER_RANK<<": Warning, k > CONFIG_MAXKMERLENGTH"<<endl;
+ cout<<"Rank "<<MASTER_RANK<<": Change CONFIG_MAXKMERLENGTH in the Makefile and recompile Ray."<<endl;
}
- m_wordSize=MAXKMERLENGTH;
+ m_wordSize=CONFIG_MAXKMERLENGTH;
}
if(m_wordSize%2==0){
@@ -883,11 +935,6 @@ void Parameters::parseCommands(){
cout<<"Enabling color-space mode"<<endl;
cout<<"All reads should be in color space."<<endl;
}
- }else if(debugSeeds.count(token)>0){
- m_debugSeeds=true;
- if(m_rank==MASTER_RANK){
- printf("Enabling seed debug mode.\n");
- }
}else if(showMemory.count(token)>0){
m_showMemoryUsage=true;
if(m_rank==MASTER_RANK){
@@ -901,9 +948,19 @@ void Parameters::parseCommands(){
}
}
- int maximumNumberOfFiles=MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(uint32_t);
-
+/*
+ * This formula is only valid when using paired reads.
+ * There is not really any limit on the number of files.
+ * The limit is on the number of paired libraries (2^sizeof(uint16_t)).
+ */
+ #ifdef ASSERT
+ LibraryHandle maximumLibraryIndex=0;
+ maximumLibraryIndex--;
+
+ int maximumNumberOfFiles=(maximumLibraryIndex+1)*2;
+
assert((int)m_singleEndReadsFile.size()<=maximumNumberOfFiles);
+ #endif
LargeCount result=1;
for(int p=0;p<m_wordSize;p++){
@@ -912,6 +969,9 @@ void Parameters::parseCommands(){
}
void Parameters::writeCommandFile(){
+
+ //cout << "DEBUG Parameters::writeCommandFile" << endl;
+
cout<<endl;
cout<<"Ray command: "<<endl;
@@ -919,8 +979,16 @@ void Parameters::writeCommandFile(){
commandFile<<getPrefix()<<"RayCommand.txt";
ofstream f(commandFile.str().c_str());
- f<<"mpiexec -n "<<getSize()<<" Ray \\"<<endl;
- cout<<"mpiexec -n "<<getSize()<<" Ray \\"<<endl;
+ int numberOfRays=getSize();
+
+ #ifdef ASSERT
+ assert(numberOfRays%m_numberOfMiniRanksPerRank==0);
+ #endif
+
+ numberOfRays/=m_numberOfMiniRanksPerRank;
+
+ f<<"mpiexec -n "<<numberOfRays<<" Ray \\"<<endl;
+ cout<<"mpiexec -n "<<numberOfRays<<" Ray \\"<<endl;
for(int i=0;i<(int)m_originalCommands.size();i++){
if(i!=(int)m_originalCommands.size()-1){
@@ -938,21 +1006,42 @@ void Parameters::writeCommandFile(){
cout<<endl;
cout<<"k-mer length: "<<m_wordSize<<endl;
-
- if(m_reducerIsActivated){
- cout<<"Memory Consumption Reducer is enabled, threshold="<<m_reducerPeriod<<endl;
- }
- cout<<endl;
- cout<<"Output files will be prefixed with "<<getPrefix()<<endl;
- cout<<endl;
ostringstream rayRuntime;
rayRuntime<<getPrefix()<<"RayVersion.txt";
ofstream f2(rayRuntime.str().c_str());
- f2<<"Ray version: "<<RAY_VERSION<<endl;
+ f2<<"Ray "<<CONFIG_RAY_VERSION<<endl;
f2.close();
writeConfigurationFile();
+
+ writeSmartCommand();
+}
+
+void Parameters::writeSmartCommand() {
+ ostringstream commandFile;
+ commandFile<<getPrefix()<<"RaySmartCommand.txt";
+ ofstream f(commandFile.str().c_str());
+
+ int numberOfRays=getSize();
+
+ #ifdef ASSERT
+ assert(numberOfRays%m_numberOfMiniRanksPerRank==0);
+ #endif
+
+ numberOfRays/=m_numberOfMiniRanksPerRank;
+
+ f<<"mpiexec -n "<<numberOfRays<<" Ray \\"<<endl;
+
+ for(int i=0;i<(int)m_commands.size();i++){
+ if(i!=(int)m_commands.size()-1){
+ f<<" "<<m_commands[i]<<" \\"<<endl;
+ }else{
+ f<<" "<<m_commands[i]<<endl;
+ }
+ }
+ f.close();
+
}
void Parameters::writeConfigurationFile(){
@@ -966,7 +1055,11 @@ void Parameters::writeConfigurationFile(){
f.close();
}
-void Parameters::constructor(int argc,char**argv,int rank,int size){
+void Parameters::constructor(int argc,char**argv,int rank,int size,
+ int miniRanksPerRank){
+
+ m_numberOfMiniRanksPerRank=miniRanksPerRank;
+
m_maximumDistance=0;
m_totalNumberOfSequences=0;
@@ -986,9 +1079,70 @@ void Parameters::constructor(int argc,char**argv,int rank,int size){
}else{
loadCommandsFromArguments(argc,argv);
}
+
+ setDefaultRoutingOptions();
+
parseCommands();
}
+void Parameters::setDefaultRoutingOptions() {
+
+ //cout << "[DEBUG] setDefaultRoutingOptions m_size " << m_size << endl;
+
+/**
+ * if the empty string is passed to the RayPlatform API,
+ * the debruijn model is used.
+ * however, the polytope model is better.
+ */
+ m_connectionType = "debruijn";
+
+/** the default is to not use a default value */
+ m_degree=2;
+
+ // preconfigure with a degree of 2
+ // or at least try to configure...
+
+ /*
+ * Vertices:= Radix^Dimension
+ *
+ * Degree:= (Radix-1)*Dimension
+ */
+
+ // we want the biggest radix possible
+
+ int actualVertices = getSize();
+
+ int radix = actualVertices - 1;
+
+ while(1) {
+
+ bool foundSolution = false;
+
+ for(int dimension = 2 ; dimension < 32 ; ++dimension) {
+
+ int computedVertices = (int)pow(radix, dimension);
+
+ if(computedVertices == actualVertices) {
+ int degree = (radix - 1) * dimension;
+
+ m_connectionType = "polytope";
+ m_degree = degree;
+
+ foundSolution = true;
+ break;
+ }
+ }
+
+ radix--;
+
+ if(radix == 1)
+ break;
+
+ if(foundSolution)
+ break;
+ }
+}
+
int Parameters::getRank(){
return m_rank;
}
@@ -1089,8 +1243,8 @@ string Parameters::getAmosFile(){
return getPrefix()+"AMOS.afg";
}
-vector<string> Parameters::getCommands(){
- return m_commands;
+vector<string>*Parameters::getCommands(){
+ return &m_commands;
}
bool Parameters::getError(){
@@ -1100,11 +1254,23 @@ bool Parameters::getError(){
void Parameters::addDistance(int library,int distance,int count){
m_observedDistances[library][distance]+=count;
}
+string Parameters::getLibraryGlobalFile(){
+ ostringstream s;
+ s<<getPrefix();
+ s<<""<<"LibraryData"<<".xml";
+ return s.str();
+}
string Parameters::getLibraryFile(int library){
ostringstream s;
s<<getPrefix();
- s<<""<<"Library"<<library<<".txt";
+
+ //s<<""<<"Library"<<library<<".txt";
+
+ // everything is now in one single file:
+
+ s << "LibraryData.xml";
+
return s.str();
}
@@ -1112,6 +1278,13 @@ string Parameters::getLibraryFile(int library){
void Parameters::computeAverageDistances(){
cout<<endl;
+
+ #ifdef WRITE_LIBRARY_OBSERVATIONS
+ string globalFileName=getLibraryGlobalFile();
+ ofstream f(globalFileName.c_str());
+ f<<"<libraryData>"<<endl;
+ #endif
+
for(map<int,map<int,int> >::iterator i=m_observedDistances.begin();
i!=m_observedDistances.end();i++){
int library=i->first;
@@ -1121,10 +1294,15 @@ void Parameters::computeAverageDistances(){
vector<int> x;
vector<int> y;
- string fileName=getLibraryFile(library);
+
#ifdef WRITE_LIBRARY_OBSERVATIONS
- ofstream f(fileName.c_str());
+ f<<"<library><handle>"<<library<<"</handle>"<<endl;
+ f<<"<data>"<<endl;
+
+ f<<"# OuterDistanceInNucleotides Frequency"<<endl;
+ f<<"# OuterDistanceInNucleotides includes the gap length and lengths of left and right reads"<<endl;
#endif
+
for(map<int,int>::iterator j=m_observedDistances[library].begin();
j!=m_observedDistances[library].end();j++){
int d=j->first;
@@ -1135,19 +1313,25 @@ void Parameters::computeAverageDistances(){
x.push_back(d);
y.push_back(count);
}
- #ifdef WRITE_LIBRARY_OBSERVATIONS
- f.close();
- #endif
vector<int> averages;
vector<int> deviations;
LibraryPeakFinder finder;
finder.findPeaks(&x,&y,&averages,&deviations);
-
+
for(int i=0;i<(int)averages.size();i++)
addLibraryData(library,averages[i],deviations[i]);
- }
+ #ifdef WRITE_LIBRARY_OBSERVATIONS
+ f<<"</data></library>"<<endl;
+ #endif
+ }
+
+ #ifdef WRITE_LIBRARY_OBSERVATIONS
+ f<<"</libraryData>"<<endl;
+ f.close();
+ #endif
+
cout<<endl;
cout<<endl;
@@ -1178,7 +1362,10 @@ void Parameters::computeAverageDistances(){
f2<<" File: "<<m_singleEndReadsFile[files[1]]<<endl;
f2<<" NumberOfSequences: "<<m_numberOfSequencesInFile[files[1]]<<endl;
}
+
f2<<" Distribution: "<<getLibraryFile(library)<<endl;
+
+
for(int j=0;j<getLibraryPeaks(library);j++){
int average=getLibraryAverageLength(library,j);
int standardDeviation=getLibraryStandardDeviation(library,j);
@@ -1201,7 +1388,7 @@ void Parameters::addLibraryData(int library,int average,int deviation){
m_libraryAverageLength[library].push_back(average);
m_libraryDeviation[library].push_back(deviation);
-
+
int distance=average+4*deviation;
if(distance>m_maximumDistance){
m_maximumDistance=distance;
@@ -1313,10 +1500,17 @@ void Parameters::showUsage(){
cout<<"NAME"<<endl<<basicSpaces<<"Ray - assemble genomes in parallel using the message-passing interface"<<endl<<endl;
cout<<"SYNOPSIS"<<endl;
- cout<<basicSpaces<<"mpiexec -n NUMBER_OF_RANKS Ray -k KMERLENGTH -p l1_1.fastq l1_2.fastq -p l2_1.fastq l2_2.fastq -o test"<<endl;
+ cout<<basicSpaces<<"mpiexec -n 80 Ray -k 31 -p l1_1.fastq l1_2.fastq -p l2_1.fastq l2_2.fastq -o test"<<endl;
+ cout<<endl;
+ cout<<basicSpaces<<"mpiexec -n 80 Ray Ray.conf # with commands in a file"<<endl;
+ cout<<endl;
+ cout<<basicSpaces<<"mpiexec -n 80 Ray -k 31 -detect-sequence-files SampleDirectory # auto-detection"<<endl;
cout<<endl;
- cout<<basicSpaces<<"mpiexec -n NUMBER_OF_RANKS Ray Ray.conf # with commands in a file"<<endl;
+
+#ifdef CONFIG_MINI_RANKS
+ cout<<basicSpaces<<"mpiexec -n 10 Ray -mini-ranks-per-rank 7 Ray.conf # with mini-ranks"<<endl;
cout<<endl;
+#endif /* CONFIG_MINI_RANKS */
cout<<"DESCRIPTION:"<<endl;
@@ -1341,6 +1535,26 @@ void Parameters::showUsage(){
showOption("-version","Displays Ray version and compilation options.");
cout<<endl;
+ cout<<" Run Ray in pure MPI mode"<<endl;
+ cout<<endl;
+ cout<<" mpiexec -n 80 Ray ..."<<endl;
+ cout<<endl;
+
+#ifdef CONFIG_MINI_RANKS
+ cout<<" Run Ray with mini-ranks on 10 machines, 8 cores / machine (MPI and IEEE POSIX threads)"<<endl;
+ cout<<endl;
+ cout<<" mpiexec -n 10 Ray -mini-ranks-per-rank 7 ..."<<endl;
+ cout<<endl;
+
+#endif /* CONFIG_MINI_RANKS */
+
+ cout<<" Run Ray on one core only (still needs MPI)"<<endl;
+ cout<<endl;
+ cout<<" Ray ..."<<endl;
+ cout<<endl;
+
+ cout<<endl;
+
cout<<" Using a configuration file"<<endl;
cout<<endl;
cout<<" Ray can be launched with"<<endl;
@@ -1352,15 +1566,22 @@ void Parameters::showUsage(){
cout<<endl;
showOption("-k kmerLength","Selects the length of k-mers. The default value is 21. ");
showOptionDescription("It must be odd because reverse-complement vertices are stored together.");
- showOptionDescription("The maximum length is defined at compilation by MAXKMERLENGTH");
+ showOptionDescription("The maximum length is defined at compilation by CONFIG_MAXKMERLENGTH");
showOptionDescription("Larger k-mers utilise more memory.");
cout<<endl;
cout<<" Inputs"<<endl;
+ cout << endl;
+
+ showOption("-detect-sequence-files SampleDirectory", "Detects files in a directory automatically.");
+ showOptionDescription("This option can generate these commands automatically for you: LoadPairedEndReads (-p) and LoadSingleEndReads (-s)");
+
cout<<endl;
showOption("-p leftSequenceFile rightSequenceFile [averageOuterDistance standardDeviation]","Provides two files containing paired-end reads.");
showOptionDescription("averageOuterDistance and standardDeviation are automatically computed if not provided.");
+ showOptionDescription("LoadPairedEndReads is equivalent to -p");
+
cout<<endl;
showOption("-i interleavedSequenceFile [averageOuterDistance standardDeviation]","Provides one file containing interleaved paired-end reads.");
@@ -1368,17 +1589,18 @@ void Parameters::showUsage(){
cout<<endl;
showOption("-s sequenceFile","Provides a file containing single-end reads.");
+ showOptionDescription("LoadSingleEndReads is equivalent to -s");
cout<<endl;
cout<<" Outputs"<<endl;
cout<<endl;
showOption("-o outputDirectory","Specifies the directory for outputted files. Default is RayOutput");
+ showOptionDescription("Other name: -output");
cout<<endl;
-
cout<<" Assembly options (defaults work well)"<<endl;
cout<<endl;
-
+
showOption("-disable-recycling","Disables read recycling during the assembly");
showOptionDescription("reads will be set free in 3 cases:");
showOptionDescription("1. the distance did not match for a pair");
@@ -1389,6 +1611,15 @@ void Parameters::showUsage(){
showOptionDescription("First Annual RECOMB Satellite Workshop on Massively Parallel Sequencing, March 26-27 2011, Vancouver, BC, Canada.");
cout<<endl;
+ showOption("-debug-recycling", "Debugs the recycling events");
+ cout<<endl;
+
+ showOption("-ignore-seeds", "Disables assembly by ignoring seeds.");
+ cout<<endl;
+
+ showOption("-merge-seeds", "Merges seeds initially to reduce running time.");
+ cout << endl;
+
showOption("-disable-scaffolder","Disables the scaffolder.");
cout<<endl;
showOption("-minimum-contig-length minimumContigLength",
@@ -1417,10 +1648,10 @@ void Parameters::showUsage(){
ostringstream text;
showOption("-bloom-filter-bits bits","Sets the number of bits for the Bloom filter");
- text<<"Default is "<<__BLOOM_DEFAULT_BITS<<" bits, 0 bits disables the Bloom filter.";
+ text<<"Default is "<<"auto"<<" bits (adaptive), 0 bits disables the Bloom filter.";
showOptionDescription(text.str());
cout<<endl;
-
+
text.str("");
text<<"Default value: "<<__DEFAULT_BUCKETS;
showOption("-hash-table-buckets buckets","Sets the initial number of buckets. Must be a power of 2 !");
@@ -1440,7 +1671,7 @@ void Parameters::showUsage(){
text<<"Default value: "<< __DEFAULT_LOAD_FACTOR_THRESHOLD<<", must be >= 0.5 and < 1";
showOptionDescription(text.str());
cout<<endl;
-
+
showOption("-hash-table-verbosity","Activates verbosity for the distributed storage engine");
cout<<endl;
@@ -1467,7 +1698,7 @@ void Parameters::showUsage(){
showOptionDescription("OntologyTerms.txt is fetched from http://geneontology.org");
showOptionDescription("Annotations.txt is a 2-column file (EMBL_CDS handle & gene ontology identifier)");
showOptionDescription("See Documentation/GeneOntology.txt");
-
+
cout<<" Other outputs"<<endl;
cout<<endl;
@@ -1547,12 +1778,13 @@ void Parameters::showUsage(){
cout<<endl;
showOption("-connection-type type","Sets the connection type for routes.");
showOptionDescription("Accepted values are debruijn, hypercube, polytope, group, random, kautz and complete. Default is debruijn.");
- showOptionDescription(" debruijn: a full de Bruijn graph a given alphabet and diameter");
- showOptionDescription(" hypercube: a hypercube, alphabet is {0,1} and the vertices is a power of 2");
+ showOptionDescription(" torus: a k-ary n-cube, radix: k, dimension: n, degree: 2*dimension, vertices: radix^dimension");
showOptionDescription(" polytope: a convex regular polytope, alphabet is {0,1,...,B-1} and the vertices is a power of B");
+ showOptionDescription(" hypercube: a hypercube, alphabet is {0,1} and the vertices is a power of 2");
+ showOptionDescription(" debruijn: a full de Bruijn graph a given alphabet and diameter");
+ showOptionDescription(" kautz: a full de Kautz graph, which is a subgraph of a de Bruijn graph");
showOptionDescription(" group: silly model where one representative per group can communicate with outsiders");
showOptionDescription(" random: Erdős–Rényi model");
- showOptionDescription(" kautz: a full de Kautz graph, which is a subgraph of a de Bruijn graph");
showOptionDescription(" complete: a full graph with all the possible connections");
showOptionDescription("With the type debruijn, the number of ranks must be a power of something.");
@@ -1585,12 +1817,21 @@ void Parameters::showUsage(){
showOptionDescription("add '-D CONFIG_SSE_4_2' in the Makefile to use hardware instruction (SSE 4.2)");
cout<<endl;
+ showOption("-write-scheduling-data", "Writes RayPlatform scheduling information to RayOutput/Scheduling/");
+ cout<<endl;
+
+ showOption("-write-plugin-data", "Writes data for plugins registered with the RayPlatform API to RayOutput/Plugins");
+ cout<<endl;
showOption("-run-profiler","Runs the profiler as the code runs. By default, only show granularity warnings.");
showOptionDescription("Running the profiler increases running times.");
cout<<endl;
showOption("-with-profiler-details","Shows number of messages sent and received in each methods during in each time slices (epochs). Needs -run-profiler.");
cout<<endl;
+
+ showOption("-debug", "Turns on -run-profiler and -with-profiler-details for debugging");
+ cout << endl;
+
showOption("-show-communication-events","Shows all messages sent and received.");
cout<<endl;
showOption("-show-read-placement","Shows read placement in the graph during the extension.");
@@ -1615,13 +1856,22 @@ void Parameters::showUsage(){
cout<<" Note: file format is determined with file extension."<<endl;
cout<<endl;
cout<<" .fasta"<<endl;
+ cout<<" .fa"<<endl;
cout<<" .fasta.gz (needs HAVE_LIBZ=y at compilation)"<<endl;
+ cout<<" .fa.gz (needs HAVE_LIBZ=y at compilation)"<<endl;
cout<<" .fasta.bz2 (needs HAVE_LIBBZ2=y at compilation)"<<endl;
+ cout<<" .fa.bz2 (needs HAVE_LIBBZ2=y at compilation)"<<endl;
cout<<" .fastq"<<endl;
+ cout<<" .fq"<<endl;
cout<<" .fastq.gz (needs HAVE_LIBZ=y at compilation)"<<endl;
+ cout<<" .fq.gz (needs HAVE_LIBZ=y at compilation)"<<endl;
cout<<" .fastq.bz2 (needs HAVE_LIBBZ2=y at compilation)"<<endl;
+ cout<<" .fq.bz2 (needs HAVE_LIBBZ2=y at compilation)"<<endl;
+ cout<<" export.txt"<<endl;
+ cout<<" qseq.txt"<<endl;
cout<<" .sff (paired reads must be extracted manually)"<<endl;
cout<<" .csfasta (color-space reads)"<<endl;
+ cout<<" .csfa (color-space reads)"<<endl;
cout<<endl;
@@ -1667,7 +1917,7 @@ void Parameters::showUsage(){
cout<<" The resulting file is not utilised by Ray."<<endl;
cout<<" The resulting file is very large."<<endl;
cout<<endl;
-
+
cout<<" Assembly steps"<<endl;
cout<<endl;
cout<<" RayOutput/SeedLengthDistribution.txt"<<endl;
@@ -1686,7 +1936,7 @@ void Parameters::showUsage(){
cout<<endl;
cout<<" RayOutput/LibraryStatistics.txt"<<endl;
cout<<" Estimation of outer distances for paired reads"<<endl;
- cout<<" RayOutput/Library<LibraryNumber>.txt"<<endl;
+ cout<<" RayOutput/LibraryData.xml"<<endl;
cout<<" Frequencies for observed outer distances (insert size + read lengths)"<<endl;
cout<<endl;
@@ -1701,9 +1951,11 @@ void Parameters::showUsage(){
cout<<" Ray software"<<endl;
cout<<endl;
cout<<" RayOutput/RayVersion.txt"<<endl;
- cout<<" The version of Ray"<<endl;
+ cout<<" The version of Ray"<<endl;
cout<<" RayOutput/RayCommand.txt"<<endl;
- cout<<" The exact same command provided "<<endl;
+ cout<<" The exact same command provided"<<endl;
+ cout<<" RayOutput/RaySmartCommand.txt"<<endl;
+ cout<<" The smart command generated by Ray"<<endl;
cout<<endl;
cout<<" AMOS"<<endl;
@@ -1715,8 +1967,6 @@ void Parameters::showUsage(){
cout<<" Communication"<<endl;
cout<<endl;
- cout<<" RayOutput/MessagePassingInterface.txt"<<endl;
- cout<<" Number of messages sent"<<endl;
cout<<" RayOutput/NetworkTest.txt"<<endl;
cout<<" Latencies in microseconds"<<endl;
cout<<" RayOutput/Rank<rank>NetworkTestData.txt"<<endl;
@@ -1755,7 +2005,7 @@ void Parameters::showUsage(){
cout<<basicSpaces<<"along with this program (see LICENSE)."<<endl;
cout<<endl;
- cout<<"Ray "<<RAY_VERSION<<endl;
+ cout<<"Ray "<<CONFIG_RAY_VERSION<<endl;
}
/* get the prefix for mmap'ed files
@@ -1768,8 +2018,10 @@ int Parameters::getReducerValue(){
return m_reducerPeriod;
}
-Rank Parameters::getRankFromGlobalId(ReadHandle a){
+Rank Parameters::getRankFromGlobalId(ReadHandle & a){
+
uint64_t elementsPerRank=m_totalNumberOfSequences/m_size;
+
Rank rank=a/elementsPerRank;
if(rank >= m_size){
@@ -1778,6 +2030,11 @@ Rank Parameters::getRankFromGlobalId(ReadHandle a){
#ifdef ASSERT
+ if(rank<0){
+ cout<<"Error: rank is < 0, rank: "<<rank<<" ReadHandle: "<<a<<" elementsPerRank: "<<elementsPerRank;
+ cout<<"self.rank is "<<m_rank<<endl;
+ }
+
assert(rank>=0);
if(rank>=m_size){
@@ -1790,7 +2047,7 @@ Rank Parameters::getRankFromGlobalId(ReadHandle a){
return rank;
}
-int Parameters::getIdFromGlobalId(ReadHandle a){
+int Parameters::getIdFromGlobalId(ReadHandle & a){
int bin=getRankFromGlobalId(a);
LargeCount x=m_totalNumberOfSequences/m_size;
return a-bin*x;
@@ -1822,7 +2079,7 @@ bool Parameters::hasPairedReads(){
return m_numberOfLibraries!=0;
}
-int Parameters::_vertexRank(Kmer*a){
+Rank Parameters::vertexRank(Kmer*a){
return a->vertexRank(m_size,m_wordSize,m_colorSpaceMode);
}
diff --git a/code/application_core/Parameters.h b/code/Mock/Parameters.h
similarity index 91%
rename from code/application_core/Parameters.h
rename to code/Mock/Parameters.h
index a917486..ebbc366 100644
--- a/code/application_core/Parameters.h
+++ b/code/Mock/Parameters.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,11 +22,14 @@
#ifndef _Parameters
#define _Parameters
-#include<map>
-#include<application_core/common_functions.h>
-#include<set>
-#include<string>
-#include<vector>
+#include "common_functions.h"
+
+#include <code/SequencesLoader/ReadHandle.h>
+
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
using namespace std;
/**
@@ -39,6 +42,8 @@ using namespace std;
* The threshold for triggering incremental resizing
* of the distributed hash table, valid for a single
* agent
+ *
+ * \see http://docs.oracle.com/javase/7/docs/api/java/util/HashMap.html
*/
#define __DEFAULT_LOAD_FACTOR_THRESHOLD 0.75 /* Like Java HashMap */
@@ -50,12 +55,6 @@ using namespace std;
*/
#define __DEFAULT_BUCKETS_PER_GROUP 64
-// 8 bytes
-/*
- * The default number of bits in the Bloom filter.
- */
-#define __BLOOM_DEFAULT_BITS 268435456 //64000000
-
/**
* This class is the implementation of an interpreter for the RayInputFile.
* It allows the following commands:
@@ -66,6 +65,7 @@ using namespace std;
*/
class Parameters{
+ int m_numberOfMiniRanksPerRank;
string m_configurationContent;
@@ -161,10 +161,15 @@ class Parameters{
void __shuffleOperationCodes();
bool isValidInteger(const char*textMessage);
+ string getLibraryGlobalFile();
+
+ void setDefaultRoutingOptions();
+
+ void writeSmartCommand();
public:
Parameters();
string getReceivedMessagesFile();
- void constructor(int argc,char**argv,Rank rank,int size);
+ void constructor(int argc,char**argv,Rank rank,int size,int miniRanksPerRank);
bool isInitiated();
vector<string> getAllFiles();
string getDirectory();
@@ -181,7 +186,7 @@ public:
string getAmosFile();
string getParametersFile();
string getCoverageDistributionFile();
- vector<string> getCommands();
+ vector<string>*getCommands();
bool getError();
void addDistance(int library,int distance,int count);
void computeAverageDistances();
@@ -205,8 +210,8 @@ public:
void showUsage();
int getReducerValue();
- Rank getRankFromGlobalId(uint64_t a);
- int getIdFromGlobalId(uint64_t a);
+ Rank getRankFromGlobalId(ReadHandle & a);
+ int getIdFromGlobalId(ReadHandle & a);
int getMaximumDistance();
uint64_t getGlobalIdFromRankAndLocalId(Rank rank,int id);
CoverageDepth getMaximumAllowedCoverage();
@@ -214,7 +219,7 @@ public:
void setMinimumCoverage(CoverageDepth a);
Kmer _complementVertex(Kmer*a);
bool hasPairedReads();
- int _vertexRank(Kmer*a);
+ Rank vertexRank(Kmer*a);
string getMemoryPrefix();
/**
* run the profiler
@@ -227,7 +232,6 @@ public:
/**
* debug the code that computes seeds
*/
- bool debugSeeds();
bool showMemoryUsage();
bool showEndingContext();
diff --git a/code/application_core/common_functions.cpp b/code/Mock/common_functions.cpp
similarity index 71%
rename from code/application_core/common_functions.cpp
rename to code/Mock/common_functions.cpp
index 551b396..805c9df 100644
--- a/code/application_core/common_functions.cpp
+++ b/code/Mock/common_functions.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,19 +19,21 @@
*/
-#include <cryptography/crypto.h>
+#include "common_functions.h"
+#include "constants.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/cryptography/crypto.h>
+
#include <assert.h>
#include <stdio.h>
-#include <application_core/constants.h>
#include <time.h>
#include <vector>
#include <fstream>
-#include <application_core/common_functions.h>
#include <stdlib.h>
#include <iostream>
#include <string>
#include <cstring>
-#include <core/OperatingSystem.h>
#include <sstream>
using namespace std;
@@ -40,7 +42,7 @@ bool isValidDNA(char*x){
int len=strlen(x);
for(int i=0;i<len;i++){
char a=x[i];
- if(!(a=='A'||a=='T'||a=='C'||a=='G'))
+ if(!(a==SYMBOL_A||a==SYMBOL_T||a==SYMBOL_C||a==SYMBOL_G))
return false;
}
return true;
@@ -56,7 +58,7 @@ string addLineBreaks(string dna,int columns){
return output.str();
}
-string convertToString(vector<Kmer>*b,int m_wordSize,bool color){
+string convertToString(GraphPath*b,int m_wordSize,bool color){
ostringstream a;
#ifdef USE_DISTANT_SEGMENTS_GRAPH
//
@@ -68,18 +70,24 @@ string convertToString(vector<Kmer>*b,int m_wordSize,bool color){
a<<codeToChar(b->at(p).getFirstSegmentFirstCode(m_wordSize));
}
#else
- a<<b->at(0).idToWord(m_wordSize,color);
+ Kmer object;
+ b->at(0,&object);
+ a<<object.idToWord(m_wordSize,color);
#endif
+
for(int j=1;j<(int)(*b).size();j++){
- a<<b->at(j).getLastSymbol(m_wordSize,color);
+ Kmer object;
+ b->at(j,&object);
+ a<<object.getLastSymbol(m_wordSize,color);
}
string contig=a.str();
+
return contig;
}
Kmer kmerAtPosition(const char*m_sequence,int pos,int w,char strand,bool color){
#ifdef ASSERT
- assert(w<=MAXKMERLENGTH);
+ assert(w<=CONFIG_MAXKMERLENGTH);
#endif
int length=strlen(m_sequence);
if(pos>length-w){
@@ -91,13 +99,13 @@ Kmer kmerAtPosition(const char*m_sequence,int pos,int w,char strand,bool color){
exit(0);
}
if(strand=='F'){
- char sequence[MAXKMERLENGTH];
+ char sequence[CONFIG_MAXKMERLENGTH];
memcpy(sequence,m_sequence+pos,w);
sequence[w]='\0';
Kmer v=wordId(sequence);
return v;
}else if(strand=='R'){
- char sequence[MAXKMERLENGTH];
+ char sequence[CONFIG_MAXKMERLENGTH];
memcpy(sequence,m_sequence+length-pos-w,w);
sequence[w]='\0';
Kmer v=wordId(sequence);
@@ -107,18 +115,18 @@ Kmer kmerAtPosition(const char*m_sequence,int pos,int w,char strand,bool color){
return error;
}
-uint64_t getPathUniqueId(Rank rank,int id){
+PathHandle getPathUniqueId(Rank rank,int id){
uint64_t a=id;
a=a*MAX_NUMBER_OF_MPI_PROCESSES+rank;
return a;
}
-int getIdFromPathUniqueId(uint64_t a){
+int getIdFromPathUniqueId(PathHandle a){
return a/MAX_NUMBER_OF_MPI_PROCESSES;
}
-Rank getRankFromPathUniqueId(uint64_t a){
- int rank=a%MAX_NUMBER_OF_MPI_PROCESSES;
+Rank getRankFromPathUniqueId(PathHandle a){
+ Rank rank=a%MAX_NUMBER_OF_MPI_PROCESSES;
return rank;
}
@@ -142,13 +150,13 @@ void print8(uint8_t a){
uint8_t charToCode(char a){
switch (a){
- case 'A':
+ case SYMBOL_A:
return RAY_NUCLEOTIDE_A;
- case 'T':
+ case SYMBOL_T:
return RAY_NUCLEOTIDE_T;
- case 'C':
+ case SYMBOL_C:
return RAY_NUCLEOTIDE_C;
- case 'G':
+ case SYMBOL_G:
return RAY_NUCLEOTIDE_G;
default:
return RAY_NUCLEOTIDE_A;
@@ -157,14 +165,14 @@ uint8_t charToCode(char a){
char complementNucleotide(char c){
switch(c){
- case 'A':
- return 'T';
- case 'T':
- return 'A';
- case 'G':
- return 'C';
- case 'C':
- return 'G';
+ case SYMBOL_A:
+ return SYMBOL_T;
+ case SYMBOL_T:
+ return SYMBOL_A;
+ case SYMBOL_G:
+ return SYMBOL_C;
+ case SYMBOL_C:
+ return SYMBOL_G;
default:
return c;
}
@@ -235,6 +243,46 @@ void unpack_pointer(void**pointer,uint64_t integerValue){
}
+#ifdef CONFIG_MPI_IO
+
+bool flushFileOperationBuffer_MPI_IO(bool force,ostringstream*buffer,MPI_File file,int bufferSize){
+
+ int available=buffer->tellp();
+
+ if(available==0)
+ return false;
+
+ if(force || available>=bufferSize){
+
+/*
+ * The const_cast is not required by MPI 3.0. MPI <= 2.2 has stupid semantic
+ * (i.e. MPI_File_write takes a non-const buffer (???))
+ */
+
+ string copy=buffer->str();
+ const char*constantCopy=copy.c_str();
+
+ char*data=const_cast<char*> ( constantCopy );
+
+ int bytes=copy.length();
+
+ MPI_Status writeStatus;
+ int returnValue=MPI_File_write(file,data,bytes,MPI_BYTE,&writeStatus);
+
+ if(returnValue!=MPI_SUCCESS){
+ cout<<"Error: could not write to file with MPI I/O."<<endl;
+ }
+
+ buffer->str("");
+
+ return true;
+ }
+
+ return false;
+}
+
+#endif
+
bool flushFileOperationBuffer(bool force,ostringstream*buffer,ostream*file,int bufferSize){
int available=buffer->tellp();
@@ -244,7 +292,9 @@ bool flushFileOperationBuffer(bool force,ostringstream*buffer,ostream*file,int b
if(force || available>=bufferSize){
- (*file)<<buffer->str();
+ string copy=buffer->str();
+ file->write(copy.c_str(),copy.length());
+
buffer->str("");
return true;
diff --git a/code/application_core/common_functions.h b/code/Mock/common_functions.h
similarity index 73%
rename from code/application_core/common_functions.h
rename to code/Mock/common_functions.h
index 10a1d11..bd18f52 100644
--- a/code/application_core/common_functions.h
+++ b/code/Mock/common_functions.h
@@ -1,6 +1,6 @@
/*
Ray
-Copyright (C) 2010, 2011 Sébastien Boisvert
+Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,19 +22,30 @@ see <http://www.gnu.org/licenses/>
#ifndef _common_functions
#define _common_functions
-#include <memory/allocator.h>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
+#include "constants.h"
+
+#ifdef CONFIG_MPI_IO
+#include <mpi.h>
+#endif
+
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/SeedingData/GraphPath.h>
+#include <code/SeedingData/PathHandle.h>
+
+#include <RayPlatform/memory/allocator.h>
+#include <RayPlatform/core/slave_modes.h>
+#include <RayPlatform/core/master_modes.h>
+#include <RayPlatform/communication/mpi_tags.h>
+
+#include <string.h>
#include <string>
-#include <application_core/constants.h>
-#include <core/slave_modes.h>
#include <iostream>
-#include <core/master_modes.h>
#include <vector>
-#include <communication/mpi_tags.h>
-#include <string.h>
+
#ifdef ASSERT
#include <assert.h>
#endif
+
using namespace std;
/*
@@ -57,7 +68,10 @@ Kmer wordId(const char*a);
*/
string addLineBreaks(string sequence,int a);
-string convertToString(vector<Kmer>*b,int m_wordSize,bool color);
+/*
+ * TODO: move this in GraphPath
+ */
+string convertToString(GraphPath*b,int m_wordSize,bool color);
Kmer kmerAtPosition(const char*string,int pos,int w,char strand,bool color);
@@ -78,10 +92,13 @@ string reverseComplement(string*a);
MessageUnit pack_pointer(void**pointer);
void unpack_pointer(void**pointer,MessageUnit integerValue);
-
bool flushFileOperationBuffer(bool force,ostringstream*buffer,ostream*file,int bufferSize);
bool flushFileOperationBuffer_FILE(bool force,ostringstream*buffer,FILE*file,int bufferSize);
+#ifdef CONFIG_MPI_IO
+bool flushFileOperationBuffer_MPI_IO(bool force,ostringstream*buffer,MPI_File file,int bufferSize);
+#endif
+
#endif
diff --git a/code/application_core/constants.h b/code/Mock/constants.h
similarity index 62%
rename from code/application_core/constants.h
rename to code/Mock/constants.h
index 76326ec..64bfbc7 100644
--- a/code/application_core/constants.h
+++ b/code/Mock/constants.h
@@ -1,37 +1,36 @@
/*
-Ray
-Copyright (C) 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2011, 2012, 2013 Sébastien Boisvert
-http://DeNovoAssembler.SourceForge.Net/
+ http://DeNovoAssembler.SourceForge.Net/
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, version 3 of the License.
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You have received a copy of the GNU General Public License
-along with this program (gpl-3.0.txt).
-see <http://www.gnu.org/licenses/>
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
*/
#ifndef _constants
#define _constants
-#ifndef RAY_VERSION
-#define RAY_VERSION "Unknown"
+#ifndef CONFIG_RAY_VERSION
+#define CONFIG_RAY_VERSION "Unknown"
#endif
/*
* Define the maximum k-mer length when
* the compiler/make does not.
*/
-#ifndef MAXKMERLENGTH
-#define MAXKMERLENGTH 32
+#ifndef CONFIG_MAXKMERLENGTH
+#define CONFIG_MAXKMERLENGTH 32
#endif
// some multipliers
@@ -57,7 +56,7 @@ see <http://www.gnu.org/licenses/>
#define __func__ __FUNCTION__
#endif
-#ifdef FORCE_PACKING
+#ifdef CONFIG_FORCE_PACKING
/*
* With gcc, one can pack data structures.
*/
@@ -82,6 +81,16 @@ see <http://www.gnu.org/licenses/>
#define RAY_NUCLEOTIDE_G 2 /* ~10 == 01 */
#define RAY_NUCLEOTIDE_T 3 /* ~11 == 00 */
+#define SYMBOL_A 'A'
+#define SYMBOL_T 'T'
+#define SYMBOL_G 'G'
+#define SYMBOL_C 'C'
+
+#define SYMBOL_LOWER_A 'a'
+#define SYMBOL_LOWER_T 't'
+#define SYMBOL_LOWER_G 'g'
+#define SYMBOL_LOWER_C 'c'
+
#define DOUBLE_ENCODING_A_COLOR '0'
#define DOUBLE_ENCODING_C_COLOR '1'
#define DOUBLE_ENCODING_G_COLOR '2'
@@ -159,17 +168,16 @@ see <http://www.gnu.org/licenses/>
/* since Lustre is not very good at caching file input/output operations
* Ray agglomerates these operations */
-//#define CONFIG_FILE_IO_BUFFER_SIZE 1048576 /* 1 MB */
-//#define CONFIG_FILE_IO_BUFFER_SIZE 4194304 /* 4 MB */
-#define CONFIG_FILE_IO_BUFFER_SIZE 16777216 /* 16 MB */
-//#define CONFIG_FILE_IO_BUFFER_SIZE 33554432 /* 32 MB */
-//#define CONFIG_FILE_IO_BUFFER_SIZE 134217728 /* 128 MB */
+#define SIZE_1M 1048576
+#define SIZE_2M 2097152
+#define SIZE_4M 4194304
+#define SIZE_8M 8388608
+#define SIZE_16M 16777216
+#define SIZE_32M 33554432
+#define SIZE_64M 67108864
+#define SIZE_128M 134217728
-/* the identifier of a read */
-typedef uint64_t ReadHandle;
-
-/* the identifier for a path in the de Bruijn graph */
-typedef uint64_t PathHandle;
+#define CONFIG_FILE_IO_BUFFER_SIZE SIZE_16M
/* a DNA strand */
typedef char Strand;
@@ -180,5 +188,21 @@ typedef uint64_t LargeCount;
/* a datatype for an index */
typedef uint64_t LargeIndex;
+typedef uint16_t LibraryHandle;
+
+
+typedef int SampleIdentifier;
+
+
+/*
+ * path storage engine
+ * CONFIG_PATH_STORAGE_DEFAULT uses arrays of vertices, which consumes memory a lot.
+ * CONFIG_PATH_STORAGE_BLOCK uses a block approach for vertices.
+ */
+#define CONFIG_PATH_STORAGE_BLOCK
+//#define CONFIG_PATH_STORAGE_DEFAULT
+
+#define BITS_PER_NUCLEOTIDE 2
+#define BITS_PER_BYTE 8
#endif
diff --git a/code/NetworkTest/Makefile b/code/NetworkTest/Makefile
new file mode 100644
index 0000000..de18c09
--- /dev/null
+++ b/code/NetworkTest/Makefile
@@ -0,0 +1,3 @@
+NetworkTest-y += code/NetworkTest/NetworkTest.o
+
+obj-y += $(NetworkTest-y)
diff --git a/code/plugin_NetworkTest/NetworkTest.cpp b/code/NetworkTest/NetworkTest.cpp
similarity index 86%
rename from code/plugin_NetworkTest/NetworkTest.cpp
rename to code/NetworkTest/NetworkTest.cpp
index 99fe617..4d73103 100644
--- a/code/plugin_NetworkTest/NetworkTest.cpp
+++ b/code/NetworkTest/NetworkTest.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -18,30 +18,29 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_NetworkTest/NetworkTest.h>
-#include <communication/Message.h>
-#include <communication/mpi_tags.h>
-#include <core/slave_modes.h>
-#include <core/master_modes.h>
-#include <sstream>
-#include <core/OperatingSystem.h>
+#include "NetworkTest.h"
+
+#include <code/Mock/constants.h>
+
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/communication/mpi_tags.h>
+#include <RayPlatform/core/slave_modes.h>
+#include <RayPlatform/core/master_modes.h>
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/core/statistics.h>
+#include <RayPlatform/core/ComputeCore.h>
+
+#include <iostream>
#include <fstream>
+#include <sstream>
#include <stdlib.h>
-#include <core/statistics.h>
-#include <application_core/constants.h>
-#include <iostream>
-#include <core/ComputeCore.h>
+using namespace std;
__CreatePlugin(NetworkTest);
- /**/
-__CreateMasterModeAdapter(NetworkTest,RAY_MASTER_MODE_TEST_NETWORK); /**/
- /**/
-__CreateSlaveModeAdapter(NetworkTest,RAY_SLAVE_MODE_TEST_NETWORK); /**/
- /**/
- /**/
-
-using namespace std;
+__CreateMasterModeAdapter(NetworkTest,RAY_MASTER_MODE_TEST_NETWORK);
+__CreateSlaveModeAdapter(NetworkTest,RAY_SLAVE_MODE_TEST_NETWORK);
+__CreateMessageTagAdapter(NetworkTest,RAY_MPI_TAG_TEST_NETWORK_MESSAGE);
#define LATENCY_INFORMATION_NOT_AVAILABLE 123123123
#define __MAXIMUM_LATENCY 4096 // microseconds
@@ -49,6 +48,7 @@ using namespace std;
/** initialize the NetworkTest */
void NetworkTest::constructor(int rank,int size,StaticVector*inbox,StaticVector*outbox,Parameters*parameters,RingAllocator*outboxAllocator,
string*name,TimePrinter*timePrinter){
+
m_timePrinter=timePrinter;
m_name=name;
m_inbox=inbox;
@@ -57,7 +57,7 @@ void NetworkTest::constructor(int rank,int size,StaticVector*inbox,StaticVector*
m_initialisedNetworkTest=false;
m_size=size;
m_rank=rank;
-
+
m_ranksFinished=0;
m_askedToWriteFiles=false;
@@ -122,7 +122,7 @@ void NetworkTest::call_RAY_SLAVE_MODE_TEST_NETWORK(){
if(!m_started){
m_started=true;
-
+
bool skip=false;
if(m_parameters->hasOption("-skip-network-test"))
@@ -154,13 +154,25 @@ void NetworkTest::call_RAY_SLAVE_MODE_TEST_NETWORK(){
m_destinations.push_back(destination);
MessageUnit *message=(MessageUnit*)m_outboxAllocator->allocate(m_numberOfWords*sizeof(MessageUnit));
+ message[0]=destination;
+
Message aMessage(message,m_numberOfWords,destination,RAY_MPI_TAG_TEST_NETWORK_MESSAGE,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_sentCurrentTestMessage=true;
- //cout<<m_rank<<" sends RAY_MPI_TAG_TEST_NETWORK_MESSAGE to "<<destination<<endl;
+
+ #ifdef CONFIG_DEBUG_NETWORK_TEST_PLUGIN
+ cout<<"[NetworkTest.plugin] Rank "<<m_rank<<" sends RAY_MPI_TAG_TEST_NETWORK_MESSAGE to "<<destination<<endl;
+ #endif
+
}else if(m_inbox->size()>0 && m_inbox->at(0)->getTag()==RAY_MPI_TAG_TEST_NETWORK_MESSAGE_REPLY){
LargeCount endingMicroSeconds=getMicroseconds();
-
+
+ #ifdef CONFIG_DEBUG_NETWORK_TEST_PLUGIN
+ Message*message=m_inbox->at(0);
+ Rank source=message->getSource();
+ cout<<"[NetworkTest.plugin] Rank "<<m_rank<<" receives RAY_MPI_TAG_TEST_NETWORK_MESSAGE_REPLY from "<<source<<endl;
+ #endif
+
m_receivedMicroseconds.push_back(endingMicroSeconds);
m_sentCurrentTestMessage=false;
@@ -199,7 +211,7 @@ void NetworkTest::call_RAY_SLAVE_MODE_TEST_NETWORK(){
strcpy(destination,m_name->c_str());
Message aMessage(message,MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit),
MASTER_RANK,RAY_MPI_TAG_TEST_NETWORK_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_sentData=true;
@@ -226,7 +238,7 @@ void NetworkTest::writeData(){
ostringstream file;
file<<m_parameters->getPrefix();
file<<"Rank"<<m_parameters->getRank()<<".NetworkTestData.txt";
-
+
cout<<"Rank "<<m_parameters->getRank()<<" is writing "<<file.str()<<" (-write-network-test-raw-data)"<<endl;
ofstream f(file.str().c_str());
@@ -242,7 +254,7 @@ void NetworkTest::writeData(){
f<<"# average latency measured in microseconds: "<<getAverageLatency()<<endl;
f<<"# next line contains column names"<<endl;
f<<"# TestMessage SourceRank DestinationRank QueryTimeInMicroseconds ReplyTimeInMicroseconds Latency MessagesSentToDestination"<<endl;
-
+
#ifdef ASSERT
assert(m_sentMicroseconds.size() == m_destinations.size());
assert(m_sentMicroseconds.size() == m_receivedMicroseconds.size());
@@ -264,7 +276,7 @@ void NetworkTest::writeData(){
ostringstream file2;
file2<<m_parameters->getPrefix();
file2<<"Rank"<<m_parameters->getRank()<<".NetworkTestDataCount.txt";
-
+
ofstream f2(file2.str().c_str());
f2<<"# DestinationRank MessagesSentToDestination"<<endl;
for(map<int,int>::iterator i=counters.begin();i!=counters.end();i++){
@@ -359,7 +371,7 @@ void NetworkTest::call_RAY_MASTER_MODE_TEST_NETWORK (){
cout<<endl;
m_timePrinter->printElapsedTime("Network testing");
cout<<endl;
-
+
if(m_parameters->hasOption("-test-network-only")){
m_switchMan->setMasterMode(RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
return;
@@ -367,11 +379,15 @@ void NetworkTest::call_RAY_MASTER_MODE_TEST_NETWORK (){
/* no files */
if(m_parameters->getNumberOfFiles()==0){
- cout<<"Rank "<<m_parameters->getRank()<<": no input files, aborting."<<endl;
+
+ if(!m_parameters->hasOption("-run-surveyor")) {
+ cout<<"Rank "<<m_parameters->getRank()<<": no input files, aborting."<<endl;
+ }
+
m_switchMan->setMasterMode(RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
}
}else if(m_ranksFinished==m_size && !m_askedToWriteFiles){
-
+
m_switchMan->sendToAll(m_outbox,m_parameters->getRank(),RAY_MPI_TAG_TEST_NETWORK_WRITE_DATA);
m_askedToWriteFiles=true;
@@ -409,7 +425,7 @@ int NetworkTest::getModeLatency(){
if(data.count(maxLatency)==0 || data[latency] > data[maxLatency])
maxLatency=latency;
}
-
+
return maxLatency;
}
@@ -417,6 +433,23 @@ void NetworkTest::setSwitchMan(SwitchMan*a){
m_switchMan=a;
}
+/* we reply with an empty message */
+void NetworkTest::call_RAY_MPI_TAG_TEST_NETWORK_MESSAGE(Message*message){
+
+ Rank source=message->getSource();
+
+ #ifdef CONFIG_DEBUG_NETWORK_TEST_PLUGIN
+ cout<<"[NetworkTest.plugin] Rank "<<m_rank<<" receives RAY_MPI_TAG_TEST_NETWORK_MESSAGE from "<<source<<endl;
+ #endif
+
+ Message aMessage(NULL,0,source,RAY_MPI_TAG_TEST_NETWORK_MESSAGE_REPLY,m_rank);
+ m_outbox->push_back(&aMessage);
+
+ #ifdef CONFIG_DEBUG_NETWORK_TEST_PLUGIN
+ cout<<"[NetworkTest.plugin] Rank "<<m_rank<<" sends RAY_MPI_TAG_TEST_NETWORK_MESSAGE_REPLY to "<<source<<endl;
+ #endif
+}
+
void NetworkTest::registerPlugin(ComputeCore*core){
PluginHandle plugin = core->allocatePluginHandle();
@@ -435,6 +468,10 @@ void NetworkTest::registerPlugin(ComputeCore*core){
core->setMasterModeObjectHandler(plugin,RAY_MASTER_MODE_TEST_NETWORK, __GetAdapter(NetworkTest,RAY_MASTER_MODE_TEST_NETWORK));
core->setMasterModeSymbol(plugin,RAY_MASTER_MODE_TEST_NETWORK,"RAY_MASTER_MODE_TEST_NETWORK");
+ RAY_MPI_TAG_TEST_NETWORK_MESSAGE=core->allocateMessageTagHandle(plugin);
+ core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_TEST_NETWORK_MESSAGE, __GetAdapter(NetworkTest,RAY_MPI_TAG_TEST_NETWORK_MESSAGE));
+ core->setMessageTagSymbol(plugin,RAY_MPI_TAG_TEST_NETWORK_MESSAGE,"RAY_MPI_TAG_TEST_NETWORK_MESSAGE");
+
RAY_MPI_TAG_TEST_NETWORK=core->allocateMessageTagHandle(plugin);
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_TEST_NETWORK,"RAY_MPI_TAG_TEST_NETWORK");
@@ -471,12 +508,17 @@ void NetworkTest::resolveSymbols(ComputeCore*core){
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_TEST_NETWORK, RAY_SLAVE_MODE_TEST_NETWORK);
+ core->setMessageTagReplyMessageTag(m_plugin, RAY_MPI_TAG_TEST_NETWORK_MESSAGE,RAY_MPI_TAG_TEST_NETWORK_MESSAGE_REPLY );
core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_TEST_NETWORK,RAY_MASTER_MODE_COUNT_FILE_ENTRIES);
m_started=false;
__BindPlugin(NetworkTest);
+
+ __BindAdapter(NetworkTest,RAY_MASTER_MODE_TEST_NETWORK);
+ __BindAdapter(NetworkTest,RAY_MPI_TAG_TEST_NETWORK_MESSAGE);
+ __BindAdapter(NetworkTest,RAY_SLAVE_MODE_TEST_NETWORK);
}
#undef __MAXIMUM_LATENCY /**/
diff --git a/code/plugin_NetworkTest/NetworkTest.h b/code/NetworkTest/NetworkTest.h
similarity index 79%
rename from code/plugin_NetworkTest/NetworkTest.h
rename to code/NetworkTest/NetworkTest.h
index 40fe8a1..4a12967 100644
--- a/code/plugin_NetworkTest/NetworkTest.h
+++ b/code/NetworkTest/NetworkTest.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -21,20 +21,25 @@
#ifndef _NetworkTest_H
#define _NetworkTest_H
-#include <structures/StaticVector.h>
-#include <application_core/Parameters.h>
-#include <profiling/TimePrinter.h>
-#include <memory/RingAllocator.h>
-#include <scheduling/SwitchMan.h>
-#include <handlers/MasterModeHandler.h>
-#include <handlers/SlaveModeHandler.h>
-#include <core/ComputeCore.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/profiling/TimePrinter.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/scheduling/SwitchMan.h>
+#include <RayPlatform/handlers/MasterModeHandler.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <string>
#include <map>
using namespace std;
+__DeclarePlugin(NetworkTest);
+__DeclareMasterModeAdapter(NetworkTest,RAY_MASTER_MODE_TEST_NETWORK);
+__DeclareSlaveModeAdapter(NetworkTest,RAY_SLAVE_MODE_TEST_NETWORK);
+__DeclareMessageTagAdapter(NetworkTest,RAY_MPI_TAG_TEST_NETWORK_MESSAGE);
/**
* This class tests the network
@@ -52,6 +57,10 @@ using namespace std;
*/
class NetworkTest : public CorePlugin {
+ __AddAdapter(NetworkTest,RAY_MASTER_MODE_TEST_NETWORK);
+ __AddAdapter(NetworkTest,RAY_SLAVE_MODE_TEST_NETWORK);
+ __AddAdapter(NetworkTest,RAY_MPI_TAG_TEST_NETWORK_MESSAGE);
+
MessageTag RAY_MPI_TAG_TEST_NETWORK;
MessageTag RAY_MPI_TAG_TEST_NETWORK_MESSAGE;
MessageTag RAY_MPI_TAG_TEST_NETWORK_MESSAGE_REPLY;
@@ -60,12 +69,11 @@ class NetworkTest : public CorePlugin {
MessageTag RAY_MPI_TAG_TEST_NETWORK_WRITE_DATA;
MasterMode RAY_MASTER_MODE_KILL_ALL_MPI_RANKS;
- MasterMode RAY_MASTER_MODE_TEST_NETWORK;
+ MasterMode RAY_MASTER_MODE_TEST_NETWORK;
MasterMode RAY_MASTER_MODE_COUNT_FILE_ENTRIES;
SlaveMode RAY_SLAVE_MODE_TEST_NETWORK;
-
SwitchMan*m_switchMan;
/** indicates if a rank got a response from master **/
@@ -141,9 +149,12 @@ public:
/** initialize the NetworkTest */
void constructor(Rank rank,int size,StaticVector*inbox,StaticVector*outbox,Parameters*parameters,RingAllocator*outboxAllocator,
string*name,TimePrinter*timePrinter);
+
/** work method for the master mode */
void call_RAY_MASTER_MODE_TEST_NETWORK ();
+ void call_RAY_MPI_TAG_TEST_NETWORK_MESSAGE(Message*message);
+
/** work method for the slave mode */
void call_RAY_SLAVE_MODE_TEST_NETWORK();
diff --git a/code/Partitioner/Makefile b/code/Partitioner/Makefile
new file mode 100644
index 0000000..bbd13f1
--- /dev/null
+++ b/code/Partitioner/Makefile
@@ -0,0 +1,3 @@
+Partitioner-y += code/Partitioner/Partitioner.o
+
+obj-y += $(Partitioner-y)
diff --git a/code/plugin_Partitioner/Partitioner.cpp b/code/Partitioner/Partitioner.cpp
similarity index 84%
rename from code/plugin_Partitioner/Partitioner.cpp
rename to code/Partitioner/Partitioner.cpp
index d0ede9b..a4895a4 100644
--- a/code/plugin_Partitioner/Partitioner.cpp
+++ b/code/Partitioner/Partitioner.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,20 +19,16 @@
*/
-#include <plugin_Partitioner/Partitioner.h>
-#include <core/OperatingSystem.h>
+#include "Partitioner.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
#include <stdlib.h>
__CreatePlugin(Partitioner);
- /**/
-__CreateMasterModeAdapter(Partitioner,RAY_MASTER_MODE_COUNT_FILE_ENTRIES); /**/
- /**/
-__CreateSlaveModeAdapter(Partitioner,RAY_SLAVE_MODE_COUNT_FILE_ENTRIES); /**/
- /**/
- /**/
-
+__CreateMasterModeAdapter(Partitioner,RAY_MASTER_MODE_COUNT_FILE_ENTRIES);
+__CreateSlaveModeAdapter(Partitioner,RAY_SLAVE_MODE_COUNT_FILE_ENTRIES);
void Partitioner::constructor(RingAllocator*outboxAllocator,StaticVector*inbox,StaticVector*outbox,Parameters*parameters,
SwitchMan*switchMan){
@@ -44,7 +40,8 @@ void Partitioner::constructor(RingAllocator*outboxAllocator,StaticVector*inbox,S
m_parameters=parameters;
m_initiatedMaster=false;
m_initiatedSlave=false;
- m_loader.constructor(m_parameters->getMemoryPrefix().c_str(),m_parameters->showMemoryAllocations());
+ m_loader.constructor(m_parameters->getMemoryPrefix().c_str(),m_parameters->showMemoryAllocations(),
+ m_parameters->getRank());
}
void Partitioner::call_RAY_MASTER_MODE_COUNT_FILE_ENTRIES(){
@@ -55,7 +52,7 @@ void Partitioner::call_RAY_MASTER_MODE_COUNT_FILE_ENTRIES(){
m_ranksDoneSending=0;
for(int destination=0;destination<m_parameters->getSize();destination++){
Message aMessage(NULL,0,destination,RAY_MPI_TAG_COUNT_FILE_ENTRIES,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
/** a peer rank finished counting the entries in its files */
}else if(m_inbox->size()>0 && m_inbox->at(0)->getTag()== RAY_MPI_TAG_COUNT_FILE_ENTRIES_REPLY){
@@ -64,7 +61,7 @@ void Partitioner::call_RAY_MASTER_MODE_COUNT_FILE_ENTRIES(){
if(m_ranksDoneCounting==m_parameters->getSize()){
for(int destination=0;destination<m_parameters->getSize();destination++){
Message aMessage(NULL,0,destination,RAY_MPI_TAG_REQUEST_FILE_ENTRY_COUNTS,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
}
/** a peer send the count for one file */
@@ -78,10 +75,11 @@ void Partitioner::call_RAY_MASTER_MODE_COUNT_FILE_ENTRIES(){
cout<<"Rank "<<m_parameters->getRank()<<" received from "<<m_inbox->at(0)->getSource()<<" File "<<file<<" Entries "<<count<<endl;
/** reply to the peer */
Message aMessage(NULL,0,m_inbox->at(0)->getSource(),RAY_MPI_TAG_FILE_ENTRY_COUNT_REPLY,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
/** a peer finished sending file counts */
}else if(m_inbox->size()>0 && m_inbox->at(0)->getTag()== RAY_MPI_TAG_REQUEST_FILE_ENTRY_COUNTS_REPLY){
m_ranksDoneSending++;
+
/** all peers have finished */
if(m_ranksDoneSending==m_parameters->getSize()){
@@ -135,7 +133,7 @@ void Partitioner::call_RAY_MASTER_MODE_COUNT_FILE_ENTRIES(){
totalSequences+=entries;
}
-
+
partitionStream.close();
f2<<endl;
@@ -162,7 +160,7 @@ void Partitioner::call_RAY_MASTER_MODE_COUNT_FILE_ENTRIES(){
if(i==m_parameters->getSize()-1){
last=totalSequences-1;
}
-
+
LargeCount count=last-first+1;
f3<<i<<"\t"<<first<<"\t"<<last<<"\t"<<count<<endl;
@@ -170,11 +168,73 @@ void Partitioner::call_RAY_MASTER_MODE_COUNT_FILE_ENTRIES(){
f3.close();
cout<<"Rank "<<m_parameters->getRank()<<" wrote "<<fileName2.str()<<endl;
+ // check if the number of sequences is valid
+ // if not, abort the mission !!!
+ if(!checkIfPairedFilesAreValid()) {
+
+ m_core->getSwitchMan()->setMasterMode(RAY_MASTER_MODE_KILL_ALL_MPI_RANKS);
+ return;
+ }
+
m_switchMan->closeMasterMode();
}
}
}
+bool Partitioner::checkIfPairedFilesAreValid() {
+
+ int numberOfFiles = m_parameters->getNumberOfFiles();
+
+ map<int, vector<int> > libraries;
+
+ for(int file = 0 ; file < numberOfFiles ; ++ file) {
+
+ if(m_parameters->isLeftFile(file)
+ || m_parameters->isRightFile(file)) {
+
+ int library = m_parameters->getLibrary(file);
+
+ libraries[library].push_back(file);
+ }
+ }
+
+ for(map<int, vector<int> >::iterator i = libraries.begin();
+ i != libraries.end(); ++i) {
+
+#ifdef CONFIG_ASSERT
+ assert(i->second.size() == 2);
+#endif
+
+ int file1 = i->second[0];
+ int file2 = i->second[1];
+
+ int count1 = m_parameters->getNumberOfSequences(file1);
+ int count2 = m_parameters->getNumberOfSequences(file2);
+
+ bool verbose = true;
+
+ if(count1 != count2) {
+
+ if(verbose) {
+ string name1 = m_parameters->getFile(file1);
+ string name2 = m_parameters->getFile(file2);
+
+ cout << endl;
+ cout << "Rank " << m_core->getRank() << " : Error, ";
+ cout << name1 << " contains " << count1 << " sequences";
+ cout << " and ";
+ cout << name2 << " contains " << count2 << " sequences";
+ cout << " (must be the same)" << endl;
+ cout << endl;
+ }
+
+ return false;
+ }
+ }
+
+ return true;
+}
+
void Partitioner::call_RAY_SLAVE_MODE_COUNT_FILE_ENTRIES(){
/** initialize the slave */
if(!m_initiatedSlave){
@@ -182,7 +242,7 @@ void Partitioner::call_RAY_SLAVE_MODE_COUNT_FILE_ENTRIES(){
m_currentFileToCount=0;
m_currentlySendingCounts=false;
m_sentCount=false;
-
+
/* possibly read the checkpoint */
if(m_parameters->hasCheckpoint("Partition")){
ifstream f(m_parameters->getCheckpointFile("Partition").c_str());
@@ -199,7 +259,7 @@ void Partitioner::call_RAY_SLAVE_MODE_COUNT_FILE_ENTRIES(){
assert(file>=0);
assert(m_slaveCounts.count(file)==0);
#endif
-
+
m_slaveCounts[file]=sequences;
#ifdef ASSERT
@@ -213,25 +273,27 @@ void Partitioner::call_RAY_SLAVE_MODE_COUNT_FILE_ENTRIES(){
/* all files were processed, tell control peer that we are done */
}else if(m_currentFileToCount==m_parameters->getNumberOfFiles()){
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_COUNT_FILE_ENTRIES_REPLY,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
/* increment it so we don't go here again. */
m_currentFileToCount++;
/* Here we write the checkpoint Partition */
if(m_parameters->writeCheckpoints() && !m_parameters->hasCheckpoint("Partition")){
ofstream f(m_parameters->getCheckpointFile("Partition").c_str());
+ ostringstream buffer;
cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint Partition"<<endl;
int count=m_slaveCounts.size();
- f.write((char*)&count,sizeof(int));
+ buffer.write((char*)&count , sizeof(int));
for(map<int,LargeCount>::iterator i=m_slaveCounts.begin();
i!=m_slaveCounts.end();i++){
int file=i->first;
LargeCount sequences=i->second;
- f.write((char*)&file,sizeof(int));
- f.write((char*)&sequences,sizeof(LargeCount));
+ buffer.write((char*)&file, sizeof(int));
+ buffer.write((char*)&sequences, sizeof(LargeCount));
+ flushFileOperationBuffer(false, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
}
-
+ flushFileOperationBuffer(true, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
f.close();
}
/** count sequences in a file */
@@ -270,7 +332,7 @@ void Partitioner::call_RAY_SLAVE_MODE_COUNT_FILE_ENTRIES(){
message[0]=m_currentFileToSend;
message[1]=m_slaveCounts[m_currentFileToSend];
Message aMessage(message,2,MASTER_RANK,RAY_MPI_TAG_FILE_ENTRY_COUNT,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_sentCount=true;
/** we got a reply, let's continue */
}else if(m_inbox->size()>0 && m_inbox->at(0)->getTag() == RAY_MPI_TAG_FILE_ENTRY_COUNT_REPLY){
@@ -280,7 +342,7 @@ void Partitioner::call_RAY_SLAVE_MODE_COUNT_FILE_ENTRIES(){
/** all counts were processed, report this to the control peer */
}else{
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_REQUEST_FILE_ENTRY_COUNTS_REPLY,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_slaveCounts.clear();
m_switchMan->setSlaveMode(RAY_SLAVE_MODE_DO_NOTHING);
@@ -291,6 +353,7 @@ void Partitioner::call_RAY_SLAVE_MODE_COUNT_FILE_ENTRIES(){
void Partitioner::registerPlugin(ComputeCore*core){
PluginHandle plugin=core->allocatePluginHandle();
m_plugin=plugin;
+ m_core = core;
core->setPluginName(plugin,"Partitioner");
core->setPluginDescription(plugin,"A plugin that counts sequences in files in parallel");
@@ -326,6 +389,9 @@ void Partitioner::registerPlugin(ComputeCore*core){
}
void Partitioner::resolveSymbols(ComputeCore*core){
+
+ RAY_MASTER_MODE_KILL_ALL_MPI_RANKS=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_KILL_ALL_MPI_RANKS");
+
RAY_SLAVE_MODE_DO_NOTHING=core->getSlaveModeFromSymbol(m_plugin,"RAY_SLAVE_MODE_DO_NOTHING");
RAY_SLAVE_MODE_COUNT_FILE_ENTRIES=core->getSlaveModeFromSymbol(m_plugin,"RAY_SLAVE_MODE_COUNT_FILE_ENTRIES");
@@ -348,4 +414,6 @@ void Partitioner::resolveSymbols(ComputeCore*core){
__BindPlugin(Partitioner);
+ __BindAdapter(Partitioner,RAY_MASTER_MODE_COUNT_FILE_ENTRIES);
+ __BindAdapter(Partitioner,RAY_SLAVE_MODE_COUNT_FILE_ENTRIES);
}
diff --git a/code/plugin_Partitioner/Partitioner.h b/code/Partitioner/Partitioner.h
similarity index 77%
rename from code/plugin_Partitioner/Partitioner.h
rename to code/Partitioner/Partitioner.h
index 8a64ad2..f2dd0a2 100644
--- a/code/plugin_Partitioner/Partitioner.h
+++ b/code/Partitioner/Partitioner.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,19 +22,23 @@
#ifndef _Partitioner_H
#define _Partitioner_H
-#include <structures/StaticVector.h>
-#include <application_core/Parameters.h>
-#include <memory/RingAllocator.h>
-#include <plugin_SequencesLoader/Loader.h>
-#include <scheduling/SwitchMan.h>
-#include <handlers/SlaveModeHandler.h>
-#include <handlers/MasterModeHandler.h>
-#include <core/ComputeCore.h>
+#include <code/Mock/Parameters.h>
+#include <code/SequencesLoader/Loader.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/scheduling/SwitchMan.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/handlers/MasterModeHandler.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <map>
using namespace std;
+__DeclarePlugin(Partitioner);
+
+__DeclareMasterModeAdapter(Partitioner,RAY_MASTER_MODE_COUNT_FILE_ENTRIES);
+__DeclareSlaveModeAdapter(Partitioner,RAY_SLAVE_MODE_COUNT_FILE_ENTRIES);
/**
* This class counts the number of entries in each input file in parallel
@@ -42,6 +46,9 @@ using namespace std;
*/
class Partitioner : public CorePlugin{
+ __AddAdapter(Partitioner,RAY_MASTER_MODE_COUNT_FILE_ENTRIES);
+ __AddAdapter(Partitioner,RAY_SLAVE_MODE_COUNT_FILE_ENTRIES);
+
MessageTag RAY_MPI_TAG_COUNT_FILE_ENTRIES;
MessageTag RAY_MPI_TAG_COUNT_FILE_ENTRIES_REPLY;
MessageTag RAY_MPI_TAG_FILE_ENTRY_COUNT;
@@ -55,7 +62,7 @@ class Partitioner : public CorePlugin{
SlaveMode RAY_SLAVE_MODE_COUNT_FILE_ENTRIES;
SlaveMode RAY_SLAVE_MODE_DO_NOTHING;
-
+ MasterMode RAY_MASTER_MODE_KILL_ALL_MPI_RANKS;
SwitchMan*m_switchMan;
@@ -94,6 +101,9 @@ class Partitioner : public CorePlugin{
/** --- */
+
+ bool checkIfPairedFilesAreValid();
+
public:
void constructor(RingAllocator*outboxAllocator,StaticVector*inbox,StaticVector*outbox,Parameters*parameters,
SwitchMan*switchMan);
diff --git a/code/PathEvaluator/Makefile b/code/PathEvaluator/Makefile
new file mode 100644
index 0000000..4348195
--- /dev/null
+++ b/code/PathEvaluator/Makefile
@@ -0,0 +1,3 @@
+PathEvaluator-y += code/PathEvaluator/PathEvaluator.o
+
+obj-y += $(PathEvaluator-y)
diff --git a/code/PathEvaluator/PathEvaluator.cpp b/code/PathEvaluator/PathEvaluator.cpp
new file mode 100644
index 0000000..a12e552
--- /dev/null
+++ b/code/PathEvaluator/PathEvaluator.cpp
@@ -0,0 +1,113 @@
+/*
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+*/
+
+#include "PathEvaluator.h"
+
+#include <fstream>
+using namespace std;
+
+__CreatePlugin(PathEvaluator);
+
+__CreateMasterModeAdapter(PathEvaluator,RAY_MASTER_MODE_EVALUATE_PATHS);
+__CreateSlaveModeAdapter(PathEvaluator,RAY_SLAVE_MODE_EVALUATE_PATHS);
+
+void PathEvaluator::call_RAY_MASTER_MODE_EVALUATE_PATHS(){
+
+ if(!m_masterModeStarted){
+ m_core->getSwitchMan()->openMasterMode(m_core->getOutbox(),m_core->getRank());
+
+ m_masterModeStarted=true;
+
+ }else if(m_core->getSwitchMan()->allRanksAreReady()){
+ m_core->getSwitchMan()->closeMasterMode();
+ }
+}
+
+void PathEvaluator::call_RAY_SLAVE_MODE_EVALUATE_PATHS(){
+
+ writeCheckpointForContigPaths();
+
+ m_core->getSwitchMan()->closeSlaveModeLocally(m_core->getOutbox(),m_core->getRank());
+}
+
+void PathEvaluator::writeCheckpointForContigPaths(){
+
+ /** possibly write the checkpoint */
+ if(m_parameters->writeCheckpoints() && !m_parameters->hasCheckpoint("ContigPaths")){
+ cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint ContigPaths"<<endl;
+ ofstream f(m_parameters->getCheckpointFile("ContigPaths").c_str());
+ ostringstream buffer;
+
+ int theSize=m_contigs->size();
+ buffer.write((char*)&theSize, sizeof(int));
+
+ /* write each path with its name and vertices */
+ for(int i=0;i<theSize;i++){
+ PathHandle name=(*m_contigNames)[i];
+ int vertices=(*m_contigs)[i].size();
+ buffer.write((char*)&name, sizeof(PathHandle));
+ buffer.write((char*)&vertices, sizeof(int));
+ for(int j=0;j<vertices;j++){
+ Kmer kmer;
+ (*m_contigs)[i].at(j,&kmer);
+ kmer.write(&buffer);
+ }
+ flushFileOperationBuffer(false, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
+ }
+ flushFileOperationBuffer(true, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
+ f.close();
+ }
+}
+
+void PathEvaluator::registerPlugin(ComputeCore*core){
+
+ PluginHandle plugin=core->allocatePluginHandle();
+ m_plugin=plugin;
+ m_core=core;
+
+ core->setPluginName(plugin,"PathEvaluator");
+ core->setPluginDescription(plugin,"Post-processing of paths.");
+ core->setPluginAuthors(plugin,"Sébastien Boisvert");
+ core->setPluginLicense(plugin,"GNU General Public License version 3");
+
+ __ConfigureMasterModeHandler(PathEvaluator, RAY_MASTER_MODE_EVALUATE_PATHS);
+ __ConfigureSlaveModeHandler(PathEvaluator, RAY_SLAVE_MODE_EVALUATE_PATHS);
+
+ RAY_MPI_TAG_EVALUATE_PATHS=core->allocateMessageTagHandle(plugin);
+ core->setMessageTagSymbol(plugin,RAY_MPI_TAG_EVALUATE_PATHS,"RAY_MPI_TAG_EVALUATE_PATHS");
+
+ __BindPlugin(PathEvaluator);
+}
+
+void PathEvaluator::resolveSymbols(ComputeCore*core){
+
+ RAY_MASTER_MODE_EVALUATE_PATHS=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_EVALUATE_PATHS");
+ RAY_MASTER_MODE_ASK_EXTENSIONS=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_ASK_EXTENSIONS");
+
+ core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_EVALUATE_PATHS,RAY_MASTER_MODE_ASK_EXTENSIONS);
+ core->setMasterModeToMessageTagSwitch(m_plugin,RAY_MASTER_MODE_EVALUATE_PATHS,RAY_MPI_TAG_EVALUATE_PATHS);
+ core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_EVALUATE_PATHS,RAY_SLAVE_MODE_EVALUATE_PATHS);
+
+ m_parameters=(Parameters*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/Parameters.ray");
+ m_contigs=(vector<GraphPath>*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/ContigPaths.ray");
+ m_contigNames=(vector<PathHandle>*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/ContigNames.ray");
+
+ m_masterModeStarted=false;
+}
diff --git a/code/PathEvaluator/PathEvaluator.h b/code/PathEvaluator/PathEvaluator.h
new file mode 100644
index 0000000..10a441c
--- /dev/null
+++ b/code/PathEvaluator/PathEvaluator.h
@@ -0,0 +1,68 @@
+/*
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+*/
+
+#ifndef _PathEvaluator_h
+#define _PathEvaluator_h
+
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/core/ComputeCore.h>
+
+__DeclarePlugin(PathEvaluator);
+
+__DeclareMasterModeAdapter(PathEvaluator,RAY_MASTER_MODE_EVALUATE_PATHS);
+__DeclareSlaveModeAdapter(PathEvaluator,RAY_SLAVE_MODE_EVALUATE_PATHS);
+
+/**
+ * Evaluate the quality of paths.
+ *
+ * \author Sébastien Boisvert
+ **/
+class PathEvaluator : public CorePlugin {
+
+ bool m_masterModeStarted;
+
+ __AddAdapter(PathEvaluator,RAY_MASTER_MODE_EVALUATE_PATHS);
+ __AddAdapter(PathEvaluator,RAY_SLAVE_MODE_EVALUATE_PATHS);
+
+ MasterMode RAY_MASTER_MODE_EVALUATE_PATHS;
+ MasterMode RAY_MASTER_MODE_ASK_EXTENSIONS;
+
+ SlaveMode RAY_SLAVE_MODE_EVALUATE_PATHS;
+ MessageTag RAY_MPI_TAG_EVALUATE_PATHS;
+
+ void writeCheckpointForContigPaths();
+
+ Parameters*m_parameters;
+
+ /** contig paths */
+ vector<GraphPath>*m_contigs;
+ vector<PathHandle>*m_contigNames;
+
+public:
+
+ void registerPlugin(ComputeCore*core);
+ void resolveSymbols(ComputeCore*core);
+
+ void call_RAY_MASTER_MODE_EVALUATE_PATHS();
+ void call_RAY_SLAVE_MODE_EVALUATE_PATHS();
+};
+
+#endif /* _PathEvaluator_h */
diff --git a/code/Scaffolder/Makefile b/code/Scaffolder/Makefile
new file mode 100644
index 0000000..8ccf29d
--- /dev/null
+++ b/code/Scaffolder/Makefile
@@ -0,0 +1,8 @@
+Scaffolder-y += code/Scaffolder/Scaffolder.o
+Scaffolder-y += code/Scaffolder/ScaffoldingLink.o
+Scaffolder-y += code/Scaffolder/SummarizedLink.o
+Scaffolder-y += code/Scaffolder/ScaffoldingAlgorithm.o
+Scaffolder-y += code/Scaffolder/ScaffoldingVertex.o
+Scaffolder-y += code/Scaffolder/ScaffoldingEdge.o
+
+obj-y += $(Scaffolder-y)
diff --git a/code/plugin_Scaffolder/Scaffolder.cpp b/code/Scaffolder/Scaffolder.cpp
similarity index 75%
rename from code/plugin_Scaffolder/Scaffolder.cpp
rename to code/Scaffolder/Scaffolder.cpp
index 1b8305c..3093471 100644
--- a/code/plugin_Scaffolder/Scaffolder.cpp
+++ b/code/Scaffolder/Scaffolder.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2011, 2012 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,37 +19,40 @@
*/
-#include <core/OperatingSystem.h>
-#include <plugin_SeedExtender/ReadFetcher.h>
-#include <plugin_Scaffolder/Scaffolder.h>
-#include <communication/Message.h>
-#include <core/ComputeCore.h>
-#include <plugin_Scaffolder/ScaffoldingVertex.h>
-#include <plugin_Scaffolder/ScaffoldingEdge.h>
-#include <plugin_Scaffolder/ScaffoldingAlgorithm.h>
-#include <core/statistics.h>
+#include "Scaffolder.h"
+#include "ScaffoldingVertex.h"
+#include "ScaffoldingEdge.h"
+#include "ScaffoldingAlgorithm.h"
+
+#include <code/SeedExtender/ReadFetcher.h>
+
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/core/statistics.h>
+#include <RayPlatform/communication/Message.h>
/* this header was missing, but the code compiled with clang++, gcc, intel, pgi, but not pathscale. pathscale was right */
-#include <stdio.h>
#include <algorithm> /* for sort */
#include <iostream>
#include <vector>
-
#include <fstream>
#include <sstream>
-#include <assert.h>
+#include <iomanip> /* for setprecision */
+using namespace std;
+
#include <math.h> /* for sqrt */
+#include <assert.h>
+#include <stdio.h>
__CreatePlugin(Scaffolder);
- /**/
-__CreateMasterModeAdapter(Scaffolder,RAY_MASTER_MODE_WRITE_SCAFFOLDS); /**/
- /**/
-__CreateSlaveModeAdapter(Scaffolder,RAY_SLAVE_MODE_SCAFFOLDER); /**/
- /**/
- /**/
+__CreateMasterModeAdapter(Scaffolder,RAY_MASTER_MODE_WRITE_SCAFFOLDS);
+__CreateSlaveModeAdapter(Scaffolder,RAY_SLAVE_MODE_SCAFFOLDER);
-using namespace std;
+__CreateMessageTagAdapter(Scaffolder,RAY_MPI_TAG_GET_CONTIG_CHUNK);
+__CreateMessageTagAdapter(Scaffolder,RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK);
+
+// #define DEBUG_SCAFFOLDER_MESSAGES
void Scaffolder::addMasterLink(SummarizedLink*a){
m_masterLinks.push_back(*a);
@@ -60,8 +63,6 @@ void Scaffolder::addMasterContig(PathHandle name,int length){
m_masterLengths.push_back(length);
}
-
-
void Scaffolder::solve(){
/*
@@ -282,6 +283,10 @@ averageValues[1],countValues[1],standardDeviationValues[1]);
}
void Scaffolder::printFinalMessage(){
+
+ if(m_contigLengths.size() == 0)
+ return;
+
ostringstream outputStat;
outputStat<<m_parameters->getPrefix()<<"OutputNumbers.txt";
ofstream f5(outputStat.str().c_str());
@@ -290,6 +295,20 @@ void Scaffolder::printFinalMessage(){
printInStream(&f5);
f5.close();
+
+ cout<<endl;
+ cout<<"Rank "<<m_parameters->getRank()<<" wrote "<<m_parameters->getOutputFile()<<endl;
+ cout<<"Rank "<<m_parameters->getRank()<<" wrote "<<m_parameters->getScaffoldFile()<<endl;
+ cout<<"Check for "<<m_parameters->getPrefix()<<"*"<<endl;
+ cout<<endl;
+ if(m_parameters->useAmos()){
+ cout<<"Rank "<< m_parameters->getRank()<<" wrote ";
+ cout <<m_parameters->getAmosFile()<<" (reads mapped onto contiguous sequences in AMOS format)"<<endl;
+
+ }
+ cout<<endl;
+
+
}
void Scaffolder::computeStatistics(vector<int>*lengths,int minimumLength,ostream*outputStream){
@@ -364,6 +383,12 @@ void Scaffolder::constructor(StaticVector*outbox,StaticVector*inbox,RingAllocato
m_parameters=parameters;
m_initialised=false;
m_workerId=0;
+
+ #ifdef ASSERT
+ assert(m_parameters!=NULL);
+ #endif
+
+ m_rank=m_parameters->getRank();
}
void Scaffolder::call_RAY_SLAVE_MODE_SCAFFOLDER(){
@@ -374,6 +399,15 @@ void Scaffolder::call_RAY_SLAVE_MODE_SCAFFOLDER(){
m_positionOnContig=0;
m_forwardDone=false;
m_coverageRequested=false;
+
+/*
+ * We compute the peak coverage since we want to dodge any
+ * repeats like plague otherwise too many messages will be
+ * sent.
+ */
+ m_coverageWasComputedWithJustice=false;
+ m_skippedRepeatedObjects=0;
+
bool hasPairedReads=m_parameters->hasPairedReads();
bool disableScaffolder=false;
@@ -411,17 +445,78 @@ void Scaffolder::call_RAY_SLAVE_MODE_SCAFFOLDER(){
}
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_I_FINISHED_SCAFFOLDING,
m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_switchMan->setSlaveMode(RAY_SLAVE_MODE_DO_NOTHING);
}
}
-void Scaffolder::setContigPaths(vector<PathHandle>*names,vector<vector<Kmer> >*paths){
+void Scaffolder::setContigPaths(vector<PathHandle>*names,vector<GraphPath>*paths){
m_contigNames=names;
m_contigs=paths;
}
+void Scaffolder::getCoverageOfBlockOfLife(){
+ if(m_positionOnContig<(int)(*m_contigs)[m_contigId].size()){
+
+// ask for the coverage value
+ if(!m_coverageRequested){
+
+ Kmer vertex2;
+ m_contigs->at(m_contigId).at(m_positionOnContig,&vertex2);
+ Kmer*vertex=&vertex2;
+
+ MessageUnit*buffer=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(Kmer));
+ int bufferPosition=0;
+ vertex->pack(buffer,&bufferPosition);
+ Message aMessage(buffer,m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT),
+ m_parameters->vertexRank(vertex),RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,m_parameters->getRank());
+ m_virtualCommunicator->pushMessage(m_workerId,&aMessage);
+ m_coverageRequested=true;
+ m_coverageReceived=false;
+
+ if(m_positionOnContig==0){
+ m_contigs->at(m_contigId).resetCoverageValues();
+ }
+
+ }else if(!m_coverageReceived
+ &&m_virtualCommunicator->isMessageProcessed(m_workerId)){
+
+ vector<MessageUnit> elements;
+ m_virtualCommunicator->getMessageResponseElements(m_workerId,&elements);
+
+ #ifdef ASSERT
+ assert(elements.size() > 0);
+ #endif
+
+ CoverageDepth coverage=elements[1];
+
+ m_contigs->at(m_contigId).addCoverageValue(coverage);
+
+ m_coverageReceived=true;
+
+/*
+ * Reset the fancy state
+ */
+ m_positionOnContig++;
+ m_coverageRequested=false;
+ }
+ }else{
+
+ m_coverageRequested=false;
+ m_positionOnContig=0; // move the head
+
+ m_contigs->at(m_contigId).computePeakCoverage();
+
+ CoverageDepth peak=m_contigs->at(m_contigId).getPeakCoverage();
+
+ cout<<"Rank "<<m_rank<<" objectName: "<<m_contigNames->at(m_contigId)<<" => peakCoverage: "<<peak<<" blockSize: ";
+ cout<<m_contigs->at(m_contigId).size()<<endl;
+
+ m_coverageWasComputedWithJustice=true;
+ }
+}
+
void Scaffolder::processContig(){
/** skip the time-consuming parts **/
@@ -436,9 +531,13 @@ void Scaffolder::processContig(){
// but send the contig meta information
m_sentContigMeta=false;
m_sentContigInfo=false;
- }
- if(m_positionOnContig<(int)(*m_contigs)[m_contigId].size()){
+ }else if(!m_coverageWasComputedWithJustice){
+ if(m_skipScaffolding)
+ m_coverageWasComputedWithJustice=true;
+ else
+ getCoverageOfBlockOfLife();
+ }else if(m_positionOnContig<(int)(*m_contigs)[m_contigId].size()){
processContigPosition();
}else if(!m_summaryPerformed){
performSummary();
@@ -449,13 +548,15 @@ void Scaffolder::processContig(){
}else{
m_contigId++;
m_positionOnContig=0;
+ m_coverageWasComputedWithJustice=false;
+ m_skippedRepeatedObjects=0;
}
}
void Scaffolder::sendContigInfo(){
if(!m_sentContigInfo){
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
- message[0]=(*m_contigNames)[m_contigId];
+ message[0]=(*m_contigNames)[m_contigId].getValue();
message[1]=(*m_contigs)[m_contigId].size();
Message aMessage(message,2,
MASTER_RANK,RAY_MPI_TAG_CONTIG_INFO,m_parameters->getRank());
@@ -495,17 +596,16 @@ void Scaffolder::sendSummary(){
void Scaffolder::performSummary(){
-
#ifdef ASSERT
assert(m_contigId < (int)m_contigs->size());
- assert(m_vertexCoverageValues.size() == (*m_contigs)[m_contigId].size());
+ assert((int)m_vertexCoverageValues.size() == (*m_contigs)[m_contigId].size());
#endif
LargeCount sum=0;
- int peakCoverage=getMode(&m_vertexCoverageValues);
+ CoverageDepth peakCoverage=m_contigs->at(m_contigId).getPeakCoverage();
- int repeatCoverage=peakCoverage*REPEAT_MULTIPLIER;
+ CoverageDepth repeatCoverage=peakCoverage*REPEAT_MULTIPLIER;
#ifdef CONFIG_USE_COVERAGE_DISTRIBUTION
repeatCoverage=m_parameters->getRepeatCoverage();
@@ -514,7 +614,7 @@ void Scaffolder::performSummary(){
map<int,int> distribution;
int n=0;
for(int i=0;i<(int)m_vertexCoverageValues.size();i++){
- int coverageValue=m_vertexCoverageValues[i];
+ CoverageDepth coverageValue=m_vertexCoverageValues[i];
distribution[coverageValue]++;
if(coverageValue < repeatCoverage){
@@ -523,7 +623,7 @@ void Scaffolder::performSummary(){
}
}
- int mean=sum;
+ CoverageDepth mean=sum;
if(n>0){
mean /= n;
@@ -531,7 +631,7 @@ void Scaffolder::performSummary(){
LargeCount sumOfSquares=0;
for(int i=0;i<(int)m_vertexCoverageValues.size();i++){
- int coverageValue=m_vertexCoverageValues[i];
+ CoverageDepth coverageValue=m_vertexCoverageValues[i];
int diff=coverageValue-mean;
if(coverageValue < repeatCoverage){
sumOfSquares+= diff*diff;
@@ -542,12 +642,7 @@ void Scaffolder::performSummary(){
sumOfSquares /= n;
}
- int standardDeviation=(int)sqrt(sumOfSquares);
-
-/*
- int peakCoverage=getMode(&m_vertexCoverageValues);
- int repeatCoverage=REPEAT_MULTIPLIER*peakCoverage;
-*/
+ CoverageDepth standardDeviation=(int)sqrt(sumOfSquares);
cout<<"contig: "<<(*m_contigNames)[m_contigId]<<" vertices: "<<m_vertexCoverageValues.size();
cout<<" averageCoverage: "<<mean<<" standardDeviation: "<<standardDeviation;
@@ -583,9 +678,12 @@ void Scaffolder::performSummary(){
fp<<"contig-"<<contigName<<endl;
fp<<vertices<<" vertices"<<endl;
fp<<"#Index Vertex Coverage"<<endl;
+
for(int i=0;i<vertices;i++){
- Kmer kmer=(*m_contigs)[m_contigId][i];
- int coverage=m_vertexCoverageValues[i];
+
+ Kmer kmer;
+ (*m_contigs)[m_contigId].at(i,&kmer);
+ CoverageDepth coverage=m_vertexCoverageValues[i];
fp<<i<<" "<<kmer.idToWord(m_parameters->getWordSize(),m_parameters->getColorSpaceMode())<<" "<<coverage<<endl;
}
@@ -613,8 +711,8 @@ void Scaffolder::performSummary(){
for(vector<ScaffoldingLink>::iterator m=l->second.begin();m!=l->second.end();m++){
int distance=(*m).getDistance();
- int coverage1=(*m).getCoverage1();
- int coverage2=(*m).getCoverage2();
+ CoverageDepth coverage1=(*m).getCoverage1();
+ CoverageDepth coverage2=(*m).getCoverage2();
int numberOfStandardDeviations=1;
@@ -657,7 +755,9 @@ void Scaffolder::processContigPosition(){
assert(m_positionOnContig<(int)(*m_contigs)[m_contigId].size());
#endif
- Kmer vertex=(*m_contigs)[m_contigId][m_positionOnContig];
+ Kmer vertex;
+ (*m_contigs)[m_contigId].at(m_positionOnContig,&vertex);
+
#ifdef ASSERT
assert(m_parameters!=NULL);
#endif
@@ -701,7 +801,7 @@ void Scaffolder::processVertex(Kmer*vertex){
int bufferPosition=0;
vertex->pack(buffer,&bufferPosition);
Message aMessage(buffer,m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT),
- m_parameters->_vertexRank(vertex),RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,m_parameters->getRank());
+ m_parameters->vertexRank(vertex),RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,m_parameters->getRank());
m_virtualCommunicator->pushMessage(m_workerId,&aMessage);
m_coverageRequested=true;
m_coverageReceived=false;
@@ -711,17 +811,23 @@ void Scaffolder::processVertex(Kmer*vertex){
m_vertexCoverageValues.clear();
}
- if(m_positionOnContig==(int)(*m_contigs)[m_contigId].size()-1){
+ if(m_positionOnContig==(int)(*m_contigs)[m_contigId].size()-1 && m_contigId%1000==0){
+
printf("Rank %i: gathering scaffold links [%i/%i] [%i/%i] (completed)\n",m_parameters->getRank(),
m_contigId+1,(int)(*m_contigs).size(),
m_positionOnContig+1,(int)(*m_contigs)[m_contigId].size());
+ cout<<"Rank "<<m_rank<<" ineligibleObjects: "<<m_skippedRepeatedObjects<<"/";
+ cout<<(*m_contigs)[m_contigId].size()<<" (";
+ cout<<fixed<<setprecision(2)<<100.0*m_skippedRepeatedObjects/(*m_contigs)[m_contigId].size();
+ cout<<"%)"<<endl;
+
if(m_parameters->showMemoryUsage()){
showMemoryUsage(m_parameters->getRank());
}
- }else if(m_positionOnContig%10000==0){
+ }else if(m_positionOnContig%10000==0 && m_positionOnContig>0){
printf("Rank %i: gathering scaffold links [%i/%i] [%i/%i]\n",m_parameters->getRank(),
m_contigId+1,(int)(*m_contigs).size(),
m_positionOnContig+1,(int)(*m_contigs)[m_contigId].size());
@@ -741,7 +847,7 @@ void Scaffolder::processVertex(Kmer*vertex){
#endif
uint8_t edges=elements[0];
- int coverage=elements[1];
+ CoverageDepth coverage=elements[1];
m_receivedCoverage=coverage;
m_coverageReceived=true;
@@ -751,8 +857,8 @@ void Scaffolder::processVertex(Kmer*vertex){
m_vertexCoverageValues.push_back(m_receivedCoverage);
/* here, make sure that the vertex has exactly 1 parent and 1 child */
- int parents=vertex->_getIngoingEdges(edges,m_parameters->getWordSize()).size();
- int children=vertex->_getOutgoingEdges(edges,m_parameters->getWordSize()).size();
+ int parents=vertex->getIngoingEdges(edges,m_parameters->getWordSize()).size();
+ int children=vertex->getOutgoingEdges(edges,m_parameters->getWordSize()).size();
#ifdef SHOW_EDGES
cout<<"/ "<<coverage<<" "<<parents<<" "<<children<<endl;
@@ -771,7 +877,19 @@ void Scaffolder::processVertex(Kmer*vertex){
}else if(m_coverageReceived){
/* anyway these entries will be checked after anyway... */
- if(1 /*m_receivedCoverage<m_parameters->getRepeatCoverage()*/){
+
+/*
+ * TODO:
+ * Restore this verification as it is required to ensure good speeds overall.
+ * Without this check, the end user is doomed to obtain very bad speeds on
+ * anything that sequences moving around in the genome (like Alu in the human genome).
+ *
+ */
+
+ CoverageDepth theLocalPeakCoverage=m_contigs->at(m_contigId).getPeakCoverage();
+ CoverageDepth limitForMessagingLayer=REPEAT_MULTIPLIER*theLocalPeakCoverage;
+
+ if(m_receivedCoverage < limitForMessagingLayer){
if(!m_initialisedFetcher){
m_readFetcher.constructor(vertex,m_outboxAllocator,m_inbox,
m_outbox,m_parameters,m_virtualCommunicator,m_workerId,
@@ -786,6 +904,7 @@ void Scaffolder::processVertex(Kmer*vertex){
processAnnotations();
}
}else{
+ m_skippedRepeatedObjects++;
m_forwardDone=true;
m_reverseDone=false;
}
@@ -910,7 +1029,7 @@ void Scaffolder::processAnnotation(){
int elementsPerQuery=m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION);
Message aMessage(buffer,elementsPerQuery,
- m_parameters->_vertexRank(&m_pairedForwardMarker),
+ m_parameters->vertexRank(&m_pairedForwardMarker),
RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION,m_parameters->getRank());
m_virtualCommunicator->pushMessage(m_workerId,&aMessage);
m_forwardDirectionsRequested=true;
@@ -932,8 +1051,8 @@ void Scaffolder::processAnnotation(){
m_forwardDirectionLengthRequested=false;
/* here, make sure that the vertex has exactly 1 parent and 1 child */
- int parents=m_pairedForwardMarker._getIngoingEdges(edges,m_parameters->getWordSize()).size();
- int children=m_pairedForwardMarker._getOutgoingEdges(edges,m_parameters->getWordSize()).size();
+ int parents=m_pairedForwardMarker.getIngoingEdges(edges,m_parameters->getWordSize()).size();
+ int children=m_pairedForwardMarker.getOutgoingEdges(edges,m_parameters->getWordSize()).size();
#ifdef SHOW_EDGES
cout<<"/ "<<m_pairedForwardMarkerCoverage<<" "<<parents<<" "<<children<<endl;
@@ -958,7 +1077,7 @@ void Scaffolder::processAnnotation(){
}else if(!m_forwardDirectionLengthRequested){
MessageUnit*buffer=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(Kmer));
int rankId=getRankFromPathUniqueId(m_pairedForwardDirectionName);
- buffer[0]=m_pairedForwardDirectionName;
+ buffer[0]=m_pairedForwardDirectionName.getValue();
Message aMessage(buffer,1,
rankId,
RAY_MPI_TAG_GET_PATH_LENGTH,m_parameters->getRank());
@@ -977,8 +1096,8 @@ void Scaffolder::processAnnotation(){
if(m_pairedForwardDirectionLength<range
||(int)(*m_contigs)[m_contigId].size()<range
- || 2*m_receivedCoverage<m_pairedForwardMarkerCoverage
- || 2*m_pairedForwardMarkerCoverage<m_receivedCoverage ){
+ || REPEAT_MULTIPLIER*m_receivedCoverage<m_pairedForwardMarkerCoverage
+ || REPEAT_MULTIPLIER*m_pairedForwardMarkerCoverage<m_receivedCoverage ){
return;
}
@@ -1127,7 +1246,7 @@ Case 13. (allowed)
int elementsPerQuery=m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION);
Message aMessage(buffer,elementsPerQuery,
- m_parameters->_vertexRank(&m_pairedReverseMarker),
+ m_parameters->vertexRank(&m_pairedReverseMarker),
RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION,m_parameters->getRank());
m_virtualCommunicator->pushMessage(m_workerId,&aMessage);
m_reverseDirectionsRequested=true;
@@ -1147,8 +1266,8 @@ Case 13. (allowed)
m_reverseDirectionLengthRequested=false;
/* here, make sure that the vertex has exactly 1 parent and 1 child */
- int parents=m_pairedReverseMarker._getIngoingEdges(edges,m_parameters->getWordSize()).size();
- int children=m_pairedReverseMarker._getOutgoingEdges(edges,m_parameters->getWordSize()).size();
+ int parents=m_pairedReverseMarker.getIngoingEdges(edges,m_parameters->getWordSize()).size();
+ int children=m_pairedReverseMarker.getOutgoingEdges(edges,m_parameters->getWordSize()).size();
#ifdef SHOW_EDGES
cout<<"/ "<<m_pairedReverseMarkerCoverage<<" "<<parents<<" "<<children<<endl;
@@ -1174,7 +1293,7 @@ Case 13. (allowed)
}else if(!m_reverseDirectionLengthRequested){
MessageUnit*buffer=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(Kmer));
Rank rankId=getRankFromPathUniqueId(m_pairedReverseDirectionName);
- buffer[0]=m_pairedReverseDirectionName;
+ buffer[0]=m_pairedReverseDirectionName.getValue();
Message aMessage(buffer,1,
rankId,RAY_MPI_TAG_GET_PATH_LENGTH,m_parameters->getRank());
m_virtualCommunicator->pushMessage(m_workerId,&aMessage);
@@ -1191,8 +1310,8 @@ Case 13. (allowed)
if(m_pairedReverseDirectionLength<range
||(int)(*m_contigs)[m_contigId].size()<range
- || 2*m_receivedCoverage<m_pairedReverseMarkerCoverage
- || 2*m_pairedReverseMarkerCoverage<m_receivedCoverage){
+ || REPEAT_MULTIPLIER *m_receivedCoverage<m_pairedReverseMarkerCoverage
+ || REPEAT_MULTIPLIER*m_pairedReverseMarkerCoverage<m_receivedCoverage){
return;
}
@@ -1328,12 +1447,112 @@ Case 16. (allowed)
}
void Scaffolder::getContigSequence(PathHandle id){
+ int mode=0;
+ int CODE_PATH_VANILLA_KMERS=mode++;
+ int CODE_PATH_PACKED_REGION=mode++;
+
+ //int configuredCodePath=CODE_PATH_VANILLA_KMERS;
+ int configuredCodePath=CODE_PATH_PACKED_REGION;
+
+ if(configuredCodePath==CODE_PATH_VANILLA_KMERS)
+ getContigSequenceFromKmers(id);
+ else if(configuredCodePath==CODE_PATH_PACKED_REGION){
+ getContigSequenceFromPackedObjects(id);
+ }
+}
+
+void Scaffolder::getContigSequenceFromPackedObjects(PathHandle id){
+
+ if(!m_hasContigSequence_Initialised){
+ m_hasContigSequence_Initialised=true;
+ m_rankIdForContig=getRankFromPathUniqueId(id);
+ m_theLengthInNucleotides=getNumberOfNucleotides(m_contigLengths[id],m_parameters->getWordSize());
+ m_position=0;
+ m_requestedContigChunk=false;
+ m_contigPathBuffer.str("");
+ }
+
+ if(m_position<m_theLengthInNucleotides){
+ if(!m_requestedContigChunk){
+ MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+
+ int bufferPosition=0;
+ message[bufferPosition++]=id.getValue();
+ message[bufferPosition++]=m_position;
+ Message aMessage(message,bufferPosition,
+ m_rankIdForContig,RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK,m_parameters->getRank());
+ m_virtualCommunicator->pushMessage(m_workerId,&aMessage);
+
+#ifdef DEBUG_SCAFFOLDER_MESSAGES
+ cout<<"[DEBUG] getContigSequenceFromPackedObjects SEND RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK"<<endl;
+#endif
+
+ m_requestedContigChunk=true;
+
+ }else if(m_virtualCommunicator->isMessageProcessed(m_workerId)){
+
+#ifdef DEBUG_SCAFFOLDER_MESSAGES
+ cout<<"[DEBUG] getContigSequenceFromPackedObjects RECEIVE RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK_REPLY"<<endl;
+#endif
+
+ bool colored=m_parameters->getColorSpaceMode();
+
+ vector<MessageUnit> data;
+ m_virtualCommunicator->getMessageResponseElements(m_workerId,&data);
+
+ /* the position in the message buffer */
+ int position=0;
+ /* the first element is the number of nucleotides */
+ int count=data[position++];
+ int iterator=0;
+
+ while(iterator<count){
+ int elementIndex=(iterator*BITS_PER_NUCLEOTIDE ) / (sizeof(MessageUnit)*BITS_PER_BYTE);
+ int offset=(iterator*BITS_PER_NUCLEOTIDE) % (sizeof(MessageUnit)*BITS_PER_BYTE);
+
+ #ifdef ASSERT
+ assert(sizeof(MessageUnit)==sizeof(uint64_t));
+ #endif
+
+ uint64_t value=data[position+elementIndex];
+
+ value <<= ( sizeof(uint64_t)*BITS_PER_BYTE - BITS_PER_NUCLEOTIDE - offset);
+ value>>= (sizeof(uint64_t)*BITS_PER_BYTE - BITS_PER_NUCLEOTIDE );
+
+ char nucleotide=codeToChar(value,colored);
+
+ m_contigPathBuffer<<nucleotide;
+
+ iterator++;
+ }
+
+ m_position+=count;
+
+ m_requestedContigChunk=false;
+ }
+ }else{
+ m_contigSequence=m_contigPathBuffer.str();
+ m_hasContigSequence=true;
+
+ #ifdef ASSERT
+ if((int)m_contigSequence.length()!=m_theLengthInNucleotides){
+ cout<<"expected: "<<m_theLengthInNucleotides<<" actual: "<<m_contigSequence.length()<<endl;
+ }
+
+ assert((int)m_contigSequence.length()==m_theLengthInNucleotides);
+ #endif
+ }
+}
+
+void Scaffolder::getContigSequenceFromKmers(PathHandle id){
+
if(!m_hasContigSequence_Initialised){
m_hasContigSequence_Initialised=true;
m_rankIdForContig=getRankFromPathUniqueId(id);
m_theLength=m_contigLengths[id];
m_position=0;
m_contigPath.clear();
+ m_contigPath.setKmerLength(m_parameters->getWordSize());
m_requestedContigChunk=false;
}
@@ -1341,7 +1560,7 @@ void Scaffolder::getContigSequence(PathHandle id){
if(!m_requestedContigChunk){
m_requestedContigChunk=true;
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
- message[0]=id;
+ message[0]=id.getValue();
message[1]=m_position;
Message aMessage(message,2,
m_rankIdForContig,RAY_MPI_TAG_GET_CONTIG_CHUNK,m_parameters->getRank());
@@ -1357,7 +1576,8 @@ void Scaffolder::getContigSequence(PathHandle id){
while(kmerIterator<count){
Kmer a;
a.unpack(&data,&pos);
- m_contigPath.push_back(a);
+ m_contigPath.push_back(&a);
+
kmerIterator++;
}
m_position+=count;
@@ -1422,19 +1642,30 @@ void Scaffolder::call_RAY_MASTER_MODE_WRITE_SCAFFOLDS(){
int columns=m_parameters->getColumns();
ostringstream outputBuffer;
+
+ bool isNotLastContig = m_contigId<(int)m_scaffoldContigs[m_scaffoldId].size()-1;
+
while(contigPosition<length){
char nucleotide=m_contigSequence[contigPosition];
outputBuffer<<nucleotide;
contigPosition++;
m_positionOnScaffold++;
- if(m_positionOnScaffold%columns==0){
+
+/*
+ * Only add a new line if there is something more to add.
+ */
+ if(m_positionOnScaffold%columns==0
+ && ( contigPosition < length || isNotLastContig)){
outputBuffer<<"\n";
}
}
m_operationBuffer<<outputBuffer.str().c_str();
- if(m_contigId<(int)m_scaffoldContigs[m_scaffoldId].size()-1){
+/*
+ * Add the gap.
+ */
+ if(isNotLastContig){
int gapSize=m_scaffoldGaps[m_scaffoldId][m_contigId];
int i=0;
int columns=m_parameters->getColumns();
@@ -1443,6 +1674,11 @@ void Scaffolder::call_RAY_MASTER_MODE_WRITE_SCAFFOLDS(){
outputBuffer2<<"N";
i++;
m_positionOnScaffold++;
+
+/*
+ * We always add the new line because a gap is always followed by
+ * a sequence.
+ */
if(m_positionOnScaffold%columns==0){
outputBuffer2<<"\n";
}
@@ -1482,6 +1718,152 @@ void Scaffolder::setTimePrinter(TimePrinter*a){
m_timePrinter=a;
}
+/**
+ * input is 2 MessageUnit objects: contigId, position
+ *
+ * output is a packed object
+ */
+void Scaffolder::call_RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK(Message*message){
+
+#ifdef DEBUG_SCAFFOLDER_MESSAGES
+ cout<<"[DEBUG] call_RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK RECEIVE RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK"<<endl;
+#endif
+
+// TODO: this returns a part of a region, in 2-bit encoding
+
+ MessageUnit*incoming=(MessageUnit*)message->getBuffer();
+ int thePosition=0;
+ PathHandle contigId=incoming[thePosition++];
+ int nucleotidePosition=incoming[thePosition++];
+
+#ifdef ASSERT
+ assert(m_contigNameIndex->count(contigId)>0);
+#endif
+
+ int kmerLength=m_parameters->getWordSize();
+
+ int index=(*m_contigNameIndex)[contigId];
+ int length=(*m_contigs)[index].size();
+ int lengthInNucleotides=getNumberOfNucleotides(length,kmerLength);
+ MessageUnit*messageContent=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+ int outputPosition=0;
+ int origin=outputPosition;
+ outputPosition++;
+ int count=0;
+
+ bool useColorSpace=m_parameters->getColorSpaceMode();
+
+ int availableMessageUnits=MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit);
+ availableMessageUnits--; // we need one for the count
+
+// clear bits
+ memset(messageContent,0,MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+
+#ifdef ASSERT
+ assert(index<(int)m_contigs->size());
+#endif
+
+ GraphPath*path=&((*m_contigs)[index]);
+
+ while(nucleotidePosition<lengthInNucleotides){
+
+ Kmer theKmerObject;
+
+ int kmerPosition=nucleotidePosition;
+
+ if(nucleotidePosition>=path->size()){
+ kmerPosition=path->size()-1;
+ }
+
+ path->at(kmerPosition,&theKmerObject);
+
+ int localPosition=nucleotidePosition-kmerPosition;
+
+#ifdef ASSERT
+ assert(localPosition>=0);
+ assert(localPosition<kmerLength);
+#endif
+
+ char symbol=theKmerObject.getSymbolAtPosition(kmerPosition,useColorSpace,localPosition);
+
+ int bitPosition=count*BITS_PER_NUCLEOTIDE;
+ int messageUnitIndex=bitPosition/(sizeof(uint64_t)*BITS_PER_BYTE);
+
+/*
+ * Our buffer is full.
+ */
+ if(messageUnitIndex>=availableMessageUnits)
+ break;
+
+ int offsetInMessageUnit=bitPosition%(sizeof(uint64_t)*BITS_PER_BYTE);
+
+ uint64_t oldValue=messageContent[outputPosition+messageUnitIndex];
+
+ uint64_t mask=charToCode(symbol);
+ mask<<=offsetInMessageUnit;
+
+ oldValue|=mask;
+ messageContent[outputPosition+messageUnitIndex]=oldValue;
+
+ count++;
+ nucleotidePosition++;
+ }
+
+// count is the number of packed nucleotides
+ messageContent[origin]=count;
+
+ Message aMessage(messageContent,
+ m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK),
+ message->getSource(),RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK_REPLY,
+ m_parameters->getRank());
+
+ m_outbox->push_back(&aMessage);
+
+#ifdef DEBUG_SCAFFOLDER_MESSAGES
+ cout<<"[DEBUG] call_RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK SEND RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK_REPLY"<<endl;
+#endif
+}
+
+void Scaffolder::call_RAY_MPI_TAG_GET_CONTIG_CHUNK(Message*message){
+ MessageUnit*incoming=(MessageUnit*)message->getBuffer();
+ int thePosition=0;
+ PathHandle contigId=incoming[thePosition++];
+ int position=incoming[thePosition++];
+
+#ifdef ASSERT
+ assert(m_contigNameIndex->count(contigId)>0);
+#endif
+
+ int index=(*m_contigNameIndex)[contigId];
+ int length=(*m_contigs)[index].size();
+ MessageUnit*messageContent=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+ int outputPosition=0;
+ int origin=outputPosition;
+ outputPosition++;
+ int count=0;
+
+ while(position<length
+ && (outputPosition+KMER_U64_ARRAY_SIZE)<(int)(MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit))){
+ Kmer theKmerObject;
+
+#ifdef ASSERT
+ assert(index<(int)m_contigs->size());
+#endif
+ (*m_contigs)[index].at(position++,&theKmerObject);
+ theKmerObject.pack(messageContent,&outputPosition);
+ count++;
+ }
+
+ messageContent[origin]=count;
+ Message aMessage(messageContent,
+ m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_GET_CONTIG_CHUNK),
+ message->getSource(),RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY,
+ m_parameters->getRank());
+ m_outbox->push_back(&aMessage);
+}
+
+
+
void Scaffolder::registerPlugin(ComputeCore*core){
PluginHandle plugin=core->allocatePluginHandle();
@@ -1505,6 +1887,19 @@ void Scaffolder::registerPlugin(ComputeCore*core){
RAY_MPI_TAG_START_SCAFFOLDER=core->allocateMessageTagHandle(plugin);
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_START_SCAFFOLDER,"RAY_MPI_TAG_START_SCAFFOLDER");
+
+ RAY_MPI_TAG_GET_CONTIG_CHUNK=core->allocateMessageTagHandle(plugin);
+ core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_GET_CONTIG_CHUNK, __GetAdapter(Scaffolder,RAY_MPI_TAG_GET_CONTIG_CHUNK));
+ core->setMessageTagSymbol(plugin,RAY_MPI_TAG_GET_CONTIG_CHUNK,"RAY_MPI_TAG_GET_CONTIG_CHUNK");
+
+ RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK=core->allocateMessageTagHandle(plugin);
+ core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK, __GetAdapter(Scaffolder,RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK));
+ core->setMessageTagSymbol(plugin,RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK,"RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK");
+
+ RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY=core->allocateMessageTagHandle(plugin);
+ core->setMessageTagSymbol(plugin,RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY,"RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY");
+ RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK_REPLY=core->allocateMessageTagHandle(plugin);
+ core->setMessageTagSymbol(plugin,RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK_REPLY,"RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK_REPLY");
}
void Scaffolder::resolveSymbols(ComputeCore*core){
@@ -1517,6 +1912,7 @@ void Scaffolder::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_CONTIG_INFO=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_CONTIG_INFO");
RAY_MPI_TAG_GET_CONTIG_CHUNK=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_CONTIG_CHUNK");
+ RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK");
RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION");
RAY_MPI_TAG_GET_PATH_LENGTH=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_PATH_LENGTH");
RAY_MPI_TAG_GET_READ_MARKERS=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_READ_MARKERS");
@@ -1531,9 +1927,37 @@ void Scaffolder::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_SCAFFOLDING_LINKS_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_SCAFFOLDING_LINKS_REPLY");
RAY_MPI_TAG_START_SCAFFOLDER=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_START_SCAFFOLDER");
+ RAY_MPI_TAG_GET_CONTIG_CHUNK=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_CONTIG_CHUNK");
+ RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY");
+ RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK");
+ RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK_REPLY");
+
core->setMessageTagToSlaveModeSwitch(m_plugin, RAY_MPI_TAG_START_SCAFFOLDER, RAY_SLAVE_MODE_SCAFFOLDER );
core->setMasterModeNextMasterMode(m_plugin,RAY_MASTER_MODE_WRITE_SCAFFOLDS, RAY_MASTER_MODE_COUNT_SEARCH_ELEMENTS);
+// the two message tags below won't multiplex very well during the transit
+// in the virtual messaging layer of Ray Platform
+
+ core->setMessageTagReplyMessageTag(m_plugin, RAY_MPI_TAG_GET_CONTIG_CHUNK, RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY );
+ core->setMessageTagSize(m_plugin, RAY_MPI_TAG_GET_CONTIG_CHUNK, MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit) );
+ core->setMessageTagReplyMessageTag(m_plugin, RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK, RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK_REPLY );
+ core->setMessageTagSize(m_plugin, RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK, MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit) );
+
__BindPlugin(Scaffolder);
+
+ __BindAdapter(Scaffolder,RAY_MASTER_MODE_WRITE_SCAFFOLDS);
+ __BindAdapter(Scaffolder,RAY_SLAVE_MODE_SCAFFOLDER);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_CONTIG_CHUNK);
+ __BindAdapter(MessageProcessor,RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK);
+
+#if 0
+ m_contigs=(vector<GraphPath>*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/ContigPaths.ray");
+#endif
+
+ m_contigNameIndex=(map<PathHandle,int>*)core->getObjectFromSymbol(m_plugin, "/RayAssembler/ObjectStore/ContigNameIndex.ray");
+
+#ifdef ASSERT
+ assert(m_contigNameIndex!=NULL);
+#endif
}
diff --git a/code/plugin_Scaffolder/Scaffolder.h b/code/Scaffolder/Scaffolder.h
similarity index 71%
rename from code/plugin_Scaffolder/Scaffolder.h
rename to code/Scaffolder/Scaffolder.h
index baf749f..ab3cb37 100644
--- a/code/plugin_Scaffolder/Scaffolder.h
+++ b/code/Scaffolder/Scaffolder.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,29 +19,36 @@
*/
-#ifndef _Scaffolder
-#define _Scaffolder
+#ifndef _Scaffolder_h
+#define _Scaffolder_h
-#include <application_core/Parameters.h>
-#include <structures/StaticVector.h>
-#include <vector>
-#include <memory/RingAllocator.h>
-#include <application_core/constants.h>
-#include <communication/VirtualCommunicator.h>
-#include <plugin_SeedExtender/ReadFetcher.h>
-#include <sstream>
-#include <plugin_Scaffolder/ScaffoldingLink.h>
-#include <plugin_Scaffolder/SummarizedLink.h>
-#include <scheduling/SwitchMan.h>
-#include <profiling/TimePrinter.h>
+#include "ScaffoldingLink.h"
+#include "SummarizedLink.h"
-#include <handlers/SlaveModeHandler.h>
-#include <handlers/MasterModeHandler.h>
-#include <plugins/CorePlugin.h>
+#include <code/SeedExtender/ReadFetcher.h>
+#include <code/Mock/Parameters.h>
+#include <code/Mock/constants.h>
+#include <code/SeedingData/GraphPath.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/scheduling/SwitchMan.h>
+#include <RayPlatform/profiling/TimePrinter.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/handlers/MasterModeHandler.h>
+#include <RayPlatform/plugins/CorePlugin.h>
+#include <vector>
+#include <sstream>
using namespace std;
+__DeclarePlugin(Scaffolder);
+
+__DeclareMasterModeAdapter(Scaffolder,RAY_MASTER_MODE_WRITE_SCAFFOLDS);
+__DeclareSlaveModeAdapter(Scaffolder,RAY_SLAVE_MODE_SCAFFOLDER);
+__DeclareMessageTagAdapter(Scaffolder,RAY_MPI_TAG_GET_CONTIG_CHUNK);
+__DeclareMessageTagAdapter(Scaffolder,RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK);
/**
* Scaffolder class, it uses MPI through the virtual communicator.
@@ -50,6 +57,11 @@ using namespace std;
*/
class Scaffolder : public CorePlugin{
+ __AddAdapter(Scaffolder,RAY_MASTER_MODE_WRITE_SCAFFOLDS);
+ __AddAdapter(Scaffolder,RAY_SLAVE_MODE_SCAFFOLDER);
+ __AddAdapter(Scaffolder,RAY_MPI_TAG_GET_CONTIG_CHUNK);
+ __AddAdapter(Scaffolder,RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK);
+
ostringstream m_operationBuffer;
MessageTag RAY_MPI_TAG_SCAFFOLDING_LINKS_REPLY;
@@ -58,6 +70,9 @@ class Scaffolder : public CorePlugin{
MessageTag RAY_MPI_TAG_REQUEST_VERTEX_READS;
MessageTag RAY_MPI_TAG_CONTIG_INFO;
MessageTag RAY_MPI_TAG_GET_CONTIG_CHUNK;
+ MessageTag RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY;
+ MessageTag RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK;
+ MessageTag RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK_REPLY;
MessageTag RAY_MPI_TAG_GET_COVERAGE_AND_DIRECTION;
MessageTag RAY_MPI_TAG_GET_PATH_LENGTH;
MessageTag RAY_MPI_TAG_GET_READ_MARKERS;
@@ -74,6 +89,8 @@ class Scaffolder : public CorePlugin{
SlaveMode RAY_SLAVE_MODE_DO_NOTHING;
SlaveMode RAY_SLAVE_MODE_SCAFFOLDER;
+ bool m_coverageWasComputedWithJustice;
+ int m_skippedRepeatedObjects;
SwitchMan*m_switchMan;
@@ -92,7 +109,7 @@ class Scaffolder : public CorePlugin{
map<PathHandle,int> m_contigLengths;
int m_position;
int m_theLength;
- vector<Kmer> m_contigPath;
+ GraphPath m_contigPath;
bool m_requestedContigChunk;
vector<int> m_allScaffoldLengths;
@@ -124,7 +141,7 @@ class Scaffolder : public CorePlugin{
bool m_reverseDirectionLengthReceived;
PathHandle m_pairedReverseDirectionName;
int m_pairedReverseDirectionPosition;
- int m_pairedReverseMarkerCoverage;
+ CoverageDepth m_pairedReverseMarkerCoverage;
bool m_pairedReverseHasDirection;
bool m_reverseDirectionLengthRequested;
int m_pairedReverseDirectionLength;
@@ -134,7 +151,7 @@ class Scaffolder : public CorePlugin{
int m_pairedForwardDirectionLength;
bool m_forwardDirectionsRequested;
bool m_forwardDirectionsReceived;
- int m_pairedForwardMarkerCoverage;
+ CoverageDepth m_pairedForwardMarkerCoverage;
bool m_pairedForwardHasDirection;
PathHandle m_pairedForwardDirectionName;
int m_pairedForwardDirectionPosition;
@@ -163,13 +180,13 @@ class Scaffolder : public CorePlugin{
VirtualCommunicator*m_virtualCommunicator;
bool m_coverageRequested;
bool m_coverageReceived;
- int m_receivedCoverage;
+ CoverageDepth m_receivedCoverage;
bool m_reverseDone;
bool m_forwardDone;
int m_contigId;
int m_positionOnContig;
- vector<vector<Kmer> >*m_contigs;
+ vector<GraphPath>*m_contigs;
vector<PathHandle>*m_contigNames;
Parameters*m_parameters;
@@ -178,10 +195,15 @@ class Scaffolder : public CorePlugin{
RingAllocator*m_outboxAllocator;
bool m_ready;
- /**
+/**
* gets a contig sequence by receiving several MPI messages
*/
+
+ map<PathHandle,int>*m_contigNameIndex;
+
void getContigSequence(PathHandle id);
+ void getContigSequenceFromKmers(PathHandle id);
+ void getContigSequenceFromPackedObjects(PathHandle id);
void processContig();
void processContigPosition();
void processVertex(Kmer*vertex);
@@ -198,26 +220,41 @@ class Scaffolder : public CorePlugin{
void computeStatistics(vector<int>*lengths,int minimumLength,ostream*outputStream);
void printInStream(ostream*outputStream);
+ void getCoverageOfBlockOfLife();
+
+ Rank m_rank;
+
+/*
+ * Variables for fetching remote paths.
+ */
+
+ int m_theLengthInNucleotides;
+ ostringstream m_contigPathBuffer;
public:
bool m_initialised;
- /**
+/**
* Number of ranks that have finished scaffolding
*/
int m_numberOfRanksFinished;
-
- /**
+/**
* Constructor of the scaffolder
*/
void constructor(StaticVector*outbox,StaticVector*inbox,RingAllocator*outboxAllocator,Parameters*parameters,
VirtualCommunicator*vc,SwitchMan*switchMan);
- void call_RAY_SLAVE_MODE_SCAFFOLDER();
- void setContigPaths(vector<PathHandle>*names,vector<vector<Kmer> >*paths);
+ void setContigPaths(vector<PathHandle>*names,vector<GraphPath>*paths);
void addMasterLink(SummarizedLink*link);
void solve();
void addMasterContig(PathHandle name,int length);
+
void call_RAY_MASTER_MODE_WRITE_SCAFFOLDS();
+
+ void call_RAY_SLAVE_MODE_SCAFFOLDER();
+
+ void call_RAY_MPI_TAG_GET_CONTIG_CHUNK(Message*message);
+ void call_RAY_MPI_TAG_GET_CONTIG_PACKED_CHUNK(Message*message);
+
void printFinalMessage();
void setTimePrinter(TimePrinter*a);
@@ -227,4 +264,3 @@ public:
};
#endif
-
diff --git a/code/plugin_Scaffolder/ScaffoldingAlgorithm.cpp b/code/Scaffolder/ScaffoldingAlgorithm.cpp
similarity index 94%
rename from code/plugin_Scaffolder/ScaffoldingAlgorithm.cpp
rename to code/Scaffolder/ScaffoldingAlgorithm.cpp
index d2d53e6..82db776 100644
--- a/code/plugin_Scaffolder/ScaffoldingAlgorithm.cpp
+++ b/code/Scaffolder/ScaffoldingAlgorithm.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -18,7 +18,8 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_Scaffolder/ScaffoldingAlgorithm.h>
+#include "ScaffoldingAlgorithm.h"
+
#include <queue>
#include <iostream>
#include <set>
@@ -120,13 +121,17 @@ void ScaffoldingAlgorithm::solve(
ScaffoldingEdge edge=j->second;
+ /**
+ * TODO This code is not clean. char should not be stored as a
+ * uint64_t.
+ */
vector<uint64_t> megaLink;
PathHandle leftContig=edge.getLeftContig();
- megaLink.push_back(leftContig);
+ megaLink.push_back(leftContig.getValue());
char leftStrand=edge.getLeftStrand();
megaLink.push_back(leftStrand);
PathHandle rightContig=edge.getRightContig();
- megaLink.push_back(rightContig);
+ megaLink.push_back(rightContig.getValue());
char rightStrand=edge.getRightStrand();
megaLink.push_back(rightStrand);
int average=edge.getGapSize();
@@ -140,8 +145,10 @@ void ScaffoldingAlgorithm::solve(
// create the graph
set<PathHandle> vertices;
- map<PathHandle,map<char,vector<vector<PathHandle> > > > parents;
- map<PathHandle,map<char,vector<vector<PathHandle> > > > children;
+
+ // TODO: fix this code, PathHandle should not be used to store a char
+ map<PathHandle,map<char,vector<vector<uint64_t> > > > parents;
+ map<PathHandle,map<char,vector<vector<uint64_t> > > > children;
for(int i=0;i<(int)megaLinks.size();i++){
PathHandle leftContig=megaLinks[i][0];
@@ -157,13 +164,14 @@ void ScaffoldingAlgorithm::solve(
char otherRightStrand='F';
if(rightStrand=='F')
otherRightStrand='R';
+
children[leftContig][leftStrand].push_back(megaLinks[i]);
parents[rightContig][rightStrand].push_back(megaLinks[i]);
vector<uint64_t> reverseLink;
- reverseLink.push_back(rightContig);
+ reverseLink.push_back(rightContig.getValue());
reverseLink.push_back(otherRightStrand);
- reverseLink.push_back(leftContig);
+ reverseLink.push_back(leftContig.getValue());
reverseLink.push_back(otherLeftStrand);
reverseLink.push_back(distance);
@@ -394,8 +402,8 @@ bool ScaffoldingAlgorithm::hasConflictWithEdgeAroundContig(ScaffoldingEdge*edgeT
}
void ScaffoldingAlgorithm::extractScaffolds(char state,map<PathHandle,int>*colors,PathHandle vertex,
- map<PathHandle,map<char,vector<vector<PathHandle> > > >*parents,
- map<PathHandle,map<char,vector<vector<PathHandle> > > >*children,set<int>*completedColours,
+ map<PathHandle,map<char,vector<vector<uint64_t> > > >*parents,
+ map<PathHandle,map<char,vector<vector<uint64_t> > > >*children,set<int>*completedColours,
vector<vector<PathHandle> >*scaffoldContigs,
vector<vector<char> >*scaffoldStrands,
diff --git a/code/plugin_Scaffolder/ScaffoldingAlgorithm.h b/code/Scaffolder/ScaffoldingAlgorithm.h
similarity index 86%
rename from code/plugin_Scaffolder/ScaffoldingAlgorithm.h
rename to code/Scaffolder/ScaffoldingAlgorithm.h
index d8797a3..1896307 100644
--- a/code/plugin_Scaffolder/ScaffoldingAlgorithm.h
+++ b/code/Scaffolder/ScaffoldingAlgorithm.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -21,11 +21,14 @@
#ifndef _ScaffoldingAlgorithm_h
#define _ScaffoldingAlgorithm_h
+#include "ScaffoldingEdge.h"
+#include "ScaffoldingVertex.h"
+
+#include <code/SeedingData/PathHandle.h>
+
#include <vector>
#include <set>
-#include <plugin_Scaffolder/ScaffoldingEdge.h>
#include <map>
-#include <plugin_Scaffolder/ScaffoldingVertex.h>
using namespace std;
class ScaffoldingAlgorithm{
@@ -58,8 +61,8 @@ public:
void extractScaffolds(char state,map<PathHandle,int>*colors,PathHandle vertex,
- map<PathHandle,map<char,vector<vector<PathHandle> > > >*parents,
- map<PathHandle,map<char,vector<vector<PathHandle> > > >*children,set<int>*completedColours,
+ map<PathHandle,map<char,vector<vector<uint64_t> > > >*parents,
+ map<PathHandle,map<char,vector<vector<uint64_t> > > >*children,set<int>*completedColours,
vector<vector<PathHandle> >*scaffoldContigs,
vector<vector<char> >*scaffoldStrands,
diff --git a/code/plugin_Scaffolder/ScaffoldingEdge.cpp b/code/Scaffolder/ScaffoldingEdge.cpp
similarity index 97%
rename from code/plugin_Scaffolder/ScaffoldingEdge.cpp
rename to code/Scaffolder/ScaffoldingEdge.cpp
index 15a6f38..7020f51 100644
--- a/code/plugin_Scaffolder/ScaffoldingEdge.cpp
+++ b/code/Scaffolder/ScaffoldingEdge.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,7 +19,8 @@
*/
-#include <plugin_Scaffolder/ScaffoldingEdge.h>
+#include "ScaffoldingEdge.h"
+
#include <iostream>
#include <sstream>
#include <string>
diff --git a/code/plugin_Scaffolder/ScaffoldingEdge.h b/code/Scaffolder/ScaffoldingEdge.h
similarity index 92%
rename from code/plugin_Scaffolder/ScaffoldingEdge.h
rename to code/Scaffolder/ScaffoldingEdge.h
index 5753eeb..9a1e594 100644
--- a/code/plugin_Scaffolder/ScaffoldingEdge.h
+++ b/code/Scaffolder/ScaffoldingEdge.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,8 +22,10 @@
#ifndef _ScaffoldingEdge_h
#define _ScaffoldingEdge_h
-#include <application_core/constants.h>
-#include <core/types.h>
+#include <code/Mock/constants.h>
+#include <code/SeedingData/PathHandle.h>
+
+#include <RayPlatform/core/types.h>
#include <stdint.h>
#include <fstream>
diff --git a/code/plugin_Scaffolder/ScaffoldingLink.cpp b/code/Scaffolder/ScaffoldingLink.cpp
similarity index 92%
rename from code/plugin_Scaffolder/ScaffoldingLink.cpp
rename to code/Scaffolder/ScaffoldingLink.cpp
index 5b38db6..0059946 100644
--- a/code/plugin_Scaffolder/ScaffoldingLink.cpp
+++ b/code/Scaffolder/ScaffoldingLink.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,7 +19,7 @@
*/
-#include <plugin_Scaffolder/ScaffoldingLink.h>
+#include "ScaffoldingLink.h"
ScaffoldingLink::ScaffoldingLink(){
}
diff --git a/code/plugin_Scaffolder/ScaffoldingLink.h b/code/Scaffolder/ScaffoldingLink.h
similarity index 95%
rename from code/plugin_Scaffolder/ScaffoldingLink.h
rename to code/Scaffolder/ScaffoldingLink.h
index 03671b4..eeb12ef 100644
--- a/code/plugin_Scaffolder/ScaffoldingLink.h
+++ b/code/Scaffolder/ScaffoldingLink.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
diff --git a/code/plugin_Scaffolder/ScaffoldingVertex.cpp b/code/Scaffolder/ScaffoldingVertex.cpp
similarity index 92%
rename from code/plugin_Scaffolder/ScaffoldingVertex.cpp
rename to code/Scaffolder/ScaffoldingVertex.cpp
index fd8b6ec..1fcd9db 100644
--- a/code/plugin_Scaffolder/ScaffoldingVertex.cpp
+++ b/code/Scaffolder/ScaffoldingVertex.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -18,7 +18,8 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_Scaffolder/ScaffoldingVertex.h>
+#include "ScaffoldingVertex.h"
+
#include <sstream>
#include <string>
#include <iostream>
diff --git a/code/plugin_Scaffolder/ScaffoldingVertex.h b/code/Scaffolder/ScaffoldingVertex.h
similarity index 85%
rename from code/plugin_Scaffolder/ScaffoldingVertex.h
rename to code/Scaffolder/ScaffoldingVertex.h
index 6af7b97..a0e7745 100644
--- a/code/plugin_Scaffolder/ScaffoldingVertex.h
+++ b/code/Scaffolder/ScaffoldingVertex.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -21,8 +21,12 @@
#ifndef _ScaffoldingVertex_h
#define _ScaffoldingVertex_h
-#include <application_core/constants.h>
-#include <core/types.h>
+#include <code/Mock/constants.h>
+
+#include <code/SeedingData/PathHandle.h>
+
+#include <RayPlatform/core/types.h>
+
#include <stdint.h>
#include <fstream>
using namespace std;
diff --git a/code/plugin_Scaffolder/SummarizedLink.cpp b/code/Scaffolder/SummarizedLink.cpp
similarity index 92%
rename from code/plugin_Scaffolder/SummarizedLink.cpp
rename to code/Scaffolder/SummarizedLink.cpp
index 01dea70..d2e85af 100644
--- a/code/plugin_Scaffolder/SummarizedLink.cpp
+++ b/code/Scaffolder/SummarizedLink.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,7 +19,7 @@
*/
-#include <plugin_Scaffolder/SummarizedLink.h>
+#include "SummarizedLink.h"
SummarizedLink::SummarizedLink(PathHandle leftContig,Strand leftStrand,PathHandle rightContig,
Strand rightStrand,int average,int count,
@@ -59,9 +59,9 @@ int SummarizedLink::getCount(){
}
void SummarizedLink::pack(MessageUnit*buffer,int*position){
- buffer[(*position)++]=getLeftContig();
+ buffer[(*position)++]=getLeftContig().getValue();
buffer[(*position)++]=getLeftStrand();
- buffer[(*position)++]=getRightContig();
+ buffer[(*position)++]=getRightContig().getValue();
buffer[(*position)++]=getRightStrand();
buffer[(*position)++]=getCount();
buffer[(*position)++]=getAverage();
diff --git a/code/plugin_Scaffolder/SummarizedLink.h b/code/Scaffolder/SummarizedLink.h
similarity index 89%
rename from code/plugin_Scaffolder/SummarizedLink.h
rename to code/Scaffolder/SummarizedLink.h
index 090de48..0166f31 100644
--- a/code/plugin_Scaffolder/SummarizedLink.h
+++ b/code/Scaffolder/SummarizedLink.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,9 +22,12 @@
#ifndef _SummarizedLink
#define _SummarizedLink
+#include <code/Mock/constants.h>
+#include <code/SeedingData/PathHandle.h>
+
+#include <RayPlatform/core/types.h>
+
#include <stdint.h>
-#include <application_core/constants.h>
-#include <core/types.h>
class SummarizedLink{
int m_count;
diff --git a/code/plugin_Searcher/ColorSet.cpp b/code/Searcher/ColorSet.cpp
similarity index 99%
rename from code/plugin_Searcher/ColorSet.cpp
rename to code/Searcher/ColorSet.cpp
index 915df26..6d73741 100644
--- a/code/plugin_Searcher/ColorSet.cpp
+++ b/code/Searcher/ColorSet.cpp
@@ -22,8 +22,9 @@
//#define CONFIG_DEBUG_VIRTUAL_COLORS
//#define ASSERT_LOW_LEVEL
-#include <plugin_Searcher/ColorSet.h>
-#include <cryptography/crypto.h>
+#include "ColorSet.h"
+
+#include <RayPlatform/cryptography/crypto.h>
#include <stdint.h>
#include <iostream>
diff --git a/code/plugin_Searcher/ColorSet.h b/code/Searcher/ColorSet.h
similarity index 99%
rename from code/plugin_Searcher/ColorSet.h
rename to code/Searcher/ColorSet.h
index ceda8d9..8301e18 100644
--- a/code/plugin_Searcher/ColorSet.h
+++ b/code/Searcher/ColorSet.h
@@ -22,7 +22,7 @@
#ifndef _ColorSet_h
#define _ColorSet_h
-#include <plugin_Searcher/VirtualKmerColor.h>
+#include "VirtualKmerColor.h"
#include <stdint.h>
#include <vector>
diff --git a/code/plugin_Searcher/ColoredPeakFinder.cpp b/code/Searcher/ColoredPeakFinder.cpp
similarity index 97%
rename from code/plugin_Searcher/ColoredPeakFinder.cpp
rename to code/Searcher/ColoredPeakFinder.cpp
index 03500a6..d39fc29 100644
--- a/code/plugin_Searcher/ColoredPeakFinder.cpp
+++ b/code/Searcher/ColoredPeakFinder.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -18,8 +18,10 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_Searcher/ColoredPeakFinder.h>
-#include <core/statistics.h>
+#include "ColoredPeakFinder.h"
+
+#include <RayPlatform/core/statistics.h>
+
#include <math.h>
#include <iostream>
#include <stdint.h>
diff --git a/code/plugin_Searcher/ColoredPeakFinder.h b/code/Searcher/ColoredPeakFinder.h
similarity index 96%
rename from code/plugin_Searcher/ColoredPeakFinder.h
rename to code/Searcher/ColoredPeakFinder.h
index 4f09906..562a248 100644
--- a/code/plugin_Searcher/ColoredPeakFinder.h
+++ b/code/Searcher/ColoredPeakFinder.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
diff --git a/code/plugin_Searcher/ContigHit.cpp b/code/Searcher/ContigHit.cpp
similarity index 96%
rename from code/plugin_Searcher/ContigHit.cpp
rename to code/Searcher/ContigHit.cpp
index 58a9caf..bcdc781 100644
--- a/code/plugin_Searcher/ContigHit.cpp
+++ b/code/Searcher/ContigHit.cpp
@@ -18,7 +18,7 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_Searcher/ContigHit.h>
+#include "ContigHit.h"
ContigHit::ContigHit(int sequence,PathHandle contig,char strand,int matches){
diff --git a/code/plugin_Searcher/ContigHit.h b/code/Searcher/ContigHit.h
similarity index 90%
rename from code/plugin_Searcher/ContigHit.h
rename to code/Searcher/ContigHit.h
index 5381963..041618b 100644
--- a/code/plugin_Searcher/ContigHit.h
+++ b/code/Searcher/ContigHit.h
@@ -21,8 +21,12 @@
#ifndef _ContigHit_h
#define _ContigHit_h
-#include <application_core/constants.h>
-#include <core/types.h>
+#include <code/Mock/constants.h>
+
+#include <code/SeedingData/PathHandle.h>
+
+#include <RayPlatform/core/types.h>
+
#include <stdint.h>
class ContigHit{
diff --git a/code/plugin_Searcher/ContigSearchEntry.cpp b/code/Searcher/ContigSearchEntry.cpp
similarity index 97%
rename from code/plugin_Searcher/ContigSearchEntry.cpp
rename to code/Searcher/ContigSearchEntry.cpp
index dff6f1a..cc9670b 100644
--- a/code/plugin_Searcher/ContigSearchEntry.cpp
+++ b/code/Searcher/ContigSearchEntry.cpp
@@ -18,7 +18,7 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_Searcher/ContigSearchEntry.h>
+#include "ContigSearchEntry.h"
ContigSearchEntry::ContigSearchEntry(PathHandle name,int length,int mode,double mean,int coloredKmers){
diff --git a/code/plugin_Searcher/ContigSearchEntry.h b/code/Searcher/ContigSearchEntry.h
similarity index 91%
rename from code/plugin_Searcher/ContigSearchEntry.h
rename to code/Searcher/ContigSearchEntry.h
index 037d41c..1d2afaf 100644
--- a/code/plugin_Searcher/ContigSearchEntry.h
+++ b/code/Searcher/ContigSearchEntry.h
@@ -18,8 +18,10 @@
see <http://www.gnu.org/licenses/>
*/
-#include <application_core/constants.h>
-#include <core/types.h>
+#include <code/SeedingData/PathHandle.h>
+#include <code/Mock/constants.h>
+
+#include <RayPlatform/core/types.h>
#include <stdint.h> /* for uint64_t */
#include <fstream>
diff --git a/code/plugin_Searcher/DistributionWriter.cpp b/code/Searcher/DistributionWriter.cpp
similarity index 96%
rename from code/plugin_Searcher/DistributionWriter.cpp
rename to code/Searcher/DistributionWriter.cpp
index b2d1fab..b1c7aba 100644
--- a/code/plugin_Searcher/DistributionWriter.cpp
+++ b/code/Searcher/DistributionWriter.cpp
@@ -18,9 +18,10 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_Searcher/DistributionWriter.h>
-#include <application_core/constants.h>
-#include <application_core/common_functions.h>
+#include "DistributionWriter.h"
+
+#include <code/Mock/constants.h>
+#include <code/Mock/common_functions.h>
#include <sstream>
#include <iostream>
diff --git a/code/plugin_Searcher/DistributionWriter.h b/code/Searcher/DistributionWriter.h
similarity index 94%
rename from code/plugin_Searcher/DistributionWriter.h
rename to code/Searcher/DistributionWriter.h
index 3ffe201..f614ee1 100644
--- a/code/plugin_Searcher/DistributionWriter.h
+++ b/code/Searcher/DistributionWriter.h
@@ -21,8 +21,9 @@
#ifndef _DistributionWriter_h
#define _DistributionWriter_h
-#include <application_core/constants.h>
-#include <core/types.h> /* for Rank */
+#include <code/Mock/constants.h>
+
+#include <RayPlatform/core/types.h> /* for Rank */
#include <map>
#include <stdint.h>
diff --git a/code/Searcher/Makefile b/code/Searcher/Makefile
new file mode 100644
index 0000000..afee577
--- /dev/null
+++ b/code/Searcher/Makefile
@@ -0,0 +1,11 @@
+Searcher-y += code/Searcher/Searcher.o
+Searcher-y += code/Searcher/SearchDirectory.o
+Searcher-y += code/Searcher/ContigSearchEntry.o
+Searcher-y += code/Searcher/ContigHit.o
+Searcher-y += code/Searcher/ColorSet.o
+Searcher-y += code/Searcher/VirtualKmerColor.o
+Searcher-y += code/Searcher/QualityCaller.o
+Searcher-y += code/Searcher/DistributionWriter.o
+Searcher-y += code/Searcher/ColoredPeakFinder.o
+
+obj-y += $(Searcher-y)
diff --git a/code/plugin_Searcher/QualityCaller.cpp b/code/Searcher/QualityCaller.cpp
similarity index 97%
rename from code/plugin_Searcher/QualityCaller.cpp
rename to code/Searcher/QualityCaller.cpp
index fade678..ea11a4c 100644
--- a/code/plugin_Searcher/QualityCaller.cpp
+++ b/code/Searcher/QualityCaller.cpp
@@ -18,8 +18,9 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_Searcher/QualityCaller.h>
-#include <core/statistics.h>
+#include "QualityCaller.h"
+
+#include <RayPlatform/core/statistics.h>
#include <iostream>
#include <assert.h>
diff --git a/code/plugin_Searcher/QualityCaller.h b/code/Searcher/QualityCaller.h
similarity index 94%
rename from code/plugin_Searcher/QualityCaller.h
rename to code/Searcher/QualityCaller.h
index f4e024f..76c4db8 100644
--- a/code/plugin_Searcher/QualityCaller.h
+++ b/code/Searcher/QualityCaller.h
@@ -21,8 +21,10 @@
#ifndef _QualityCaller_h
#define _QualityCaller_h
-#include <application_core/constants.h>
-#include <core/types.h>
+#include <code/Mock/constants.h>
+
+#include <RayPlatform/core/types.h>
+
#include <stdint.h>
#include <vector>
#include <map>
diff --git a/code/plugin_Searcher/SearchDirectory.cpp b/code/Searcher/SearchDirectory.cpp
similarity index 96%
rename from code/plugin_Searcher/SearchDirectory.cpp
rename to code/Searcher/SearchDirectory.cpp
index 2cb836e..2f0bb62 100644
--- a/code/plugin_Searcher/SearchDirectory.cpp
+++ b/code/Searcher/SearchDirectory.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2012 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -21,7 +21,7 @@
//#define CONFIG_SEARCH_DIR_VERBOSE
-#include <plugin_Searcher/SearchDirectory.h>
+#include "SearchDirectory.h"
#include <assert.h>
#include <string.h>
@@ -37,7 +37,7 @@
#include <iostream>
#include <fstream>
#include <sstream>
-#include <application_core/common_functions.h> /* for wordId */
+#include <code/Mock/common_functions.h> /* for wordId */
using namespace std;
@@ -99,7 +99,6 @@ int SearchDirectory::getCount(int i){
cout<<"Error, file= "<<i<<" but size= "<<m_counts.size()<<endl;
cout<<"Directory= "<<getDirectoryName()<<endl;
- cout.flush();
}
assert(i<(int)m_counts.size());
@@ -436,22 +435,22 @@ void SearchDirectory::getNextKmer(int kmerLength,Kmer*kmer){
for(int i=0;i<kmerLength;i++){
char nucleotide=sequenceBuffer[i];
switch (nucleotide){
- case 'a':
- nucleotide='A';
+ case SYMBOL_LOWER_A:
+ nucleotide=SYMBOL_A;
break;
- case 't':
- nucleotide='T';
+ case SYMBOL_LOWER_T:
+ nucleotide=SYMBOL_T;
break;
- case 'c':
- nucleotide='C';
+ case SYMBOL_LOWER_C:
+ nucleotide=SYMBOL_C;
break;
- case 'g':
- nucleotide='G';
+ case SYMBOL_LOWER_G:
+ nucleotide=SYMBOL_G;
break;
}
- if(nucleotide!='A' && nucleotide!='T' && nucleotide!='C'
- && nucleotide!='G'){
+ if(nucleotide!=SYMBOL_A && nucleotide!=SYMBOL_T && nucleotide!=SYMBOL_C
+ && nucleotide!=SYMBOL_G){
m_hasN=true;
}
diff --git a/code/plugin_Searcher/SearchDirectory.h b/code/Searcher/SearchDirectory.h
similarity index 93%
rename from code/plugin_Searcher/SearchDirectory.h
rename to code/Searcher/SearchDirectory.h
index 99d7670..3907fb6 100644
--- a/code/plugin_Searcher/SearchDirectory.h
+++ b/code/Searcher/SearchDirectory.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2012 Sébastien Boisvert
+ Copyright (C) 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,9 +22,10 @@
#ifndef _SearchDirectory_h
#define _SearchDirectory_h
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <plugin_GeneOntology/KeyEncoder.h>
-#include <plugin_Searcher/VirtualKmerColor.h>
+#include "VirtualKmerColor.h"
+
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/GeneOntology/KeyEncoder.h>
#include <set>
#include <string>
diff --git a/code/plugin_Searcher/Searcher.cpp b/code/Searcher/Searcher.cpp
similarity index 98%
rename from code/plugin_Searcher/Searcher.cpp
rename to code/Searcher/Searcher.cpp
index e6ea75d..15bbb2a 100644
--- a/code/plugin_Searcher/Searcher.cpp
+++ b/code/Searcher/Searcher.cpp
@@ -28,12 +28,14 @@
#define CONFIG_COLORED_GRAPH_DEBUG
#endif /* DEBUG_GRAPH_BROWSING */
-#include <plugin_Searcher/Searcher.h>
-#include <plugin_VerticesExtractor/Vertex.h>
-#include <core/OperatingSystem.h>
-#include <core/ComputeCore.h>
-#include <plugin_Searcher/ColoredPeakFinder.h>
-#include <plugin_VerticesExtractor/GridTableIterator.h>
+#include "Searcher.h"
+#include "ColoredPeakFinder.h"
+
+#include <code/VerticesExtractor/Vertex.h>
+#include <code/VerticesExtractor/GridTableIterator.h>
+
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <stdio.h> /* for fopen, fprintf and fclose */
#include <fstream>
@@ -173,7 +175,7 @@ void Searcher::call_RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_AND_COLORS(Message*messa
Rank rank=m_parameters->getRank();
Message aMessage(message2,count,source,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_AND_COLORS_REPLY,rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
@@ -466,7 +468,7 @@ void Searcher::call_RAY_SLAVE_MODE_COUNT_SEARCH_ELEMENTS(){
Message aMessage(buffer2,bufferSize,MASTER_RANK,
RAY_MPI_TAG_SEARCH_ELEMENTS,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_waiting=true;
@@ -527,7 +529,7 @@ void Searcher::countKmerObservations(LargeCount*localAssembledKmerObservations,
/* here, we just want to find a path with
* a good progression */
- Direction*a=node->m_directions;
+ Direction*a=node->getFirstDirection();
bool nicelyAssembled=false;
while(a!=NULL){
@@ -705,7 +707,7 @@ void Searcher::call_RAY_MASTER_MODE_CONTIG_BIOLOGICAL_ABUNDANCES(){
message->getSource(),
RAY_MPI_TAG_CONTIG_ABUNDANCE_REPLY,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}else if(m_switchMan->allRanksAreReady()){
// write the contig file
@@ -716,12 +718,12 @@ void Searcher::call_RAY_MASTER_MODE_CONTIG_BIOLOGICAL_ABUNDANCES(){
contigSummaryFile.open(summary.str().c_str(), ios_base::app);
- contigSummaryFile<<"#Contig name K-mer length Length in k-mers";
+ contigSummaryFile<<"#Contig name K-mer length Contig length in k-mers";
contigSummaryFile<<" Colored k-mers";
- contigSummaryFile<<" Proportion";
+ contigSummaryFile<<" Proportion of colored k-mers in contig";
contigSummaryFile<<" Mode k-mer coverage depth";
contigSummaryFile<<" K-mer observations Total";
- contigSummaryFile<<" Proportion"<<endl;
+ contigSummaryFile<<" Proportion of k-mer observations in sample"<<endl;
// count the total
LargeCount total=m_totalNumberOfAssembledKmerObservations;
@@ -876,12 +878,12 @@ void Searcher::call_RAY_SLAVE_MODE_CONTIG_BIOLOGICAL_ABUNDANCES(){
if(m_writeDetailedFiles){ /* now we write coverage frequencies */
#ifdef ASSERT
- assert(m_coverageValues.size() == (*m_contigs)[m_contig].size());
+ assert((int)m_coverageValues.size() == (*m_contigs)[m_contig].size());
#endif
for(int i=0;i<(int)m_coverageValues.size();i++){
int position=i+1;
- int coverage=m_coverageValues[i];
+ CoverageDepth coverage=m_coverageValues[i];
int colors=m_colorValues[i];
//*************
@@ -892,7 +894,9 @@ void Searcher::call_RAY_SLAVE_MODE_CONTIG_BIOLOGICAL_ABUNDANCES(){
// anyway the code path for getColorSpaceMode=true is not tested very well
bool coloredMode=m_parameters->getColorSpaceMode();
- Kmer*kmer=&((*m_contigs)[m_contig][i]);
+ Kmer kmer2;
+ (*m_contigs)[m_contig].at(i,&kmer2);
+ Kmer*kmer=&kmer2;
double gcRatio=kmer->getGuanineCytosineProportion(kmerLength,coloredMode);
#ifdef ASSERT
@@ -950,7 +954,7 @@ void Searcher::call_RAY_SLAVE_MODE_CONTIG_BIOLOGICAL_ABUNDANCES(){
MessageUnit*buffer=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
int bufferSize=0;
- buffer[bufferSize++]=contigName;
+ buffer[bufferSize++]=contigName.getValue();
buffer[bufferSize++]=lengthInKmers;
buffer[bufferSize++]=m_coloredKmers;
buffer[bufferSize++]=mode;
@@ -1051,9 +1055,11 @@ void Searcher::call_RAY_SLAVE_MODE_CONTIG_BIOLOGICAL_ABUNDANCES(){
#endif
// get the kmer
- Kmer*kmer=&((*m_contigs)[m_contig][m_contigPosition]);
+ Kmer kmer2;
+ (*m_contigs)[m_contig].at(m_contigPosition,&kmer2);
+ Kmer*kmer=&kmer2;
- int rankToFlush=m_parameters->_vertexRank(kmer);
+ int rankToFlush=m_parameters->vertexRank(kmer);
int period=m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_AND_COLORS);
int added=0;
@@ -1192,7 +1198,7 @@ void Searcher::call_RAY_SLAVE_MODE_CONTIG_BIOLOGICAL_ABUNDANCES(){
}
}
-void Searcher::setContigs(vector<vector<Kmer> >*paths,vector<PathHandle>*names){
+void Searcher::setContigs(vector<GraphPath>*paths,vector<PathHandle>*names){
m_contigs=paths;
m_contigNames=names;
}
@@ -1410,7 +1416,7 @@ void Searcher::browseColoredGraph(){
Message aMessage(messageBuffer,position,MASTER_RANK,
RAY_MPI_TAG_VIRTUAL_COLOR_DATA,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_messageSent=true;
m_messageReceived=false;
@@ -1646,7 +1652,7 @@ void Searcher::call_RAY_SLAVE_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES(){
MessageUnit*messageBuffer=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
int bufferPosition=0;
- messageBuffer[bufferPosition++]=contig;
+ messageBuffer[bufferPosition++]=contig.getValue();
messageBuffer[bufferPosition++]=strand;
messageBuffer[bufferPosition++]=count;
messageBuffer[bufferPosition++]=m_directoryIterator;
@@ -1676,7 +1682,7 @@ void Searcher::call_RAY_SLAVE_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES(){
Message aMessage(messageBuffer,MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit),
getWriter(m_directoryIterator),RAY_MPI_TAG_CONTIG_IDENTIFICATION,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_pendingMessages++;
@@ -1929,7 +1935,6 @@ void Searcher::call_RAY_SLAVE_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES(){
#endif
//cout<<"contig-"<<contig<<" has "<<matches<<" matches on strand 'F'"<<endl;
- //cout.flush();
}
for(map<PathHandle,set<int> >::iterator i=m_contigCounts['R'].begin();i!=m_contigCounts['R'].end();i++){
@@ -1945,7 +1950,6 @@ void Searcher::call_RAY_SLAVE_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES(){
#endif
//cout<<"contig-"<<contig<<" has "<<matches<<" matches on strand 'R'"<<endl;
- //cout.flush();
}
// send a message to write the abundances
@@ -2003,7 +2007,7 @@ void Searcher::call_RAY_SLAVE_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES(){
writer,RAY_MPI_TAG_WRITE_SEQUENCE_ABUNDANCE_ENTRY,
m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
#ifdef ASSERT
assert(m_directoryIterator==(int)buffer[0]);
@@ -2124,7 +2128,7 @@ void Searcher::call_RAY_SLAVE_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES(){
continue;
}
- int rankToFlush=m_parameters->_vertexRank(&kmer);
+ int rankToFlush=m_parameters->vertexRank(&kmer);
int added=0;
// pack the k-mer
@@ -2366,7 +2370,7 @@ void Searcher::call_RAY_SLAVE_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES(){
if(m_numberOfKmers%1000==0)
cout<<"Received coverage position = "<<m_numberOfKmers<<" val= "<<coverage<<endl;
#endif
-
+
if(false && m_numberOfKmers%10000==0 && m_numberOfKmers > 0){
cout<<"Rank "<<m_parameters->getRank()<<" processing sequence "<<m_globalSequenceIterator;
cout<<" ProcessedKmers= "<<m_numberOfKmers<<endl;
@@ -2378,12 +2382,12 @@ void Searcher::call_RAY_SLAVE_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES(){
/*
Message aMessage(message2,count,source,RAY_MPI_TAG_GET_COVERAGE_AND_PATHS_REPLY,
m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
*/
}else{
m_requestedCoverage=false;
-
+
// clear observed contigs for future sequence positions
m_observedPaths.clear();
}
@@ -2799,7 +2803,7 @@ void Searcher::call_RAY_MPI_TAG_GET_COVERAGE_AND_PATHS(Message*message){
if(node!=NULL){
- Kmer lowerKey=node->m_lowerKey;
+ Kmer lowerKey=node->getKey();
bool weHaveLowerKey=lowerKey.isEqual(&vertex);
@@ -2808,7 +2812,7 @@ void Searcher::call_RAY_MPI_TAG_GET_COVERAGE_AND_PATHS(Message*message){
/* here, we just want to find a path with
* a good progression */
- Direction*a=node->m_directions;
+ Direction*a=node->getFirstDirection();
bool nicelyAssembled=false;
@@ -2857,7 +2861,7 @@ void Searcher::call_RAY_MPI_TAG_GET_COVERAGE_AND_PATHS(Message*message){
#endif
//store the contig path
- message2[outputBufferPosition++]=path; // path identifier
+ message2[outputBufferPosition++]=path.getValue(); // path identifier
message2[outputBufferPosition++]=contigPosition; // path position
message2[outputBufferPosition++]=strand;
@@ -2888,7 +2892,7 @@ void Searcher::call_RAY_MPI_TAG_GET_COVERAGE_AND_PATHS(Message*message){
Message aMessage(message2,count,source,RAY_MPI_TAG_GET_COVERAGE_AND_PATHS_REPLY,
m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void Searcher::call_RAY_MPI_TAG_ADD_KMER_COLOR(Message*message){
@@ -3380,7 +3384,7 @@ void Searcher::call_RAY_SLAVE_MODE_ADD_COLORS(){
continue;
}
- int rankToFlush=m_parameters->_vertexRank(&kmer);
+ Rank rankToFlush=m_parameters->vertexRank(&kmer);
int added=0;
// pack the k-mer
@@ -4012,7 +4016,9 @@ void Searcher::generateSummaryOfColoredDeBruijnGraph(){
if(references==0)
continue;
+ #ifdef ASSERT
int numberOfPhysicalColors=m_masterColorSet.getNumberOfPhysicalColors(currentVirtualColor);
+ #endif
set<PhysicalKmerColor>*colors=m_masterColorSet.getPhysicalColors(currentVirtualColor);
@@ -4275,7 +4281,7 @@ void Searcher::registerPlugin(ComputeCore*core){
core->setObjectSymbol(m_plugin,m_subgraph,"/RayAssembler/ObjectStore/deBruijnGraph_part.ray");
core->setObjectSymbol(m_plugin,&m_colorSet,"/RayAssembler/ObjectStore/VirtualColorManagementUnit.ray");
core->setObjectSymbol(m_plugin,m_timePrinter,"/RayAssembler/ObjectStore/Timer.ray");
- core->setObjectSymbol(m_plugin,this,"/RayAssembler/ObjectStore/plugin_Searcher.ray");
+ core->setObjectSymbol(m_plugin,this,"/RayAssembler/ObjectStore/Searcher.ray");
core->setObjectSymbol(m_plugin,&m_contigLengths,"/RayAssembler/ObjectStore/ContigLengths.ray");
m_totalNumberOfColoredKmerObservations=0;
@@ -4397,4 +4403,23 @@ void Searcher::resolveSymbols(ComputeCore*core){
m_useOneColorPerFile=m_parameters->hasOption("-one-color-per-file");
__BindPlugin(Searcher);
+
+ __BindAdapter(Searcher,RAY_MASTER_MODE_COUNT_SEARCH_ELEMENTS);
+ __BindAdapter(Searcher,RAY_MASTER_MODE_CONTIG_BIOLOGICAL_ABUNDANCES);
+ __BindAdapter(Searcher,RAY_MASTER_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES);
+ __BindAdapter(Searcher,RAY_MASTER_MODE_ADD_COLORS);
+ __BindAdapter(Searcher,RAY_MASTER_MODE_SEARCHER_CLOSE);
+ __BindAdapter(Searcher,RAY_SLAVE_MODE_COUNT_SEARCH_ELEMENTS);
+ __BindAdapter(Searcher,RAY_SLAVE_MODE_CONTIG_BIOLOGICAL_ABUNDANCES);
+ __BindAdapter(Searcher,RAY_SLAVE_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES);
+ __BindAdapter(Searcher,RAY_SLAVE_MODE_ADD_COLORS);
+ __BindAdapter(Searcher,RAY_SLAVE_MODE_SEARCHER_CLOSE);
+ __BindAdapter(Searcher,RAY_MPI_TAG_ADD_KMER_COLOR);
+ __BindAdapter(Searcher,RAY_MPI_TAG_CONTIG_IDENTIFICATION);
+ __BindAdapter(Searcher,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_AND_COLORS);
+ __BindAdapter(Searcher,RAY_MPI_TAG_GET_COVERAGE_AND_PATHS);
+ __BindAdapter(Searcher,RAY_MPI_TAG_WRITE_SEQUENCE_ABUNDANCE_ENTRY);
+ __BindAdapter(Searcher,RAY_MPI_TAG_GRAPH_COUNTS);
+ __BindAdapter(Searcher,RAY_MPI_TAG_VIRTUAL_COLOR_DATA);
+ __BindAdapter(Searcher,RAY_MPI_TAG_VIRTUAL_COLOR_DATA_REPLY);
}
diff --git a/code/plugin_Searcher/Searcher.h b/code/Searcher/Searcher.h
similarity index 76%
rename from code/plugin_Searcher/Searcher.h
rename to code/Searcher/Searcher.h
index fa69383..f4740ca 100644
--- a/code/plugin_Searcher/Searcher.h
+++ b/code/Searcher/Searcher.h
@@ -14,13 +14,16 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
#ifndef _Searcher_h
#define _Searcher_h
+#include <code/VerticesExtractor/GridTable.h>
+#include <code/SeedingData/GraphPath.h>
+
#include <stdint.h>
#include <fstream>
#include <stdint.h> /* for uint64_t */
@@ -29,39 +32,87 @@
#include <string>
#include <map>
#include <sstream>
-#include <plugin_VerticesExtractor/GridTable.h>
using namespace std;
-#include <application_core/Parameters.h>
-#include <scheduling/SwitchMan.h>
-#include <communication/VirtualCommunicator.h>
-#include <communication/BufferedData.h>
-#include <memory/RingAllocator.h>
-#include <plugin_Searcher/SearchDirectory.h>
-#include <plugin_Searcher/ContigSearchEntry.h>
-#include <structures/StaticVector.h>
-#include <profiling/TimePrinter.h>
-#include <profiling/Derivative.h>
-#include <handlers/SlaveModeHandler.h>
-#include <handlers/MasterModeHandler.h>
-#include <handlers/MessageTagHandler.h>
-#include <plugin_Searcher/ContigHit.h>
-#include <plugin_Searcher/ColorSet.h>
-#include <plugins/CorePlugin.h>
-#include <plugin_Searcher/QualityCaller.h>
-#include <plugin_Searcher/DistributionWriter.h>
-#include <core/ComputeCore.h>
-
+#include <code/Mock/Parameters.h>
+#include <RayPlatform/scheduling/SwitchMan.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/communication/BufferedData.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <code/Searcher/SearchDirectory.h>
+#include <code/Searcher/ContigSearchEntry.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/profiling/TimePrinter.h>
+#include <RayPlatform/profiling/Derivative.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/handlers/MasterModeHandler.h>
+#include <RayPlatform/handlers/MessageTagHandler.h>
+#include <code/Searcher/ContigHit.h>
+#include <code/Searcher/ColorSet.h>
+#include <RayPlatform/plugins/CorePlugin.h>
+#include <code/Searcher/QualityCaller.h>
+#include <code/Searcher/DistributionWriter.h>
+#include <RayPlatform/core/ComputeCore.h>
#define CONFIG_NICELY_ASSEMBLED_KMER_POSITION 0
+__DeclarePlugin(Searcher);
+
+__DeclareMasterModeAdapter(Searcher,RAY_MASTER_MODE_COUNT_SEARCH_ELEMENTS);
+__DeclareMasterModeAdapter(Searcher,RAY_MASTER_MODE_CONTIG_BIOLOGICAL_ABUNDANCES);
+__DeclareMasterModeAdapter(Searcher,RAY_MASTER_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES);
+__DeclareMasterModeAdapter(Searcher,RAY_MASTER_MODE_ADD_COLORS);
+__DeclareMasterModeAdapter(Searcher,RAY_MASTER_MODE_SEARCHER_CLOSE);
+
+__DeclareSlaveModeAdapter(Searcher,RAY_SLAVE_MODE_COUNT_SEARCH_ELEMENTS);
+__DeclareSlaveModeAdapter(Searcher,RAY_SLAVE_MODE_CONTIG_BIOLOGICAL_ABUNDANCES);
+__DeclareSlaveModeAdapter(Searcher,RAY_SLAVE_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES);
+__DeclareSlaveModeAdapter(Searcher,RAY_SLAVE_MODE_ADD_COLORS);
+__DeclareSlaveModeAdapter(Searcher,RAY_SLAVE_MODE_SEARCHER_CLOSE);
+
+__DeclareMessageTagAdapter(Searcher,RAY_MPI_TAG_ADD_KMER_COLOR);
+__DeclareMessageTagAdapter(Searcher,RAY_MPI_TAG_CONTIG_IDENTIFICATION);
+__DeclareMessageTagAdapter(Searcher,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_AND_COLORS);
+__DeclareMessageTagAdapter(Searcher,RAY_MPI_TAG_GET_COVERAGE_AND_PATHS);
+__DeclareMessageTagAdapter(Searcher,RAY_MPI_TAG_WRITE_SEQUENCE_ABUNDANCE_ENTRY);
+__DeclareMessageTagAdapter(Searcher,RAY_MPI_TAG_GRAPH_COUNTS);
+__DeclareMessageTagAdapter(Searcher,RAY_MPI_TAG_VIRTUAL_COLOR_DATA);
+__DeclareMessageTagAdapter(Searcher,RAY_MPI_TAG_VIRTUAL_COLOR_DATA_REPLY);
+
/**
* This class searches for sequences in the de Bruijn graph
* It outputs biological abundance readouts
+ *
+ * This class is part of the "Ray Communities" workflow.
+ *
+ * Methods in this class are also used for the "Ray Ontology"
+ * workflow.
+ *
* \author Sébastien Boisvert
**/
class Searcher : public CorePlugin {
+ __AddAdapter(Searcher,RAY_MASTER_MODE_COUNT_SEARCH_ELEMENTS);
+ __AddAdapter(Searcher,RAY_MASTER_MODE_CONTIG_BIOLOGICAL_ABUNDANCES);
+ __AddAdapter(Searcher,RAY_MASTER_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES);
+ __AddAdapter(Searcher,RAY_MASTER_MODE_ADD_COLORS);
+ __AddAdapter(Searcher,RAY_MASTER_MODE_SEARCHER_CLOSE);
+
+ __AddAdapter(Searcher,RAY_SLAVE_MODE_COUNT_SEARCH_ELEMENTS);
+ __AddAdapter(Searcher,RAY_SLAVE_MODE_CONTIG_BIOLOGICAL_ABUNDANCES);
+ __AddAdapter(Searcher,RAY_SLAVE_MODE_SEQUENCE_BIOLOGICAL_ABUNDANCES);
+ __AddAdapter(Searcher,RAY_SLAVE_MODE_ADD_COLORS);
+ __AddAdapter(Searcher,RAY_SLAVE_MODE_SEARCHER_CLOSE);
+
+ __AddAdapter(Searcher,RAY_MPI_TAG_ADD_KMER_COLOR);
+ __AddAdapter(Searcher,RAY_MPI_TAG_CONTIG_IDENTIFICATION);
+ __AddAdapter(Searcher,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_AND_COLORS);
+ __AddAdapter(Searcher,RAY_MPI_TAG_GET_COVERAGE_AND_PATHS);
+ __AddAdapter(Searcher,RAY_MPI_TAG_WRITE_SEQUENCE_ABUNDANCE_ENTRY);
+ __AddAdapter(Searcher,RAY_MPI_TAG_GRAPH_COUNTS);
+ __AddAdapter(Searcher,RAY_MPI_TAG_VIRTUAL_COLOR_DATA);
+ __AddAdapter(Searcher,RAY_MPI_TAG_VIRTUAL_COLOR_DATA_REPLY);
+
/* the number of colored k-mers for a contig */
int m_coloredKmers;
@@ -255,7 +306,7 @@ class Searcher : public CorePlugin {
bool m_writeDetailedFiles;
/** contig paths */
- vector<vector<Kmer> >*m_contigs;
+ vector<GraphPath>*m_contigs;
vector<PathHandle>*m_contigNames;
// synchronization
@@ -438,7 +489,7 @@ public:
VirtualCommunicator*m_vc,StaticVector*inbox,RingAllocator*outboxAllocator,
GridTable*graph);
- void setContigs(vector<vector<Kmer> >*paths,vector<PathHandle>*names);
+ void setContigs(vector<GraphPath>*paths,vector<PathHandle>*names);
void call_RAY_MPI_TAG_GET_COVERAGE_AND_PATHS(Message*message);
void call_RAY_MPI_TAG_WRITE_SEQUENCE_ABUNDANCE_ENTRY(Message*message);
diff --git a/code/plugin_Searcher/VirtualKmerColor.cpp b/code/Searcher/VirtualKmerColor.cpp
similarity index 97%
rename from code/plugin_Searcher/VirtualKmerColor.cpp
rename to code/Searcher/VirtualKmerColor.cpp
index 8079e1f..0c28240 100644
--- a/code/plugin_Searcher/VirtualKmerColor.cpp
+++ b/code/Searcher/VirtualKmerColor.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,7 +19,8 @@
*/
-#include <plugin_Searcher/VirtualKmerColor.h>
+#include "VirtualKmerColor.h"
+
#include <iostream>
using namespace std;
#ifdef ASSERT
diff --git a/code/plugin_Searcher/VirtualKmerColor.h b/code/Searcher/VirtualKmerColor.h
similarity index 95%
rename from code/plugin_Searcher/VirtualKmerColor.h
rename to code/Searcher/VirtualKmerColor.h
index 95ee96e..d11a914 100644
--- a/code/plugin_Searcher/VirtualKmerColor.h
+++ b/code/Searcher/VirtualKmerColor.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -23,7 +23,8 @@
#ifndef _VirtualKmerColor_h
#define _VirtualKmerColor_h
-#include <application_core/constants.h>
+#include <code/Mock/constants.h>
+
#include <stdint.h>
#include <vector>
#include <set>
diff --git a/code/plugin_SeedExtender/BubbleData.h b/code/SeedExtender/BubbleData.h
similarity index 86%
rename from code/plugin_SeedExtender/BubbleData.h
rename to code/SeedExtender/BubbleData.h
index feb0f9e..b713740 100644
--- a/code/plugin_SeedExtender/BubbleData.h
+++ b/code/SeedExtender/BubbleData.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,12 +22,13 @@
#ifndef _BubbleData
#define _BubbleData
-#include<vector>
-#include<map>
-#include<application_core/common_functions.h>
-#include<stdio.h>
-#include<stdlib.h>
-#include<set>
+#include <code/Mock/common_functions.h>
+
+#include <vector>
+#include <map>
+#include <stdio.h>
+#include <stdlib.h>
+#include <set>
using namespace std;
/*
diff --git a/code/plugin_SeedExtender/BubbleTool.cpp b/code/SeedExtender/BubbleTool.cpp
similarity index 97%
rename from code/plugin_SeedExtender/BubbleTool.cpp
rename to code/SeedExtender/BubbleTool.cpp
index a2bc5e7..f2cf6d9 100644
--- a/code/plugin_SeedExtender/BubbleTool.cpp
+++ b/code/SeedExtender/BubbleTool.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,8 +19,10 @@
*/
-#include <plugin_SeedExtender/BubbleTool.h>
-#include <application_core/common_functions.h>
+#include "BubbleTool.h"
+
+#include <code/Mock/common_functions.h>
+
#include <assert.h>
#include <map>
#include <set>
diff --git a/code/plugin_SeedExtender/BubbleTool.h b/code/SeedExtender/BubbleTool.h
similarity index 87%
rename from code/plugin_SeedExtender/BubbleTool.h
rename to code/SeedExtender/BubbleTool.h
index b2850e7..1b259c2 100644
--- a/code/plugin_SeedExtender/BubbleTool.h
+++ b/code/SeedExtender/BubbleTool.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,11 +22,12 @@
#ifndef _BubbleTool
#define _BubbleTool
-#include <vector>
-#include <application_core/common_functions.h>
+#include <code/Mock/common_functions.h>
+#include <code/Mock/Parameters.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+
#include <map>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <application_core/Parameters.h>
+#include <vector>
using namespace std;
/*
diff --git a/code/plugin_SeedExtender/Chooser.cpp b/code/SeedExtender/Chooser.cpp
similarity index 94%
rename from code/plugin_SeedExtender/Chooser.cpp
rename to code/SeedExtender/Chooser.cpp
index 3420d88..affd24d 100644
--- a/code/plugin_SeedExtender/Chooser.cpp
+++ b/code/SeedExtender/Chooser.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,8 +19,9 @@
*/
-#include <plugin_SeedExtender/Chooser.h>
-#include <application_core/common_functions.h>
+#include "Chooser.h"
+
+#include <code/Mock/common_functions.h>
void Chooser::chooseWithPairedReads(ExtensionData*m_ed,
double __PAIRED_MULTIPLIER,
diff --git a/code/plugin_SeedExtender/Chooser.h b/code/SeedExtender/Chooser.h
similarity index 88%
rename from code/plugin_SeedExtender/Chooser.h
rename to code/SeedExtender/Chooser.h
index f650b5c..2ed8550 100644
--- a/code/plugin_SeedExtender/Chooser.h
+++ b/code/SeedExtender/Chooser.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,8 +22,9 @@
#ifndef _Chooser
#define _Chooser
-#include <plugin_SeedExtender/ExtensionData.h>
-#include <application_core/Parameters.h>
+#include "ExtensionData.h"
+
+#include <code/Mock/Parameters.h>
#define IMPOSSIBLE_CHOICE -1
diff --git a/code/plugin_SeedExtender/DepthFirstSearchData.cpp b/code/SeedExtender/DepthFirstSearchData.cpp
similarity index 95%
rename from code/plugin_SeedExtender/DepthFirstSearchData.cpp
rename to code/SeedExtender/DepthFirstSearchData.cpp
index ca111fc..8f0167d 100644
--- a/code/plugin_SeedExtender/DepthFirstSearchData.cpp
+++ b/code/SeedExtender/DepthFirstSearchData.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,8 +19,10 @@
*/
-#include <application_core/constants.h>
-#include <plugin_SeedExtender/DepthFirstSearchData.h>
+#include "DepthFirstSearchData.h"
+
+#include <code/Mock/constants.h>
+#include <code/SeedingData/SeedingData.h>
/*
* do a depth first search with max depth of maxDepth;
@@ -69,10 +71,10 @@ void DepthFirstSearchData::depthFirstSearch(Kmer root,Kmer a,int maxDepth,
MessageUnit*message=(MessageUnit*)(*outboxAllocator).allocate(KMER_U64_ARRAY_SIZE*sizeof(MessageUnit));
int j=0;
vertexToVisit.pack(message,&j);
- int dest=parameters->_vertexRank(&vertexToVisit);
+ int dest=parameters->vertexRank(&vertexToVisit);
Message aMessage(message,j,dest,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE,theRank);
- (*outbox).push_back(aMessage);
+ (*outbox).push_back(&aMessage);
}else if((*vertexCoverageReceived)){
if(!(*edgesRequested)){
m_coverages[vertexToVisit]=(*receivedVertexCoverage);
@@ -87,9 +89,9 @@ void DepthFirstSearchData::depthFirstSearch(Kmer root,Kmer a,int maxDepth,
MessageUnit*message=(MessageUnit*)(*outboxAllocator).allocate(1*sizeof(MessageUnit));
int bufferPosition=0;
vertexToVisit.pack(message,&bufferPosition);
- int destination=parameters->_vertexRank(&vertexToVisit);
+ int destination=parameters->vertexRank(&vertexToVisit);
Message aMessage(message,bufferPosition,destination,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES,theRank);
- (*outbox).push_back(aMessage);
+ (*outbox).push_back(&aMessage);
(*edgesRequested)=true;
(*edgesReceived)=false;
}else if((*edgesReceived)){
@@ -194,9 +196,9 @@ void DepthFirstSearchData::depthFirstSearchBidirectional(Kmer a,int maxDepth,
MessageUnit*message=(MessageUnit*)(*outboxAllocator).allocate(KMER_U64_ARRAY_SIZE*sizeof(MessageUnit));
int bufferPosition=0;
vertexToVisit.pack(message,&bufferPosition);
- int dest=parameters->_vertexRank(&vertexToVisit);
+ int dest=parameters->vertexRank(&vertexToVisit);
Message aMessage(message,bufferPosition,dest,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE,theRank);
- (*outbox).push_back(aMessage);
+ (*outbox).push_back(&aMessage);
}else if((*vertexCoverageReceived)){
if(!(*edgesRequested)){
m_coverages[vertexToVisit]=(*receivedVertexCoverage);
@@ -232,10 +234,10 @@ void DepthFirstSearchData::depthFirstSearchBidirectional(Kmer a,int maxDepth,
MessageUnit*message=(MessageUnit*)(*outboxAllocator).allocate(1*sizeof(MessageUnit));
int bufferPosition=0;
vertexToVisit.pack(message,&bufferPosition);
- int destination=parameters->_vertexRank(&vertexToVisit);
+ int destination=parameters->vertexRank(&vertexToVisit);
Message aMessage(message,bufferPosition,destination,RAY_MPI_TAG_REQUEST_VERTEX_EDGES,theRank);
- (*outbox).push_back(aMessage);
+ (*outbox).push_back(&aMessage);
(*edgesRequested)=true;
(*edgesReceived)=false;
}else if((*edgesReceived)){
diff --git a/code/plugin_SeedExtender/DepthFirstSearchData.h b/code/SeedExtender/DepthFirstSearchData.h
similarity index 88%
rename from code/plugin_SeedExtender/DepthFirstSearchData.h
rename to code/SeedExtender/DepthFirstSearchData.h
index e9c792f..8b65051 100644
--- a/code/plugin_SeedExtender/DepthFirstSearchData.h
+++ b/code/SeedExtender/DepthFirstSearchData.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,17 +22,21 @@
#ifndef _DepthFirstSearchData
#define _DepthFirstSearchData
+#include <code/Mock/Parameters.h>
+#include <code/Mock/common_functions.h>
+#include <code/SeedingData/SeedingData.h>
+
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/structures/MyStack.h>
+
#include <vector>
-#include <structures/MyStack.h>
#include <map>
-#include <plugin_SeedingData/SeedingData.h>
#include <set>
-#include <application_core/Parameters.h>
-#include <memory/RingAllocator.h>
-#include <structures/StaticVector.h>
-#include <application_core/common_functions.h>
using namespace std;
+class SeedingData;
+
/*
* Data for depth first search.
*
diff --git a/code/plugin_SeedExtender/Direction.cpp b/code/SeedExtender/Direction.cpp
similarity index 85%
rename from code/plugin_SeedExtender/Direction.cpp
rename to code/SeedExtender/Direction.cpp
index 82cbe0e..934e7fd 100644
--- a/code/plugin_SeedExtender/Direction.cpp
+++ b/code/SeedExtender/Direction.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,7 +19,8 @@
*/
-#include <plugin_SeedExtender/Direction.h>
+#include "Direction.h"
+
#include <stdlib.h>
void Direction::constructor(PathHandle wave,int progression,bool lower){
@@ -57,3 +58,11 @@ void Direction::setNext(Direction*e){
bool Direction::isLower(){
return m_lower;
}
+
+PathHandle Direction::getPathHandle() {
+ return getWave();
+}
+
+int Direction::getPosition() {
+ return getProgression();
+}
diff --git a/code/plugin_SeedExtender/Direction.h b/code/SeedExtender/Direction.h
similarity index 87%
rename from code/plugin_SeedExtender/Direction.h
rename to code/SeedExtender/Direction.h
index 3403ba1..1cee00e 100644
--- a/code/plugin_SeedExtender/Direction.h
+++ b/code/SeedExtender/Direction.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,14 +22,17 @@
#ifndef _Direction
#define _Direction
+#include <code/SeedingData/PathHandle.h>
+#include <code/Mock/common_functions.h>
+
#include <stdint.h>
-#include <application_core/common_functions.h>
/*
* a direction is a wave and a progression.
*
* A wave is the flow given by the Parallel_Ray_Engine in the graph.
* directions of the flow are stored with Directions. (as linked lists).
+ *
* \author Sébastien Boisvert
*/
class Direction{
@@ -44,6 +47,9 @@ public:
Direction*getNext();
void setNext(Direction*e);
bool isLower();
+ PathHandle getPathHandle();
+ int getPosition();
+
} ATTRIBUTE_PACKED;
#endif
diff --git a/code/plugin_SeedExtender/ExtensionData.cpp b/code/SeedExtender/ExtensionData.cpp
similarity index 82%
rename from code/plugin_SeedExtender/ExtensionData.cpp
rename to code/SeedExtender/ExtensionData.cpp
index bf22f4f..8c8e60d 100644
--- a/code/plugin_SeedExtender/ExtensionData.cpp
+++ b/code/SeedExtender/ExtensionData.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,11 +19,13 @@
*/
-#include <structures/SplayTreeIterator.h>
-#include <plugin_SeedExtender/ExtensionData.h>
+#include "ExtensionData.h"
+
+#include <RayPlatform/structures/SplayTreeIterator.h>
+#include <RayPlatform/cryptography/crypto.h>
+
#include <string.h>
#include <sstream>
-#include <cryptography/crypto.h>
using namespace std;
void ExtensionData::constructor(Parameters*parameters){
@@ -67,13 +69,10 @@ void ExtensionData::destroyStructures(Profiler*m_profiler){
MACRO_COLLECT_PROFILING_INFORMATION();
- //cout<<"m_pairedReadsWithoutMate= "<<m_pairedReadsWithoutMate.size()<<endl;
m_pairedReadsWithoutMate.clear();
MACRO_COLLECT_PROFILING_INFORMATION();
- //cout<<"m_expirations.size= "<<m_expirations.size()<<endl;
-
m_expirations.clear();
MACRO_COLLECT_PROFILING_INFORMATION();
@@ -103,9 +102,9 @@ void ExtensionData::destructor(){
}
ExtensionElement*ExtensionData::getUsedRead(ReadHandle a){
- int bin=0;//uniform_hashing_function_1_64_64(a)%m_numberOfBins;
- SplayNode<ReadHandle,ExtensionElement>*node=m_database[bin].find(a,true);
- if(node!=NULL && node->getValue()->m_activated){
+ int bin=0;
+ SplayNode<ReadHandle,ExtensionElement>*node=m_database[bin].find(a,false);
+ if(node!=NULL && node->getValue()->isActive()){
return node->getValue();
}
return NULL;
@@ -114,14 +113,35 @@ ExtensionElement*ExtensionData::getUsedRead(ReadHandle a){
ExtensionElement*ExtensionData::addUsedRead(ReadHandle a){
bool val;
int bin=0;
+
+ SplayNode<ReadHandle,ExtensionElement>*node=m_database[bin].find(a,false);
+
+ if(node!=NULL){
+ ExtensionElement*element=node->getValue();
+ element->activate();
+ element->increaseNumberOfTries();
+
+ return element;
+ }
+
ExtensionElement*element=m_database[bin].insert(a,&m_allocator,&val)->getValue();
element->constructor();
- element->m_activated=true;
+ element->activate();
+ element->increaseNumberOfTries();
return element;
}
void ExtensionData::removeSequence(ReadHandle a){
int bin=0;
+
+ SplayNode<ReadHandle,ExtensionElement>*node=m_database[bin].find(a,false);
+
+ if(node!=NULL){
+ node->getValue()->deactivate();
+ return;
+ }
+
+// the line below is useless
m_database[bin].remove(a,false,&m_allocator);
}
@@ -134,6 +154,6 @@ void ExtensionData::lazyDestructor(){
i.constructor(&(m_database[0]));
while(i.hasNext()){
ExtensionElement*element=i.next()->getValue();
- element->m_activated=false;
+ element->deactivate();
}
}
diff --git a/code/plugin_SeedExtender/ExtensionData.h b/code/SeedExtender/ExtensionData.h
similarity index 81%
rename from code/plugin_SeedExtender/ExtensionData.h
rename to code/SeedExtender/ExtensionData.h
index 1695f5a..4246952 100644
--- a/code/plugin_SeedExtender/ExtensionData.h
+++ b/code/SeedExtender/ExtensionData.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,15 +22,19 @@
#ifndef _ExtensionData
#define _ExtensionData
-#include <plugin_SequencesIndexer/PairedRead.h>
-#include <plugin_SequencesLoader/Read.h>
-#include <application_core/Parameters.h>
-#include <plugin_SeedExtender/ExtensionElement.h>
-#include <plugin_SeedingData/AssemblySeed.h>
-#include <profiling/Profiler.h>
-#include <plugin_SequencesIndexer/ReadAnnotation.h>
-#include <application_core/common_functions.h>
-#include <structures/SplayTree.h>
+#include "ExtensionElement.h"
+
+#include <code/SequencesLoader/ReadHandle.h>
+#include <code/SequencesIndexer/PairedRead.h>
+#include <code/SequencesIndexer/ReadAnnotation.h>
+#include <code/SequencesLoader/Read.h>
+#include <code/Mock/Parameters.h>
+#include <code/Mock/common_functions.h>
+#include <code/SeedingData/GraphPath.h>
+
+#include <RayPlatform/structures/SplayTree.h>
+#include <RayPlatform/profiling/Profiler.h>
+
#include <map>
#include <set>
#include <vector>
@@ -38,6 +42,11 @@ using namespace std;
/*
* Stores the information for a seed extension.
+ *
+ * This class is really just a helper for SeedExtender.
+ * There are so many things to store to implement the heuristics
+ * correctly.
+ *
* \author Sébastien Boisvert
*/
class ExtensionData{
@@ -52,10 +61,11 @@ class ExtensionData{
public:
int m_readType;
- vector<int> m_EXTENSION_coverages;
- vector<Kmer > m_EXTENSION_extension;
- vector<int> m_extensionCoverageValues;
+ vector<CoverageDepth> m_EXTENSION_coverages;
+ GraphPath m_EXTENSION_extension;
+ vector<CoverageDepth> m_extensionCoverageValues;
vector<int> m_repeatedValues;
+
// EXTENSION MODE
vector<Kmer> m_enumerateChoices_outgoingEdges;
bool m_doChoice_tips_Detected;
@@ -78,10 +88,10 @@ public:
int m_flowNumber;
/** TODO could be a pointer to the original thing... */
- AssemblySeed m_EXTENSION_currentSeed;
+ GraphPath m_EXTENSION_currentSeed;
int m_EXTENSION_numberOfRanksDone;
- vector<vector<Kmer > > m_EXTENSION_contigs;
+ vector<GraphPath> m_EXTENSION_contigs;
bool m_EXTENSION_checkedIfCurrentVertexIsAssembled;
bool m_EXTENSION_VertexMarkAssembled_requested;
bool m_EXTENSION_reverseComplement_requested;
diff --git a/code/plugin_SeedExtender/ExtensionElement.cpp b/code/SeedExtender/ExtensionElement.cpp
similarity index 78%
rename from code/plugin_SeedExtender/ExtensionElement.cpp
rename to code/SeedExtender/ExtensionElement.cpp
index e68708e..1552dc5 100644
--- a/code/plugin_SeedExtender/ExtensionElement.cpp
+++ b/code/SeedExtender/ExtensionElement.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,8 +19,10 @@
*/
-#include <plugin_SequencesLoader/Read.h>
-#include <plugin_SeedExtender/ExtensionElement.h>
+#include "ExtensionElement.h"
+
+#include <code/SequencesLoader/Read.h>
+
#include <string.h>
/** set the sequence */
@@ -92,10 +94,23 @@ void ExtensionElement::removeSequence(){
void ExtensionElement::constructor(){
m_canMove=true;
-
+ m_numberOfTries=0;
+ m_activated=false;
m_agreement=0;
}
+void ExtensionElement::increaseNumberOfTries(){
+ m_numberOfTries++;
+
+#if 0
+ cout<<"Number of tries is "<<(int)m_numberOfTries<<endl;
+#endif
+}
+
+int ExtensionElement::getNumberOfTries()const{
+ return m_numberOfTries;
+}
+
bool ExtensionElement::canMove(){
return m_canMove;
}
@@ -115,3 +130,15 @@ int ExtensionElement::getAgreement(){
int ExtensionElement::getReadLength(){
return m_read.length();
}
+
+void ExtensionElement::deactivate(){
+ m_activated=false;
+}
+
+void ExtensionElement::activate(){
+ m_activated=true;
+}
+
+bool ExtensionElement::isActive()const{
+ return m_activated;
+}
diff --git a/code/plugin_SeedExtender/ExtensionElement.h b/code/SeedExtender/ExtensionElement.h
similarity index 79%
rename from code/plugin_SeedExtender/ExtensionElement.h
rename to code/SeedExtender/ExtensionElement.h
index 8de3ac4..b0be828 100644
--- a/code/plugin_SeedExtender/ExtensionElement.h
+++ b/code/SeedExtender/ExtensionElement.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,13 +22,16 @@
#ifndef _ExtensionElement
#define _ExtensionElement
-#include <plugin_SequencesIndexer/PairedRead.h>
-#include <memory/MyAllocator.h>
-#include <application_core/Parameters.h>
+#include <code/SequencesIndexer/PairedRead.h>
+#include <code/SequencesLoader/Read.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/memory/MyAllocator.h>
/*
* An extension element is a read mapped on an
* extension for an MPI rank.
+ *
* \author Sébastien Boisvert
*/
class ExtensionElement{
@@ -43,9 +46,11 @@ class ExtensionElement{
/** the overall agreement of this molecular target */
uint16_t m_agreement;
+
+ uint8_t m_numberOfTries;
+ bool m_activated;
public:
- bool m_activated;
void setStrandPosition(int a);
int getStrandPosition();
@@ -71,6 +76,13 @@ public:
void increaseAgreement();
int getAgreement();
+ void deactivate();
+ void activate();
+ bool isActive()const;
+
+ int getNumberOfTries()const;
+ void increaseNumberOfTries();
+
} ATTRIBUTE_PACKED;
#endif
diff --git a/code/SeedExtender/Makefile b/code/SeedExtender/Makefile
new file mode 100644
index 0000000..36dd35e
--- /dev/null
+++ b/code/SeedExtender/Makefile
@@ -0,0 +1,14 @@
+SeedExtender-y += code/SeedExtender/SeedExtender.o
+SeedExtender-y += code/SeedExtender/Direction.o
+SeedExtender-y += code/SeedExtender/VertexMessenger.o
+SeedExtender-y += code/SeedExtender/ReadFetcher.o
+SeedExtender-y += code/SeedExtender/BubbleTool.o
+SeedExtender-y += code/SeedExtender/Chooser.o
+SeedExtender-y += code/SeedExtender/OpenAssemblerChooser.o
+SeedExtender-y += code/SeedExtender/TipWatchdog.o
+SeedExtender-y += code/SeedExtender/NovaEngine.o
+SeedExtender-y += code/SeedExtender/ExtensionElement.o
+SeedExtender-y += code/SeedExtender/DepthFirstSearchData.o
+SeedExtender-y += code/SeedExtender/ExtensionData.o
+
+obj-y += $(SeedExtender-y)
diff --git a/code/plugin_SeedExtender/NovaEngine.cpp b/code/SeedExtender/NovaEngine.cpp
similarity index 98%
rename from code/plugin_SeedExtender/NovaEngine.cpp
rename to code/SeedExtender/NovaEngine.cpp
index a6f2963..ebf7d9b 100644
--- a/code/plugin_SeedExtender/NovaEngine.cpp
+++ b/code/SeedExtender/NovaEngine.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,8 +19,9 @@
*/
+#include "NovaEngine.h"
+
#include <math.h>
-#include <plugin_SeedExtender/NovaEngine.h>
#include <assert.h>
/**
diff --git a/code/plugin_SeedExtender/NovaEngine.h b/code/SeedExtender/NovaEngine.h
similarity index 92%
rename from code/plugin_SeedExtender/NovaEngine.h
rename to code/SeedExtender/NovaEngine.h
index 5ffcafb..91fe7c7 100644
--- a/code/plugin_SeedExtender/NovaEngine.h
+++ b/code/SeedExtender/NovaEngine.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,8 +22,9 @@
#ifndef _NovaEngine_H
#define _NovaEngine_H
+#include "Chooser.h"
+
#include <vector>
-#include <plugin_SeedExtender/Chooser.h>
#include <map>
#include <set>
using namespace std;
diff --git a/code/plugin_SeedExtender/OpenAssemblerChooser.cpp b/code/SeedExtender/OpenAssemblerChooser.cpp
similarity index 98%
rename from code/plugin_SeedExtender/OpenAssemblerChooser.cpp
rename to code/SeedExtender/OpenAssemblerChooser.cpp
index c821118..dcb1392 100644
--- a/code/plugin_SeedExtender/OpenAssemblerChooser.cpp
+++ b/code/SeedExtender/OpenAssemblerChooser.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,8 +19,8 @@
*/
-#include<plugin_SeedExtender/OpenAssemblerChooser.h>
-#include<plugin_SeedExtender/Chooser.h>
+#include "OpenAssemblerChooser.h"
+#include "Chooser.h"
void OpenAssemblerChooser::updateMultiplicators(){
m_singleEndMultiplicator=2.0;
diff --git a/code/plugin_SeedExtender/OpenAssemblerChooser.h b/code/SeedExtender/OpenAssemblerChooser.h
similarity index 91%
rename from code/plugin_SeedExtender/OpenAssemblerChooser.h
rename to code/SeedExtender/OpenAssemblerChooser.h
index f838ed4..02573f2 100644
--- a/code/plugin_SeedExtender/OpenAssemblerChooser.h
+++ b/code/SeedExtender/OpenAssemblerChooser.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,8 +22,8 @@
#ifndef _OpenAssemblerChooser
#define _OpenAssemblerChooser
-#include <plugin_SeedExtender/Chooser.h> // for IMPOSSIBLE_CHOICE
-#include <plugin_SeedExtender/NovaEngine.h>
+#include "Chooser.h" // for IMPOSSIBLE_CHOICE
+#include "NovaEngine.h"
/**
* de Bruijn heuristic to choose extension direction in a graph, described in paper
diff --git a/code/plugin_SeedExtender/ReadFetcher.cpp b/code/SeedExtender/ReadFetcher.cpp
similarity index 93%
rename from code/plugin_SeedExtender/ReadFetcher.cpp
rename to code/SeedExtender/ReadFetcher.cpp
index 5f7b61d..5337eef 100644
--- a/code/plugin_SeedExtender/ReadFetcher.cpp
+++ b/code/SeedExtender/ReadFetcher.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,8 +19,9 @@
*/
-#include <plugin_SeedExtender/ReadFetcher.h>
-#include <core/OperatingSystem.h>
+#include "ReadFetcher.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
//#define GUILLIMIN_BUG
@@ -87,7 +88,7 @@ void ReadFetcher::work(){
#endif
- int destination=m_parameters->_vertexRank(&m_vertex);
+ int destination=m_parameters->vertexRank(&m_vertex);
#ifdef GUILLIMIN_BUG
if(m_parameters->getRank()==destination){
@@ -119,7 +120,7 @@ void ReadFetcher::work(){
//. ------
#ifdef GUILLIMIN_BUG
- int destination=m_parameters->_vertexRank(&m_vertex);
+ int destination=m_parameters->vertexRank(&m_vertex);
if(m_parameters->getRank()==destination){
cout<<endl;
cout<<"worker: "<<m_workerId<<endl;
@@ -154,7 +155,7 @@ void ReadFetcher::work(){
char strand=(char)buffer[4];
#ifdef ASSERT
- int destination=m_parameters->_vertexRank(&m_vertex);
+ int destination=m_parameters->vertexRank(&m_vertex);
assert(readIndex>=0);
assert(position>=0);
@@ -190,7 +191,7 @@ void ReadFetcher::work(){
}
- int destination=m_parameters->_vertexRank(&m_vertex);
+ int destination=m_parameters->vertexRank(&m_vertex);
Message aMessage(message2,period,destination,RAY_MPI_TAG_REQUEST_VERTEX_READS,m_parameters->getRank());
#ifdef GUILLIMIN_BUG
diff --git a/code/plugin_SeedExtender/ReadFetcher.h b/code/SeedExtender/ReadFetcher.h
similarity index 85%
rename from code/plugin_SeedExtender/ReadFetcher.h
rename to code/SeedExtender/ReadFetcher.h
index 705f168..ad2fe80 100644
--- a/code/plugin_SeedExtender/ReadFetcher.h
+++ b/code/SeedExtender/ReadFetcher.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,13 +22,15 @@
#ifndef _ReadFetcher
#define _ReadFetcher
+#include <code/SequencesIndexer/ReadAnnotation.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+
#include <vector>
-#include <plugin_SequencesIndexer/ReadAnnotation.h>
-#include <structures/StaticVector.h>
-#include <application_core/Parameters.h>
#include <stdint.h>
-#include <memory/RingAllocator.h>
-#include <communication/VirtualCommunicator.h>
using namespace std;
/**
diff --git a/code/plugin_SeedExtender/SeedExtender.cpp b/code/SeedExtender/SeedExtender.cpp
similarity index 90%
rename from code/plugin_SeedExtender/SeedExtender.cpp
rename to code/SeedExtender/SeedExtender.cpp
index dceaa84..c0c1b59 100644
--- a/code/plugin_SeedExtender/SeedExtender.cpp
+++ b/code/SeedExtender/SeedExtender.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,44 +14,44 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-// This option disables metagenome and transcriptome assembly
-// #define CONFIG_USE_COVERAGE_DISTRIBUTION
+#include "BubbleTool.h"
+#include "TipWatchdog.h"
+#include "SeedExtender.h"
+#include "Chooser.h"
-//#define CONFIG_DEBUG_SEED_EXTENSION
-
-/* TODO: free sequence in ExtensionElement objects when they are not needed anymore */
+#include <code/Mock/constants.h>
-#define __PROGRESSION_PERIOD 1000
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/cryptography/crypto.h>
-#include <application_core/constants.h>
-#include <string.h>
-#include <structures/StaticVector.h>
-#include <plugin_SeedExtender/TipWatchdog.h>
-#include <plugin_SeedExtender/SeedExtender.h>
-#include <core/OperatingSystem.h>
#include <fstream>
-#include <plugin_SeedExtender/Chooser.h>
+#include <math.h> /* sqrt */
+#include <string.h>
#include <sstream>
#include <assert.h>
-#include <plugin_SeedExtender/BubbleTool.h>
-#include <cryptography/crypto.h>
-#include <math.h> /* sqrt */
+
+// This option disables metagenome and transcriptome assembly
+// #define CONFIG_USE_COVERAGE_DISTRIBUTION
+
+//#define CONFIG_DEBUG_SEED_EXTENSION
+
+/* TODO: free sequence in ExtensionElement objects when they are not needed anymore */
+#define __PROGRESSION_PERIOD 10000
+
+#define MINIMUM_UNITS_FOR_VERBOSITY 1024
__CreatePlugin(SeedExtender);
- /**/
- /**/
__CreateSlaveModeAdapter(SeedExtender,RAY_SLAVE_MODE_EXTENSION); /**/
__CreateMessageTagAdapter(SeedExtender,RAY_MPI_TAG_ADD_GRAPH_PATH);
__CreateMessageTagAdapter(SeedExtender,RAY_MPI_TAG_ASK_IS_ASSEMBLED); /**/
__CreateMessageTagAdapter(SeedExtender,RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY); /**/
- /**/
- /**/
using namespace std;
@@ -69,18 +69,18 @@ void SeedExtender::closePathFile(){
void SeedExtender::call_RAY_SLAVE_MODE_EXTENSION(){
MACRO_COLLECT_PROFILING_INFORMATION();
-
+
/* read the checkpoint */
if(!m_checkedCheckpoint){
m_checkedCheckpoint=true;
if(m_parameters->hasCheckpoint("Extensions")){
readCheckpoint(m_fusionData);
-
+
(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_EXTENSION_IS_DONE,m_parameters->getRank());
- m_outbox->push_back(aMessage);
-
+ m_outbox->push_back(&aMessage);
+
return;
}
@@ -111,7 +111,7 @@ void SeedExtender::call_RAY_SLAVE_MODE_EXTENSION(){
// choose it
// else
// use read paths or pairs of reads to resolve the repeat.
-
+
// only check that at bootstrap.
if(!m_ed->m_EXTENSION_checkedIfCurrentVertexIsAssembled){
@@ -123,16 +123,16 @@ m_currentVertex,m_parameters->getRank(),m_vertexCoverageRequested,m_parameters->
}else if(
(
/* the first flow */
- m_ed->m_flowNumber==1
+ m_ed->m_flowNumber==1
/* vertex is assembled already */
- && m_ed->m_EXTENSION_vertexIsAssembledResult
+ && m_ed->m_EXTENSION_vertexIsAssembledResult
/* we have not exited the seed */
&& m_ed->m_EXTENSION_currentPosition<(int)m_ed->m_EXTENSION_currentSeed.size()
&& m_ed->m_EXTENSION_currentPosition==0
)
||
m_hotSkippingMode){
-
+
skipSeed(m_seeds);
}else if(!m_ed->m_EXTENSION_markedCurrentVertexAsAssembled){
@@ -178,7 +178,7 @@ bool*vertexCoverageReceived,int size,int*receivedVertexCoverage,Chooser*chooser,
// the code quality is awful, especially with all these arguments for each
// private method.
//
- // indeed, this information is fetched inside
+ // indeed, this information is fetched inside
// SeedExtender::markCurrentVertexAsAssembled
if(!(*edgesRequested)){
ed->m_EXTENSION_coverages.clear();
@@ -222,11 +222,11 @@ bool*vertexCoverageReceived,int size,int*receivedVertexCoverage,Chooser*chooser,
MessageUnit*message=(MessageUnit*)(*outboxAllocator).allocate(KMER_U64_ARRAY_SIZE*sizeof(MessageUnit));
int bufferPosition=0;
kmer.pack(message,&bufferPosition);
- Rank dest=m_parameters->_vertexRank(&kmer);
+ Rank dest=m_parameters->vertexRank(&kmer);
Message aMessage(message,bufferPosition,dest,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE,theRank);
- (*outbox).push_back(aMessage);
+ (*outbox).push_back(&aMessage);
(*vertexCoverageRequested)=true;
(*vertexCoverageReceived)=false;
@@ -245,7 +245,7 @@ bool*vertexCoverageReceived,int size,int*receivedVertexCoverage,Chooser*chooser,
#ifdef ASSERT
if(coverageValue==0){
- Rank dest=m_parameters->_vertexRank(&kmer);
+ Rank dest=m_parameters->vertexRank(&kmer);
cout<<"The kmer has a coverage of 0: ";
cout<<kmer.idToWord(m_parameters->getWordSize(),
@@ -298,7 +298,7 @@ bool*vertexCoverageReceived,int size,int*receivedVertexCoverage,Chooser*chooser,
#endif
}
}
-
+
MACRO_COLLECT_PROFILING_INFORMATION();
}
@@ -317,7 +317,7 @@ void SeedExtender::doChoice(RingAllocator*outboxAllocator,int*outgoingEdgeIndex,
Kmer*currentVertex,BubbleData*bubbleData,int theRank,
int wordSize,
ExtensionData*ed,int minimumCoverage,OpenAssemblerChooser*oa,Chooser*chooser,
- vector<AssemblySeed>*seeds,
+ vector<GraphPath>*seeds,
bool*edgesRequested,bool*vertexCoverageRequested,bool*vertexCoverageReceived,int size,
int*receivedVertexCoverage,bool*edgesReceived,vector<Kmer>*receivedOutgoingEdges
){
@@ -325,7 +325,7 @@ int*receivedVertexCoverage,bool*edgesReceived,vector<Kmer>*receivedOutgoingEdges
MACRO_COLLECT_PROFILING_INFORMATION();
if(m_expiredReads.count(ed->m_EXTENSION_currentPosition)>0){
-
+
processExpiredReads();
return;
@@ -347,7 +347,7 @@ int*receivedVertexCoverage,bool*edgesReceived,vector<Kmer>*receivedOutgoingEdges
m_removedUnfitLibraries=false;
// we received the vertex for that read,
- // now check if it matches one of
+ // now check if it matches one of
// the many choices we have
ReadHandle uniqueId=*(ed->m_EXTENSION_readIterator);
ExtensionElement*element=ed->getUsedRead(uniqueId);
@@ -403,7 +403,7 @@ Presently, insertions or deletions up to 8 are supported.
#endif
// the read is now out-of-range
- ed->m_EXTENSION_readIterator++;
+ ed->m_EXTENSION_readIterator++;
return;
}
@@ -434,7 +434,7 @@ Presently, insertions or deletions up to 8 are supported.
// do something about the agreement
element->increaseAgreement();
match=true;
-
+
//cout<<"Matched "<<uniqueId<<endl;
break;
@@ -465,7 +465,7 @@ Presently, insertions or deletions up to 8 are supported.
/* we only use single-end reads on
non-repeated vertices */
if(ed->m_currentCoverage< theRepeatedCoverage){
- ed->m_EXTENSION_readPositionsForVertices[ed->m_EXTENSION_receivedReadVertex].push_back(distance);
+ ed->m_EXTENSION_readPositionsForVertices[ed->m_EXTENSION_receivedReadVertex].push_back(distance);
}
ed->m_EXTENSION_readIterator++;
}else{// the read is paired
@@ -495,8 +495,8 @@ Presently, insertions or deletions up to 8 are supported.
int expectedFragmentLength=m_parameters->getLibraryAverageLength(library,peak);
int expectedDeviation=m_parameters->getLibraryStandardDeviation(library,peak);
- if(expectedFragmentLength-multiplier*expectedDeviation<=observedFragmentLength
- && observedFragmentLength <= expectedFragmentLength+multiplier*expectedDeviation
+ if(expectedFragmentLength-multiplier*expectedDeviation<=observedFragmentLength
+ && observedFragmentLength <= expectedFragmentLength+multiplier*expectedDeviation
&&( (theLeftStrand=='F' && theRightStrand=='R')
||(theLeftStrand=='R' && theRightStrand=='F'))
// the bridging pair is meaningless if both start in repeats
@@ -568,17 +568,30 @@ Presently, insertions or deletions up to 8 are supported.
}
ReadHandle uniqueId=ed->m_sequencesToFree[i];
+ ExtensionElement*element=ed->getUsedRead(uniqueId);
+
+/*
+ * We can't recycle a read after so many attempts !
+ */
+ int maximumNumberOfTries=16;
+
+ if(element->getNumberOfTries() > maximumNumberOfTries)
+ continue;
+
m_ed->m_pairedReadsWithoutMate.erase(uniqueId);
// free the sequence
#ifdef ASSERT
- ExtensionElement*element=ed->getUsedRead(uniqueId);
if(element==NULL){
cout<<"element "<<uniqueId<<" not found now="<<m_ed->m_EXTENSION_extension.size()-1<<""<<endl;
}
assert(element!=NULL);
#endif
+ if(m_parameters->hasOption("-debug-recycling")){
+ cout<<"Rank "<<m_rank<<" recycles read "<<uniqueId<<" at position ";
+ cout<<m_ed->m_EXTENSION_extension.size()-1<<endl;
+ }
// remove it
ed->removeSequence(uniqueId);
@@ -594,7 +607,7 @@ Presently, insertions or deletions up to 8 are supported.
if(m_parameters->showExtensionChoice()){
inspect(ed,currentVertex);
-
+
}
MACRO_COLLECT_PROFILING_INFORMATION();
@@ -605,7 +618,7 @@ Presently, insertions or deletions up to 8 are supported.
MACRO_COLLECT_PROFILING_INFORMATION();
- //int choice=IMPOSSIBLE_CHOICE;
+ //int choice=IMPOSSIBLE_CHOICE;
int choice=chooseWithSeed();
// square root sounds better than divided by something...
@@ -651,7 +664,7 @@ Presently, insertions or deletions up to 8 are supported.
}else if(!ed->m_doChoice_tips_Detected && ed->m_EXTENSION_readsInRange.size()>0){
//for each entries in ed->m_enumerateChoices_outgoingEdges, do a dfs of max depth 40.
//if the reached depth is 40, it is not a tip, otherwise, it is.
-
+
int maxDepth=2*m_parameters->getWordSize();
if(!m_dfsData->m_doChoice_tips_Initiated){
m_dfsData->m_doChoice_tips_i=0;
@@ -702,7 +715,7 @@ size,theRank,outbox,receivedVertexCoverage,receivedOutgoingEdges,minimumCoverage
}
}else{
// we have a winner with tips investigation.
- if(m_dfsData->m_doChoice_tips_newEdges.size()==1 && ed->m_EXTENSION_readsInRange.size()>0
+ if(m_dfsData->m_doChoice_tips_newEdges.size()==1 && ed->m_EXTENSION_readsInRange.size()>0
&& ed->m_EXTENSION_readPositionsForVertices[ed->m_enumerateChoices_outgoingEdges[m_dfsData->m_doChoice_tips_newEdges[0]]].size()>0
){
// tip watchdog!
@@ -733,7 +746,7 @@ size,theRank,outbox,receivedVertexCoverage,receivedOutgoingEdges,minimumCoverage
return;
// bubbles detection aims polymorphisms and homopolymers stretches.
}else if(!bubbleData->m_doChoice_bubbles_Detected && ed->m_EXTENSION_readsInRange.size()>0){
-
+
MACRO_COLLECT_PROFILING_INFORMATION();
int repeatCoverage=REPEAT_MULTIPLIER*m_currentPeakCoverage;
@@ -773,7 +786,7 @@ size,theRank,outbox,receivedVertexCoverage,receivedOutgoingEdges,minimumCoverage
mustFlowAgain=false;
// we may get more range
- if(ed->m_previouslyFlowedVertices < m_parameters->getMaximumDistance()
+ if(ed->m_previouslyFlowedVertices < m_parameters->getMaximumDistance()
&& (int)ed->m_EXTENSION_extension.size() >= m_parameters->getMaximumDistance()){
mustFlowAgain=true;
}
@@ -783,7 +796,7 @@ size,theRank,outbox,receivedVertexCoverage,receivedOutgoingEdges,minimumCoverage
int maximumNumberOfFlowCycles=8;
// no choice possible...
- if((int)ed->m_EXTENSION_extension.size() > ed->m_previouslyFlowedVertices && mustFlowAgain
+ if((int)ed->m_EXTENSION_extension.size() > ed->m_previouslyFlowedVertices && mustFlowAgain
&& ed->m_flowNumber < maximumNumberOfFlowCycles){
MACRO_COLLECT_PROFILING_INFORMATION();
@@ -791,6 +804,8 @@ size,theRank,outbox,receivedVertexCoverage,receivedOutgoingEdges,minimumCoverage
if(!m_slicedComputationStarted){
m_slicedComputationStarted=true;
m_complementedSeed.clear();
+ m_complementedSeed.setKmerLength(m_parameters->getWordSize());
+
m_slicedProgression = ed->m_EXTENSION_extension.size()-1;
//cout<<"INITIATING SLICED COMPUTATION"<<endl;
}
@@ -804,7 +819,9 @@ size,theRank,outbox,receivedVertexCoverage,receivedOutgoingEdges,minimumCoverage
// this code must be run in many slices...
for(int i= m_slicedProgression;i>=0;i--){
- Kmer newKmer=ed->m_EXTENSION_extension.at(i).complementVertex(wordSize,
+ Kmer theKmer;
+ ed->m_EXTENSION_extension.at(i,&theKmer);
+ Kmer newKmer=theKmer.complementVertex(wordSize,
m_parameters->getColorSpaceMode());
m_complementedSeed.push_back(&newKmer);
@@ -840,8 +857,11 @@ size,theRank,outbox,receivedVertexCoverage,receivedOutgoingEdges,minimumCoverage
printExtensionStatus(currentVertex);
- cout<<"Rank "<<m_parameters->getRank()<<" is changing direction."<<endl;
-
+ bool verbose=ed->m_EXTENSION_extension.size()>=MINIMUM_UNITS_FOR_VERBOSITY;
+
+ if(verbose)
+ cout<<"Rank "<<m_parameters->getRank()<<" is changing direction."<<endl;
+
#ifdef ASSERT
assert(m_complementedSeed.size() == (int)ed->m_EXTENSION_extension.size());
#endif
@@ -855,7 +875,9 @@ size,theRank,outbox,receivedVertexCoverage,receivedOutgoingEdges,minimumCoverage
ed->m_EXTENSION_currentSeed=m_complementedSeed;
ed->m_EXTENSION_checkedIfCurrentVertexIsAssembled=false;
- Kmer*theKmer=ed->m_EXTENSION_currentSeed.at(ed->m_EXTENSION_currentPosition);
+ Kmer aKmer;
+ ed->m_EXTENSION_currentSeed.at(ed->m_EXTENSION_currentPosition,&aKmer);
+ Kmer*theKmer=&aKmer;
(*currentVertex)=*theKmer;
MACRO_COLLECT_PROFILING_INFORMATION();
@@ -866,7 +888,7 @@ size,theRank,outbox,receivedVertexCoverage,receivedOutgoingEdges,minimumCoverage
// TODO: this needs to be sliced or optimized
m_matesToMeet.clear();
-
+
MACRO_COLLECT_PROFILING_INFORMATION();
/*
m_cacheAllocator.clear();
@@ -929,11 +951,12 @@ map<Kmer,set<Kmer> >*arcs,map<Kmer,int>*coverages,int depth,set<Kmer>*visited){
}
/** store the extension and do the next one right now */
-void SeedExtender::storeExtensionAndGetNextOne(ExtensionData*ed,int theRank,vector<AssemblySeed>*seeds,
+void SeedExtender::storeExtensionAndGetNextOne(ExtensionData*ed,int theRank,vector<GraphPath>*seeds,
Kmer *currentVertex,BubbleData*bubbleData){
int length=getNumberOfNucleotides(ed->m_EXTENSION_extension.size(),m_parameters->getWordSize());
+
if(length>=m_parameters->getMinimumContigLength()){
MACRO_COLLECT_PROFILING_INFORMATION();
@@ -942,7 +965,9 @@ Kmer *currentVertex,BubbleData*bubbleData){
if(!m_slicedComputationStarted){
m_slicedComputationStarted = true;
m_slicedProgression = 0;
- vector<Kmer> emptyOne;
+ GraphPath emptyOne;
+ emptyOne.setKmerLength(m_parameters->getWordSize());
+
ed->m_EXTENSION_contigs.push_back(emptyOne);
MACRO_COLLECT_PROFILING_INFORMATION();
@@ -955,7 +980,10 @@ Kmer *currentVertex,BubbleData*bubbleData){
// this hunk needs to be time-sliced...
if(m_slicedProgression < (int) ed->m_EXTENSION_extension.size()){
- ed->m_EXTENSION_contigs[ed->m_EXTENSION_contigs.size()-1].push_back(ed->m_EXTENSION_extension[m_slicedProgression]);
+
+ Kmer kmer;
+ ed->m_EXTENSION_extension.at(m_slicedProgression,&kmer);
+ ed->m_EXTENSION_contigs[ed->m_EXTENSION_contigs.size()-1].push_back(&kmer);
m_slicedProgression++;
return;
}
@@ -968,11 +996,25 @@ Kmer *currentVertex,BubbleData*bubbleData){
return;
}
+/*
+ * The time-slice job has finished at this point.
+ */
+ m_nucleotidesAssembled+=length;
+ if(m_nucleotidesAssembled>= m_lastNucleotideAssembled+m_nucleotidePeriod){
+
+ cout<<"Rank "<<m_rank<<" traversed "<<m_nucleotidesAssembled<<" nucleotide symbols"<<endl;
+
+ m_lastNucleotideAssembled=m_nucleotidesAssembled;
+ }
+
+
// reuse the state later
m_slicedComputationStarted = false;
m_flowedVertices.push_back(ed->m_EXTENSION_extension.size());
+ bool verbose=ed->m_EXTENSION_extension.size()>=MINIMUM_UNITS_FOR_VERBOSITY;
+
MACRO_COLLECT_PROFILING_INFORMATION();
if(m_parameters->showEndingContext()){
@@ -1017,19 +1059,22 @@ Kmer *currentVertex,BubbleData*bubbleData){
MACRO_COLLECT_PROFILING_INFORMATION();
- cout<<"Rank "<<theRank<<" (extension done) NumberOfFlows: "<<ed->m_flowNumber<<endl;
+ if(verbose)
+ cout<<"Rank "<<theRank<<" (extension done) NumberOfFlows: "<<ed->m_flowNumber<<endl;
m_extended++;
MACRO_COLLECT_PROFILING_INFORMATION();
- cout<<"Rank "<<m_parameters->getRank()<<" FlowedVertices:";
- for(int i=0;i<(int)m_flowedVertices.size();i++){
- cout<<" "<<i<<" "<<m_flowedVertices[i];
+ if(verbose){
+ cout<<"Rank "<<m_parameters->getRank()<<" FlowedVertices:";
+ for(int i=0;i<(int)m_flowedVertices.size();i++){
+ cout<<" "<<i<<" "<<m_flowedVertices[i];
+ }
+
+ cout<<endl;
}
- cout<<endl;
-
MACRO_COLLECT_PROFILING_INFORMATION();
if(m_parameters->hasOption("-show-distance-summary")){
@@ -1042,13 +1087,13 @@ Kmer *currentVertex,BubbleData*bubbleData){
int average=m_parameters->getLibraryAverageLength(lib,peak);
int deviation=m_parameters->getLibraryStandardDeviation(lib,peak);
LargeCount count=j->second;
-
+
cout<<"Rank "<<theRank<<" Library: "<<lib<<" LibraryPeak: "<<peak<<" PeakAverage: "<<average<<" PeakDeviation: "<<deviation<<" Pairs: "<<count<<endl;
}
}
}
-
+
PathHandle id=getPathUniqueId(theRank,ed->m_EXTENSION_currentSeedIndex);
ed->m_EXTENSION_identifiers.push_back(id);
@@ -1062,12 +1107,12 @@ Kmer *currentVertex,BubbleData*bubbleData){
int pathLength=ed->m_EXTENSION_extension.size();
int flows=ed->m_flowNumber;
- buffer[bufferPosition++]=id;
+ buffer[bufferPosition++]=id.getValue();
buffer[bufferPosition++]=pathLength;
buffer[bufferPosition++]=flows;
m_switchMan->sendMessage(buffer,bufferPosition,m_outbox,m_parameters->getRank(),MASTER_RANK,RAY_MPI_TAG_ADD_GRAPH_PATH);
-
+
// we don't need a reply for this message
//
@@ -1084,8 +1129,6 @@ Kmer *currentVertex,BubbleData*bubbleData){
m_cacheAllocator.clear();
m_cache.clear();
- fflush(stdout);
-
if(m_parameters->showMemoryUsage()){
showMemoryUsage(theRank);
}
@@ -1095,8 +1138,9 @@ Kmer *currentVertex,BubbleData*bubbleData){
ed->m_EXTENSION_currentPosition=0;
if(ed->m_EXTENSION_currentSeedIndex<(int)(*seeds).size()){
ed->m_EXTENSION_currentSeed=(*seeds)[ed->m_EXTENSION_currentSeedIndex];
- Kmer*theKmer=ed->m_EXTENSION_currentSeed.at(ed->m_EXTENSION_currentPosition);
- (*currentVertex)=*theKmer;
+ Kmer theKmer;
+ ed->m_EXTENSION_currentSeed.at(ed->m_EXTENSION_currentPosition,&theKmer);
+ (*currentVertex)=theKmer;
}
ed->m_EXTENSION_checkedIfCurrentVertexIsAssembled=false;
@@ -1109,21 +1153,10 @@ Kmer *currentVertex,BubbleData*bubbleData){
}
void SeedExtender::checkIfCurrentVertexIsAssembled(ExtensionData*ed,StaticVector*outbox,RingAllocator*outboxAllocator,
- int*outgoingEdgeIndex,int*last_value,Kmer*currentVertex,int theRank,bool*vertexCoverageRequested,int wordSize,int size,vector<AssemblySeed>*seeds){
+ int*outgoingEdgeIndex,int*last_value,Kmer*currentVertex,int theRank,bool*vertexCoverageRequested,int wordSize,int size,vector<GraphPath>*seeds){
MACRO_COLLECT_PROFILING_INFORMATION();
- //cout<<"checkIfCurrentVertexIsAssembled "<<ed->m_EXTENSION_currentPosition<<endl;
-
-/*
- if(!(ed->m_EXTENSION_currentPosition<(int)ed->m_EXTENSION_currentSeed.size())
- && ed->m_flowNumber==1){
-
- checkedCurrentVertex();
-
- }else
-*/
-
if(!ed->m_EXTENSION_directVertexDone){
if(!ed->m_EXTENSION_VertexAssembled_requested){
@@ -1133,12 +1166,11 @@ void SeedExtender::checkIfCurrentVertexIsAssembled(ExtensionData*ed,StaticVector
m_dfsData->setTags(RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE, RAY_MPI_TAG_REQUEST_VERTEX_EDGES,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES);
m_receivedDirections.clear();
- if(ed->m_EXTENSION_currentSeedIndex%10==0 && ed->m_EXTENSION_currentPosition==0
+ if(ed->m_EXTENSION_currentSeedIndex%1000==0 && ed->m_EXTENSION_currentPosition==0
&&(*last_value)!=ed->m_EXTENSION_currentSeedIndex){
(*last_value)=ed->m_EXTENSION_currentSeedIndex;
printf("Rank %i is extending seeds [%i/%i] \n",theRank,(int)ed->m_EXTENSION_currentSeedIndex+1,(int)(*seeds).size());
- fflush(stdout);
-
+
}
if(ed->m_EXTENSION_currentPosition==0){
@@ -1147,7 +1179,7 @@ void SeedExtender::checkIfCurrentVertexIsAssembled(ExtensionData*ed,StaticVector
ed->m_EXTENSION_VertexAssembled_requested=true;
ed->m_EXTENSION_VertexAssembled_received=false;
-
+
MACRO_COLLECT_PROFILING_INFORMATION();
/* if the position is not 0 on flow 0, we don't need to send this message */
@@ -1166,9 +1198,9 @@ void SeedExtender::checkIfCurrentVertexIsAssembled(ExtensionData*ed,StaticVector
currentVertex->pack(message,&bufferPosition);
message[bufferPosition++]=m_rank;
- Rank destination=m_parameters->_vertexRank(currentVertex);
+ Rank destination=m_parameters->vertexRank(currentVertex);
Message aMessage(message,bufferPosition,destination,RAY_MPI_TAG_ASK_IS_ASSEMBLED,theRank);
- (*outbox).push_back(aMessage);
+ (*outbox).push_back(&aMessage);
MACRO_COLLECT_PROFILING_INFORMATION();
@@ -1176,7 +1208,6 @@ void SeedExtender::checkIfCurrentVertexIsAssembled(ExtensionData*ed,StaticVector
if(m_theProcessIsRedundantByAGreaterAndMightyRank){
m_redundantProcessingVirtualMachineCycles++;
-
}
#ifdef CONFIG_DEBUG_SEED_EXTENSION
@@ -1185,13 +1216,12 @@ void SeedExtender::checkIfCurrentVertexIsAssembled(ExtensionData*ed,StaticVector
cout<<" position "<<ed->m_EXTENSION_currentPosition<<endl;
#endif
- if(m_redundantProcessingVirtualMachineCycles>m_hotSkippingThreshold
- && !m_hotSkippingMode){
-
- m_hotSkippingMode=true;
- cout<<"[SeedExtender] activating Hot Skipping !"<<endl;
+ if(m_redundantProcessingVirtualMachineCycles > m_hotSkippingThreshold
+ && m_hotSkippingMode){
ed->m_EXTENSION_extension.clear();
+ ed->m_EXTENSION_extension.setKmerLength(m_parameters->getWordSize());
+
storeExtensionAndGetNextOne(m_ed,m_rank,m_seeds,currentVertex,m_bubbleData);
}
@@ -1207,7 +1237,7 @@ void SeedExtender::checkIfCurrentVertexIsAssembled(ExtensionData*ed,StaticVector
ed->m_EXTENSION_directVertexDone=false;
ed->m_EXTENSION_reads_requested=false;
m_messengerInitiated=false;
-
+
}
}
}else if(!ed->m_EXTENSION_reverseVertexDone){
@@ -1230,13 +1260,13 @@ void SeedExtender::checkedCurrentVertex(){
* in this method:
* - the coverage of the vertex is obtained
* - the reads having a read marker on this vertex are gathered
- * - the owner of the vertex is advised that there is a path passing on it
+ * - the owner of the vertex is advised that there is a path passing on it
* */
-void SeedExtender::markCurrentVertexAsAssembled(Kmer*currentVertex,RingAllocator*outboxAllocator,int*outgoingEdgeIndex,
+void SeedExtender::markCurrentVertexAsAssembled(Kmer*currentVertex,RingAllocator*outboxAllocator,int*outgoingEdgeIndex,
StaticVector*outbox,int size,int theRank,ExtensionData*ed,bool*vertexCoverageRequested,bool*vertexCoverageReceived,
int*receivedVertexCoverage,bool*edgesRequested,
vector<Kmer>*receivedOutgoingEdges,Chooser*chooser,
-BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,vector<AssemblySeed>*seeds
+BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,vector<GraphPath>*seeds
){
MACRO_COLLECT_PROFILING_INFORMATION();
@@ -1262,7 +1292,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
if(m_ed->m_expirations.count(previousPosition) > 0){
vector<ReadHandle>*expired=&(m_ed->m_expirations)[previousPosition];
- // erase these reads from the list of reads without mate
+ // erase these reads from the list of reads without mate
// because they are expired...
// this just free some memory and does not change the result.
for(int i=0;i<(int)expired->size();i++){
@@ -1297,20 +1327,26 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
ed->m_EXTENSION_currentSeedIndex,
(int)ed->m_EXTENSION_currentSeed.size(),ed->m_flowNumber,
ed->m_EXTENSION_currentSeedIndex,(int)(*seeds).size());
- fflush(stdout);
m_flowedVertices.push_back(ed->m_EXTENSION_currentSeed.size());
-
+
+ bool verbose=ed->m_EXTENSION_currentSeed.size()>=MINIMUM_UNITS_FOR_VERBOSITY;
+
/* flow #0 is the seed */
ed->m_flowNumber++;
m_currentPeakCoverage=m_ed->m_EXTENSION_currentSeed.getPeakCoverage();
+ #ifdef ASSERT
+ assert(m_currentPeakCoverage>=2);
+ #endif
+
// Ray v1.7 and earlier have CONFIG_USE_COVERAGE_DISTRIBUTION=y behavior
#ifdef CONFIG_USE_COVERAGE_DISTRIBUTION
m_currentPeakCoverage=m_parameters->getPeakCoverage();
#endif
- cout<<"Current peak coverage -> "<<m_currentPeakCoverage<<endl;
+ if(verbose)
+ cout<<"Current peak coverage -> "<<m_currentPeakCoverage<<endl;
}
printExtensionStatus(currentVertex);
}
@@ -1333,7 +1369,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
---------------------------------------- seed
* current position
-
+
<------> maximum outer distance
@@ -1347,6 +1383,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
int progression=ed->m_EXTENSION_extension.size()-1;
int threshold=ed->m_EXTENSION_currentSeed.size()-m_parameters->getMaximumDistance();
bool getReads=false;
+
if(progression>=threshold)
getReads=true;
@@ -1406,11 +1443,14 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
uint8_t compactEdges=m_vertexMessenger.getEdges();
m_compactEdges=compactEdges;
- *receivedOutgoingEdges=currentVertex->_getOutgoingEdges(compactEdges,m_parameters->getWordSize());
+ *receivedOutgoingEdges=currentVertex->getOutgoingEdges(compactEdges,m_parameters->getWordSize());
MACRO_COLLECT_PROFILING_INFORMATION();
- ed->m_EXTENSION_extension.push_back((*currentVertex));
+ if(ed->m_EXTENSION_extension.size()==0)
+ ed->m_EXTENSION_extension.setKmerLength(m_parameters->getWordSize());
+
+ ed->m_EXTENSION_extension.push_back((currentVertex));
ed->m_extensionCoverageValues.push_back(*receivedVertexCoverage);
#ifdef ASSERT
@@ -1424,7 +1464,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
MACRO_COLLECT_PROFILING_INFORMATION();
}else{
- // process each received marker and decide if
+ // process each received marker and decide if
// if it will be utilised or not
if(m_sequenceIndexToCache<(int)ed->m_EXTENSION_receivedReads.size()){
MACRO_COLLECT_PROFILING_INFORMATION();
@@ -1441,7 +1481,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
/**
* CAse 1. we already saw the read
*
- * if the read is still within the range of the peak, update it
+ * if the read is still within the range of the peak, update it
*
* this complicated code add-on avoids the collapsing of
* tandemly-repeated repeats.
@@ -1483,8 +1523,8 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
int expectedFragmentLength=m_parameters->getLibraryAverageLength(library,peak);
int expectedDeviation=m_parameters->getLibraryStandardDeviation(library,peak);
- if(expectedFragmentLength-multiplier*expectedDeviation<=observedFragmentLength
- && observedFragmentLength <= expectedFragmentLength+multiplier*expectedDeviation
+ if(expectedFragmentLength-multiplier*expectedDeviation<=observedFragmentLength
+ && observedFragmentLength <= expectedFragmentLength+multiplier*expectedDeviation
&&( (theLeftStrand=='F' && theRightStrand=='R')
||(theLeftStrand=='R' && theRightStrand=='F'))
// the bridging pair is meaningless if both start in repeats
@@ -1497,7 +1537,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
break;
}
}
-
+
if(updateRead && anElement->canMove()){
anElement->setStartingPosition(startPosition);
ed->m_EXTENSION_readsInRange.insert(uniqueId);
@@ -1559,7 +1599,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
MessageUnit*message=(MessageUnit*)(*outboxAllocator).allocate(1*sizeof(MessageUnit));
message[0]=ed->m_EXTENSION_receivedReads[m_sequenceIndexToCache].getReadIndex();
Message aMessage(message,1,sequenceRank,RAY_MPI_TAG_REQUEST_READ_SEQUENCE,theRank);
- outbox->push_back(aMessage);
+ outbox->push_back(&aMessage);
MACRO_COLLECT_PROFILING_INFORMATION();
@@ -1585,7 +1625,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
// don't add it up if its is marked on a repeated vertex and
// its mate was not seen yet.
-
+
// Just to be sure, we use a conservative multiplicator of XYZ
// this use to be 2.0
@@ -1605,7 +1645,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
if(ed->m_EXTENSION_pairedRead.getLibrary()!=DUMMY_LIBRARY){
ReadHandle mateId=ed->m_EXTENSION_pairedRead.getUniqueId();
// the mate is required to allow proper placement
-
+
ExtensionElement*extensionElement=ed->getUsedRead(mateId);
if(extensionElement==NULL){
@@ -1619,7 +1659,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
if(addRead && ed->m_EXTENSION_pairedRead.getLibrary()!=DUMMY_LIBRARY){
ReadHandle mateId=ed->m_EXTENSION_pairedRead.getUniqueId();
// the mate is required to allow proper placement
-
+
ExtensionElement*extensionElement=ed->getUsedRead(mateId);
if(extensionElement!=NULL){// use to be via readsPositions
@@ -1640,8 +1680,8 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
int expectedFragmentLength=m_parameters->getLibraryAverageLength(library,peak);
int expectedDeviation=m_parameters->getLibraryStandardDeviation(library,peak);
- if(expectedFragmentLength-multiplier*expectedDeviation<=observedFragmentLength
- && observedFragmentLength <= expectedFragmentLength+multiplier*expectedDeviation
+ if(expectedFragmentLength-multiplier*expectedDeviation<=observedFragmentLength
+ && observedFragmentLength <= expectedFragmentLength+multiplier*expectedDeviation
&&( (theLeftStrand=='F' && theRightStrand=='R')
||(theLeftStrand=='R' && theRightStrand=='F'))
// the bridging pair is meaningless if both start in repeats
@@ -1664,7 +1704,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
/* after making sure the read is sane, we can add it here for sure */
if(addRead){
-
+
if(m_parameters->showReadPlacement()){
cout<<"[showReadPlacement] Adding read "<<uniqueId<<" at "<<position;
cout<<" with read offset "<<positionOnStrand<<endl;
@@ -1672,7 +1712,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
m_matesToMeet.erase(uniqueId);
ExtensionElement*element=ed->addUsedRead(uniqueId);
-
+
// the first vertex obviously agrees.
element->increaseAgreement();
@@ -1689,11 +1729,11 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
element->getSequence(m_receivedString,m_parameters);
assert(readLength==(int)strlen(m_receivedString));
#endif
-
+
// without the +1, it would be the last k-mer provided
// by the read
int expiryPosition=position+readLength-positionOnStrand-wordSize+1;
-
+
if(m_parameters->showReadPlacement()){
cout<<"[showReadPlacement] Read "<<uniqueId<<" will expire at "<<expiryPosition<<endl;
}
@@ -1718,7 +1758,7 @@ BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,v
int expiration=startPosition+expectedFragmentLength+3*expectedDeviation;
(ed->m_expirations)[expiration].push_back(uniqueId);
-
+
MACRO_COLLECT_PROFILING_INFORMATION();
m_matesToMeet.insert(mateId);
@@ -1759,12 +1799,12 @@ set<PathHandle>*SeedExtender::getEliminatedSeeds(){
return &m_eliminatedSeeds;
}
-void SeedExtender::constructor(Parameters*parameters,MyAllocator*m_directionsAllocator,ExtensionData*ed,
+void SeedExtender::constructor(Parameters*parameters,ExtensionData*ed,
GridTable*subgraph,StaticVector*inbox,Profiler*profiler,StaticVector*outbox,
SeedingData*seedingData,int*mode,
bool*vertexCoverageRequested,
bool*vertexCoverageReceived,RingAllocator*outboxAllocator,FusionData*fusionData,
-vector<AssemblySeed>*seeds,BubbleData*bubbleData,
+vector<GraphPath>*seeds,BubbleData*bubbleData,
bool*edgesRequested,bool*edgesReceived,
int*outgoingEdgeIndex,Kmer*currentVertex,int*receivedVertexCoverage,
vector<Kmer>*receivedOutgoingEdges,
@@ -1808,7 +1848,6 @@ Chooser*chooser,OpenAssemblerChooser*oa
m_dfsData->setTags(RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE, RAY_MPI_TAG_REQUEST_VERTEX_EDGES,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES);
m_cache.constructor();
m_ed=ed;
- this->m_directionsAllocator=m_directionsAllocator;
m_bubbleTool.constructor(parameters);
m_profiler=profiler;
@@ -1818,16 +1857,16 @@ Chooser*chooser,OpenAssemblerChooser*oa
void SeedExtender::configureTheBeautifulHotSkippingTechnology(){
- //cout<<"calling configureTheBeautifulHotSkippingTechnology"<<endl;
-
- m_hotSkippingThreshold=99999999;
- m_redundantProcessingVirtualMachineCycles=1/1000;
+ m_hotSkippingThreshold = m_parameters->getMaximumDistance() * 1.1;
+ m_redundantProcessingVirtualMachineCycles = 0;
- #ifdef ASSERT
- assert(m_redundantProcessingVirtualMachineCycles==0x0);
- #endif
+ m_hotSkippingMode = false;
- m_hotSkippingMode=false;
+/*
+ * enable the hot skipping technology for short seeds.
+ */
+ if(m_ed->m_EXTENSION_currentSeed.size() < 3 * m_parameters->getWordSize())
+ m_hotSkippingMode = true;
}
void SeedExtender::inspect(ExtensionData*ed,Kmer*currentVertex){
@@ -1924,7 +1963,7 @@ void SeedExtender::removeUnfitLibraries(){
continue;
int mean=sum/n;
-
+
int minimumNumberOfBridges=2;
if(averageLength>=5000){
@@ -1991,21 +2030,25 @@ void SeedExtender::showReadsInRange(){
cout<<" "<<*i;
}
cout<<endl;
- cout.flush();
}
void SeedExtender::printExtensionStatus(Kmer*currentVertex){
int theRank=m_parameters->getRank();
- printf("Rank %i reached %i vertices from seed %i, flow %i\n",theRank,
- (int)m_ed->m_EXTENSION_extension.size(),
- m_ed->m_EXTENSION_currentSeedIndex,m_ed->m_flowNumber);
+ bool verbose=m_ed->m_EXTENSION_extension.size()>=MINIMUM_UNITS_FOR_VERBOSITY;
- fflush(stdout);
+ if(verbose){
+ printf("Rank %i reached %i vertices from seed %i, flow %i\n",theRank,
+ (int)m_ed->m_EXTENSION_extension.size(),
+ m_ed->m_EXTENSION_currentSeedIndex,m_ed->m_flowNumber);
+ }
m_derivative.addX(m_ed->m_EXTENSION_extension.size());
- m_derivative.printStatus(SLAVE_MODES[RAY_SLAVE_MODE_EXTENSION],
- RAY_SLAVE_MODE_EXTENSION);
+
+ if(verbose){
+ m_derivative.printStatus(SLAVE_MODES[RAY_SLAVE_MODE_EXTENSION],
+ RAY_SLAVE_MODE_EXTENSION);
+ }
/*
cout<<"Expiration.size= "<<(m_ed->m_expirations).size()<<endl;
@@ -2015,7 +2058,7 @@ void SeedExtender::printExtensionStatus(Kmer*currentVertex){
}
*/
- if(m_parameters->showMemoryUsage()){
+ if(verbose && m_parameters->showMemoryUsage()){
showMemoryUsage(theRank);
}
@@ -2027,17 +2070,22 @@ void SeedExtender::printExtensionStatus(Kmer*currentVertex){
void SeedExtender::writeCheckpoint(){
cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint Extensions"<<endl;
ofstream f(m_parameters->getCheckpointFile("Extensions").c_str());
+ ostringstream buffer;
int count=m_ed->m_EXTENSION_contigs.size();
- f.write((char*)&count,sizeof(int));
+ buffer.write((char*)&count, sizeof(int));
for(int i=0;i<count;i++){
int length=m_ed->m_EXTENSION_contigs[i].size();
- f.write((char*)&length,sizeof(int));
+ buffer.write((char*)&length, sizeof(int));
for(int j=0;j<length;j++){
- m_ed->m_EXTENSION_contigs[i][j].write(&f);
+ Kmer object;
+ m_ed->m_EXTENSION_contigs[i].at(j,&object);
+ object.write(&buffer);
}
+ flushFileOperationBuffer(false, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
}
+ flushFileOperationBuffer(true, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
f.close();
}
@@ -2058,11 +2106,13 @@ void SeedExtender::readCheckpoint(FusionData*fusionData){
#ifdef ASSERT
assert(length>0);
#endif
- vector<Kmer> extension;
+ GraphPath extension;
+ extension.setKmerLength(m_parameters->getWordSize());
+
for(int j=0;j<length;j++){
Kmer kmer;
kmer.read(&f);
- extension.push_back(kmer);
+ extension.push_back(&kmer);
}
m_ed->m_EXTENSION_contigs.push_back(extension);
@@ -2074,7 +2124,7 @@ void SeedExtender::readCheckpoint(FusionData*fusionData){
f.close();
cout<<"Rank "<<m_parameters->getRank()<<" loaded "<<count<<" extensions from checkpoint Extensions"<<endl;
-
+
#ifdef ASSERT
assert(count==(int)m_ed->m_EXTENSION_contigs.size());
assert(m_ed->m_EXTENSION_identifiers.size()==m_ed->m_EXTENSION_contigs.size());
@@ -2103,10 +2153,13 @@ void SeedExtender::showSequences(){
}
// print the contig
- vector<Kmer> lastBits;
+ GraphPath lastBits;
+ lastBits.setKmerLength(m_parameters->getWordSize());
for(int i=firstPosition;i<(int)m_ed->m_EXTENSION_extension.size();i++){
- lastBits.push_back(m_ed->m_EXTENSION_extension[i]);
+ Kmer object;
+ m_ed->m_EXTENSION_extension.at(i,&object);
+ lastBits.push_back(&object);
}
string sequence = convertToString(&lastBits,
@@ -2190,7 +2243,7 @@ void SeedExtender::processExpiredReads(){
}else{
cout<<" fair enough !"<<endl;
}
-
+
}
#ifdef ASSERT
@@ -2205,7 +2258,7 @@ void SeedExtender::processExpiredReads(){
#ifdef ASSERT
assert(read!=NULL);
#endif
-
+
element->removeSequence();
@@ -2236,7 +2289,9 @@ void SeedExtender::printSeed(){
cout<<" Position Kmer Coverage"<<endl;
while(position>=0){
- Kmer*kmer= (m_ed->m_EXTENSION_currentSeed.at(position));
+ Kmer object;
+ m_ed->m_EXTENSION_currentSeed.at(position,&object);
+ Kmer*kmer=&object;
CoverageDepth depth=m_ed->m_EXTENSION_currentSeed.getCoverageAt(position);
cout<<" "<<position<<" "<<kmer->idToWord(wordSize,colored)<<" ";
@@ -2264,11 +2319,13 @@ int SeedExtender::chooseWithSeed(){
bool colored=m_parameters->getColorSpaceMode();
- Kmer*kmerInSeed=m_ed->m_EXTENSION_currentSeed.at(m_ed->m_EXTENSION_currentPosition);
+ Kmer object;
+ m_ed->m_EXTENSION_currentSeed.at(m_ed->m_EXTENSION_currentPosition,&object);
+ Kmer*kmerInSeed=&object;
// find a perfect match
for(int i=0;i<(int)m_ed->m_enumerateChoices_outgoingEdges.size();i++){
-
+
if(m_ed->m_enumerateChoices_outgoingEdges[i]== *kmerInSeed ){
return i;
}
@@ -2279,7 +2336,11 @@ int SeedExtender::chooseWithSeed(){
int wordSize = m_parameters->getWordSize();
cout<<"Error: The seed contains a choice not supported by the graph."<<endl;
cout<<"Extension length: "<<m_ed->m_EXTENSION_extension.size()<<" vertices"<<endl;
- cout<<"position="<<m_ed->m_EXTENSION_currentPosition<<" "<<m_ed->m_EXTENSION_currentSeed.at(m_ed->m_EXTENSION_currentPosition)->idToWord(wordSize,m_parameters->getColorSpaceMode())<<" with "<<m_ed->m_enumerateChoices_outgoingEdges.size()<<" choices ";
+ cout<<"position="<<m_ed->m_EXTENSION_currentPosition<<" ";
+ Kmer kmerObject;
+ m_ed->m_EXTENSION_currentSeed.at(m_ed->m_EXTENSION_currentPosition,&kmerObject);
+ cout<<kmerObject.idToWord(wordSize,m_parameters->getColorSpaceMode());
+ cout<<" with "<<m_ed->m_enumerateChoices_outgoingEdges.size()<<" choices ";
cout<<endl;
cout<<"The previous kmer is ";
@@ -2287,7 +2348,9 @@ int SeedExtender::chooseWithSeed(){
int previousPosition=m_ed->m_EXTENSION_currentPosition-1;
if(previousPosition>=0){
- cout<<m_ed->m_EXTENSION_currentSeed.at(previousPosition)->idToWord(wordSize,m_parameters->getColorSpaceMode());
+ Kmer kmerObject;
+ m_ed->m_EXTENSION_currentSeed.at(previousPosition,&kmerObject);
+ cout<<kmerObject.idToWord(wordSize,m_parameters->getColorSpaceMode());
}
cout<<endl;
@@ -2316,7 +2379,7 @@ int SeedExtender::chooseWithSeed(){
assert(m_ed->m_EXTENSION_coverages.size()==m_ed->m_enumerateChoices_outgoingEdges.size());
#endif
- vector<Kmer> compactData=m_currentVertex->_getOutgoingEdges(m_compactEdges,
+ vector<Kmer> compactData=m_currentVertex->getOutgoingEdges(m_compactEdges,
m_parameters->getWordSize());
cout<<"Ray info ***********************"<<endl;
@@ -2339,7 +2402,10 @@ int SeedExtender::chooseWithSeed(){
cout<<" Position Kmer Coverage"<<endl;
while(position>=0){
- Kmer*kmer=&(m_ed->m_EXTENSION_extension[position]);
+ Kmer theKmer;
+ m_ed->m_EXTENSION_extension.at(position,&theKmer);
+
+ Kmer*kmer=&theKmer;
CoverageDepth depth=m_ed->m_extensionCoverageValues[position];
cout<<" "<<position<<" "<<kmer->idToWord(wordSize,colored)<<" ";
@@ -2360,7 +2426,7 @@ int SeedExtender::chooseWithSeed(){
return IMPOSSIBLE_CHOICE;
}
-void SeedExtender::finalizeExtensions(vector<AssemblySeed>*seeds,FusionData*fusionData){
+void SeedExtender::finalizeExtensions(vector<GraphPath>*seeds,FusionData*fusionData){
MACRO_COLLECT_PROFILING_INFORMATION();
@@ -2386,7 +2452,6 @@ void SeedExtender::finalizeExtensions(vector<AssemblySeed>*seeds,FusionData*fusi
printf("Rank %i extended %i seeds out of %i (%.2f%%)\n",m_parameters->getRank(),
m_extended,(int)seeds->size(),ratio);
- fflush(stdout);
MACRO_COLLECT_PROFILING_INFORMATION();
if(m_parameters->showMemoryUsage()){
@@ -2414,7 +2479,7 @@ void SeedExtender::finalizeExtensions(vector<AssemblySeed>*seeds,FusionData*fusi
(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_EXTENSION_IS_DONE,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_derivative.writeFile(&cout);
@@ -2423,7 +2488,7 @@ void SeedExtender::finalizeExtensions(vector<AssemblySeed>*seeds,FusionData*fusi
/** write extensions for debugging purposes */
if(m_parameters->hasOption("-write-extensions")){
ostringstream fileName;
- fileName<<m_parameters->getPrefix()<<"Rank"<<m_parameters->getRank()<<"RayExtensions.fasta";
+ fileName<<m_parameters->getPrefix()<<"Rank"<<m_parameters->getRank()<<".RayExtensions.fasta";
ofstream f(fileName.str().c_str());
for(int i=0;i<(int)m_ed->m_EXTENSION_identifiers.size();i++){
@@ -2441,13 +2506,17 @@ void SeedExtender::finalizeExtensions(vector<AssemblySeed>*seeds,FusionData*fusi
MACRO_COLLECT_PROFILING_INFORMATION();
}
-void SeedExtender::initializeExtensions(vector<AssemblySeed>*seeds){
+void SeedExtender::initializeExtensions(vector<GraphPath>*seeds){
MACRO_COLLECT_PROFILING_INFORMATION();
m_ed->m_EXTENSION_initiated=true;
m_ed->m_EXTENSION_currentSeedIndex=0;
m_ed->m_EXTENSION_currentPosition=0;
+ m_nucleotidesAssembled=0;
+ m_lastNucleotideAssembled=0;
+ m_nucleotidePeriod=1000;
+
MACRO_COLLECT_PROFILING_INFORMATION();
// this will probably needs to be sliced...
@@ -2455,7 +2524,9 @@ void SeedExtender::initializeExtensions(vector<AssemblySeed>*seeds){
MACRO_COLLECT_PROFILING_INFORMATION();
- Kmer*theKmer=m_ed->m_EXTENSION_currentSeed.at(m_ed->m_EXTENSION_currentPosition);
+ Kmer theKmerObject;
+ m_ed->m_EXTENSION_currentSeed.at(m_ed->m_EXTENSION_currentPosition,&theKmerObject);
+ Kmer*theKmer=&theKmerObject;
(m_seedingData->m_SEEDING_currentVertex)=*theKmer;
@@ -2495,7 +2566,7 @@ void SeedExtender::call_RAY_MPI_TAG_ASK_IS_ASSEMBLED(Message*message){
message2[outputPosition++]=m_subgraph->isAssembledByGreaterRank(&vertex,origin);
Message aMessage(message2,outputPosition,source,RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}
void SeedExtender::call_RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY(Message*message){
@@ -2506,9 +2577,11 @@ void SeedExtender::call_RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY(Message*message){
int position=0;
- (m_ed->m_EXTENSION_vertexIsAssembledResult)=(bool)incoming[position++];
- m_theProcessIsRedundantByAGreaterAndMightyRank=(bool)incoming[position++];
+ bool isAssembled = (bool)incoming[position++];
+
+ (m_ed->m_EXTENSION_vertexIsAssembledResult) = isAssembled;
+ m_theProcessIsRedundantByAGreaterAndMightyRank = isAssembled;
#ifdef CONFIG_DEBUG_SEED_EXTENSION
if(m_theProcessIsRedundantByAGreaterAndMightyRank)
@@ -2524,12 +2597,14 @@ void SeedExtender::call_RAY_MPI_TAG_ADD_GRAPH_PATH(Message*message){
MessageUnit*buffer=message->getBuffer();
Rank source=message->getSource();
- PathHandle pathHandle=buffer[0];
- int pathLength=buffer[1];
- int flows=buffer[2];
+
+ int position = 0;
+ PathHandle pathHandle=buffer[position++];
+ int pathLength=buffer[position++];
+ int flows=buffer[position++];
if(!m_pathFile.is_open()){
-
+
ostringstream fileName;
fileName<<m_parameters->getPrefix()<<"/ParallelPaths.txt";
@@ -2540,15 +2615,21 @@ void SeedExtender::call_RAY_MPI_TAG_ADD_GRAPH_PATH(Message*message){
assert(m_pathFile.is_open());
#endif
- m_pathFileBuffer<<"#Rank Path Length"<<endl;
+ m_pathFileBuffer<<"#Rank Path LengthInKmers Flows"<<endl;
}
m_pathFileBuffer<<source<<" "<<pathHandle<<" "<<pathLength<<" "<<flows<<endl;
}
-void SeedExtender::skipSeed(vector<AssemblySeed>*seeds){
- cout<<"Rank "<<m_parameters->getRank()<<" skips seed ["<<m_ed->m_EXTENSION_currentSeedIndex<<"/"<<
- (*seeds).size()<<"]"<<endl;
+void SeedExtender::skipSeed(vector<GraphPath>*seeds){
+
+ bool verbose=m_ed->m_EXTENSION_currentSeed.size()>=MINIMUM_UNITS_FOR_VERBOSITY;
+
+ if(verbose){
+ cout<<"Rank "<<m_parameters->getRank()<<" skips seed [";
+ cout<<m_ed->m_EXTENSION_currentSeedIndex<<"/";
+ cout<<(*seeds).size()<<"]"<<endl;
+ }
m_ed->m_EXTENSION_currentSeedIndex++;// skip the current one.
m_ed->m_EXTENSION_currentPosition=0;
@@ -2560,7 +2641,9 @@ void SeedExtender::skipSeed(vector<AssemblySeed>*seeds){
if(m_ed->m_EXTENSION_currentSeedIndex<(int)(*seeds).size()){
m_ed->m_EXTENSION_currentSeed=(*seeds)[m_ed->m_EXTENSION_currentSeedIndex];
- Kmer*theKmer=m_ed->m_EXTENSION_currentSeed.at(m_ed->m_EXTENSION_currentPosition);
+ Kmer object;
+ m_ed->m_EXTENSION_currentSeed.at(m_ed->m_EXTENSION_currentPosition,&object);
+ Kmer*theKmer=&object;
m_seedingData->m_SEEDING_currentVertex=*theKmer;
}
m_ed->m_previouslyFlowedVertices=0;
@@ -2571,6 +2654,9 @@ void SeedExtender::skipSeed(vector<AssemblySeed>*seeds){
}
void SeedExtender::registerPlugin(ComputeCore*core){
+
+ m_core=core;
+
PluginHandle plugin=core->allocatePluginHandle();
m_plugin=plugin;
@@ -2590,9 +2676,6 @@ void SeedExtender::registerPlugin(ComputeCore*core){
RAY_MPI_TAG_CONTIG_INFO_REPLY=core->allocateMessageTagHandle(plugin);
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_CONTIG_INFO_REPLY,"RAY_MPI_TAG_CONTIG_INFO_REPLY");
- RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY=core->allocateMessageTagHandle(plugin);
- core->setMessageTagSymbol(plugin,RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY,"RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY");
-
m_switchMan=core->getSwitchMan();
RAY_MPI_TAG_ASK_IS_ASSEMBLED=core->allocateMessageTagHandle(plugin);
@@ -2603,7 +2686,9 @@ void SeedExtender::registerPlugin(ComputeCore*core){
core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY, __GetAdapter(SeedExtender,RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY));
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY,"RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY");
+// this needs to be started here because it is shared between plugins
+ m_core->setObjectSymbol(m_plugin,&m_directionsAllocatorInstance,"/RayAssembler/ObjectStore/directionMemoryPool.ray");
}
void SeedExtender::resolveSymbols(ComputeCore*core){
@@ -2623,11 +2708,20 @@ void SeedExtender::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_VERTEX_READS_FROM_LIST_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_VERTEX_READS_FROM_LIST_REPLY");
RAY_MPI_TAG_VERTEX_READS_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_VERTEX_READS_REPLY");
RAY_MPI_TAG_CONTIG_INFO_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_CONTIG_INFO_REPLY");
- RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY");
-
RAY_MPI_TAG_ASK_IS_ASSEMBLED=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_ASK_IS_ASSEMBLED");
RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY");
__BindPlugin(SeedExtender);
+
+ __BindAdapter(SeedExtender,RAY_SLAVE_MODE_EXTENSION); /**/
+ __BindAdapter(SeedExtender,RAY_MPI_TAG_ADD_GRAPH_PATH);
+ __BindAdapter(SeedExtender,RAY_MPI_TAG_ASK_IS_ASSEMBLED); /**/
+ __BindAdapter(SeedExtender,RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY); /**/
+
+ m_parameters=(Parameters*)m_core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/Parameters.ray");
+
+ int directionAllocatorChunkSize=4194304; // 4 MiB
+ m_directionsAllocatorInstance.constructor(directionAllocatorChunkSize,"RAY_MALLOC_TYPE_WAVE_ALLOCATOR",
+ m_parameters->showMemoryAllocations());
}
diff --git a/code/plugin_SeedExtender/SeedExtender.h b/code/SeedExtender/SeedExtender.h
similarity index 73%
rename from code/plugin_SeedExtender/SeedExtender.h
rename to code/SeedExtender/SeedExtender.h
index 22141ae..e3b4bf5 100644
--- a/code/plugin_SeedExtender/SeedExtender.h
+++ b/code/SeedExtender/SeedExtender.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,41 +22,67 @@
#ifndef _SeedExtender
#define _SeedExtender
-class FusionData;
-class DepthFirstSearchData;
+#include "ReadFetcher.h"
+#include "BubbleData.h"
+#include "DepthFirstSearchData.h"
+#include "BubbleTool.h"
+#include "OpenAssemblerChooser.h"
+#include "VertexMessenger.h"
+#include "ExtensionData.h"
+
+#include <code/SequencesLoader/ReadHandle.h>
+#include <code/Mock/common_functions.h>
+#include <code/Mock/constants.h>
+#include <code/Mock/Parameters.h>
+#include <code/SeedingData/GraphPath.h>
+#include <code/SeedingData/SeedingData.h>
+#include <code/FusionData/FusionData.h>
+#include <code/VerticesExtractor/GridTable.h>
+
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/memory/MyAllocator.h>
+#include <RayPlatform/profiling/Derivative.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/profiling/Profiler.h>
#include <vector>
#include <fstream>
+using namespace std;
-#include <application_core/common_functions.h>
-#include <communication/Message.h>
-#include <profiling/Profiler.h>
-#include <plugin_SeedExtender/VertexMessenger.h>
-#include <plugin_SeedExtender/ExtensionData.h>
-#include <application_core/Parameters.h>
-#include <plugin_SeedingData/AssemblySeed.h>
-#include <memory/RingAllocator.h>
-#include <memory/MyAllocator.h>
-#include <profiling/Derivative.h>
-#include <plugin_SeedExtender/ReadFetcher.h>
-#include <plugin_FusionData/FusionData.h>
-#include <plugin_SeedExtender/BubbleData.h>
-#include <plugin_SeedExtender/DepthFirstSearchData.h>
-#include <plugin_SeedExtender/BubbleTool.h>
-#include <plugin_SeedExtender/OpenAssemblerChooser.h>
-#include <plugin_VerticesExtractor/GridTable.h>
-#include <plugin_SeedingData/SeedingData.h>
-#include <handlers/SlaveModeHandler.h>
-#include <core/ComputeCore.h>
+class FusionData;
+class DepthFirstSearchData;
+class SeedingData;
-using namespace std;
+__DeclarePlugin(SeedExtender);
+
+__DeclareSlaveModeAdapter(SeedExtender,RAY_SLAVE_MODE_EXTENSION); /**/
+__DeclareMessageTagAdapter(SeedExtender,RAY_MPI_TAG_ADD_GRAPH_PATH);
+__DeclareMessageTagAdapter(SeedExtender,RAY_MPI_TAG_ASK_IS_ASSEMBLED); /**/
+__DeclareMessageTagAdapter(SeedExtender,RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY); /**/
/*
* Performs the extension of seeds.
+ *
+ * This class includes modifications for the workflow
+ * called "Ray Meta"
+ *
+ * \see http://genomebiology.com/2012/13/12/R122
+ *
* \author Sébastien Boisvert
*/
class SeedExtender: public CorePlugin {
+ uint64_t m_nucleotidesAssembled;
+ uint64_t m_lastNucleotideAssembled;
+ int m_nucleotidePeriod;
+
+ __AddAdapter(SeedExtender,RAY_SLAVE_MODE_EXTENSION); /**/
+ __AddAdapter(SeedExtender,RAY_MPI_TAG_ADD_GRAPH_PATH);
+ __AddAdapter(SeedExtender,RAY_MPI_TAG_ASK_IS_ASSEMBLED); /**/
+ __AddAdapter(SeedExtender,RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY); /**/
+
/** hot skipping technology (TM) **/
int m_redundantProcessingVirtualMachineCycles;
@@ -66,11 +92,9 @@ class SeedExtender: public CorePlugin {
void configureTheBeautifulHotSkippingTechnology();
-
Rank m_rank;
MessageTag RAY_MPI_TAG_CONTIG_INFO_REPLY;
- MessageTag RAY_MPI_TAG_GET_CONTIG_CHUNK_REPLY;
MessageTag RAY_MPI_TAG_ASK_IS_ASSEMBLED;
MessageTag RAY_MPI_TAG_ASK_IS_ASSEMBLED_REPLY;
@@ -90,10 +114,8 @@ class SeedExtender: public CorePlugin {
SlaveMode RAY_SLAVE_MODE_EXTENSION;
SlaveMode RAY_SLAVE_MODE_DO_NOTHING;
-
-
// all these parameters are not attributes.
- vector<AssemblySeed>*m_seeds;
+ vector<GraphPath>*m_seeds;
Kmer*m_currentVertex;
FusionData*m_fusionData;
RingAllocator*m_outboxAllocator;
@@ -124,7 +146,7 @@ class SeedExtender: public CorePlugin {
SeedingData*m_seedingData;
/* for sliced computation */
- AssemblySeed m_complementedSeed;
+ GraphPath m_complementedSeed;
void printSeed();
@@ -154,13 +176,15 @@ class SeedExtender: public CorePlugin {
Parameters*m_parameters;
BubbleTool m_bubbleTool;
ExtensionData*m_ed;
- MyAllocator*m_directionsAllocator;
+
+ // allocator for directions in the de Bruijn graph
+ MyAllocator m_directionsAllocatorInstance;
set<ReadHandle> m_matesToMeet;
bool m_messengerInitiated;
VertexMessenger m_vertexMessenger;
- set<ReadHandle> m_eliminatedSeeds;
+ set<PathHandle> m_eliminatedSeeds;
map<int,vector<ReadHandle> >m_expiredReads;
void inspect(ExtensionData*ed,Kmer*currentVertex);
@@ -184,13 +208,13 @@ map<Kmer,set<Kmer> >*arcs,map<Kmer,int>*coverages,int depth,set<Kmer>*visited);
void processExpiredReads();
int chooseWithSeed();
- void initializeExtensions(vector<AssemblySeed>*seeds);
- void finalizeExtensions(vector<AssemblySeed>*seeds,FusionData*fusionData);
+ void initializeExtensions(vector<GraphPath>*seeds);
+ void finalizeExtensions(vector<GraphPath>*seeds,FusionData*fusionData);
void checkedCurrentVertex();
- void skipSeed(vector<AssemblySeed>*seeds);
+ void skipSeed(vector<GraphPath>*seeds);
/** store the current extension and fetch the next one **/
- void storeExtensionAndGetNextOne(ExtensionData*ed,int theRank,vector<AssemblySeed>*seeds,Kmer*currentVertex,
+ void storeExtensionAndGetNextOne(ExtensionData*ed,int theRank,vector<GraphPath>*seeds,Kmer*currentVertex,
BubbleData*bubbleData);
/** given the current vertex, enumerate the choices **/
@@ -203,7 +227,7 @@ int wordSize);
/** check if the current vertex is already assembled **/
void checkIfCurrentVertexIsAssembled(ExtensionData*ed,StaticVector*outbox,RingAllocator*outboxAllocator,
int*outgoingEdgeIndex,int*last_value,Kmer*currentVertex,int theRank,bool*vertexCoverageRequested,
- int wordSize,int size,vector<AssemblySeed>*seeds);
+ int wordSize,int size,vector<GraphPath>*seeds);
/** mark the current vertex as assembled **/
void markCurrentVertexAsAssembled(Kmer *currentVertex,RingAllocator*outboxAllocator,int*outgoingEdgeIndex,
@@ -211,13 +235,13 @@ int wordSize);
bool*vertexCoverageReceived,int*receivedVertexCoverage,
bool*edgesRequested,
vector<Kmer>*receivedOutgoingEdges,Chooser*chooser,
-BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,vector<AssemblySeed>*seeds);
+BubbleData*bubbleData,int minimumCoverage,OpenAssemblerChooser*oa,int wordSize,vector<GraphPath>*seeds);
/** choose where to go next **/
void doChoice(RingAllocator*outboxAllocator,int*outgoingEdgeIndex,StaticVector*outbox,Kmer*currentVertex,
BubbleData*bubbleData,int theRank,int wordSize,
ExtensionData*ed,int minimumCoverage,OpenAssemblerChooser*oa,Chooser*chooser,
- vector<AssemblySeed>*seeds,
+ vector<GraphPath>*seeds,
bool*edgesRequested,bool*vertexCoverageRequested,bool*vertexCoverageReceived,int size,
int*receivedVertexCoverage,bool*edgesReceived,vector<Kmer>*receivedOutgoingEdges);
@@ -244,10 +268,10 @@ public:
set<PathHandle>*getEliminatedSeeds();
- void constructor(Parameters*parameters,MyAllocator*m_directionsAllocator,ExtensionData*ed,GridTable*table,StaticVector*inbox,
+ void constructor(Parameters*parameters,ExtensionData*ed,GridTable*table,StaticVector*inbox,
Profiler*profiler,StaticVector*outbox,SeedingData*seedingData,int*mode,
bool*vertexCoverageRequested,bool*vertexCoverageReceived,RingAllocator*outboxAllocator,
- FusionData*fusionData,vector<AssemblySeed>*seeds,BubbleData*bubbleData,
+ FusionData*fusionData,vector<GraphPath>*seeds,BubbleData*bubbleData,
bool*edgesRequested,bool*edgesReceived,int*outgoingEdgeIndex,Kmer*currentVertex,
int*receivedVertexCoverage,vector<Kmer>*receivedOutgoingEdges,Chooser*chooser,
OpenAssemblerChooser*oa);
diff --git a/code/plugin_SeedExtender/TipWatchdog.cpp b/code/SeedExtender/TipWatchdog.cpp
similarity index 96%
rename from code/plugin_SeedExtender/TipWatchdog.cpp
rename to code/SeedExtender/TipWatchdog.cpp
index 33f1c4e..a8eca83 100644
--- a/code/plugin_SeedExtender/TipWatchdog.cpp
+++ b/code/SeedExtender/TipWatchdog.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -18,7 +18,7 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_SeedExtender/TipWatchdog.h>
+#include "TipWatchdog.h"
bool TipWatchdog::getApproval(ExtensionData*ed,DepthFirstSearchData*dfsData,int minimumCoverage,Kmer SEEDING_currentVertex,
int w,BubbleData*bubbleData){
diff --git a/code/plugin_SeedExtender/TipWatchdog.h b/code/SeedExtender/TipWatchdog.h
similarity index 82%
rename from code/plugin_SeedExtender/TipWatchdog.h
rename to code/SeedExtender/TipWatchdog.h
index bc3a018..2f22d7f 100644
--- a/code/plugin_SeedExtender/TipWatchdog.h
+++ b/code/SeedExtender/TipWatchdog.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,9 +22,9 @@
#ifndef _TipWatchdog
#define _TipWatchdog
-#include <plugin_SeedExtender/ExtensionData.h>
-#include <plugin_SeedExtender/DepthFirstSearchData.h>
-#include <plugin_SeedExtender/BubbleData.h>
+#include "ExtensionData.h"
+#include "DepthFirstSearchData.h"
+#include "BubbleData.h"
/*
* Watch for unwanted things.
diff --git a/code/plugin_SeedExtender/VertexMessenger.cpp b/code/SeedExtender/VertexMessenger.cpp
similarity index 92%
rename from code/plugin_SeedExtender/VertexMessenger.cpp
rename to code/SeedExtender/VertexMessenger.cpp
index 2183d0f..5add909 100644
--- a/code/plugin_SeedExtender/VertexMessenger.cpp
+++ b/code/SeedExtender/VertexMessenger.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -18,9 +18,10 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_SeedExtender/VertexMessenger.h>
-#include <communication/Message.h>
-#include <core/OperatingSystem.h>
+#include "VertexMessenger.h"
+
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/core/OperatingSystem.h>
#include <stdint.h>
#include <assert.h>
@@ -40,10 +41,10 @@ void VertexMessenger::work(){
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(3*sizeof(MessageUnit));
int j=0;
m_vertex.pack(message,&j);
- message[j++]=m_waveId;
+ message[j++]=m_waveId.getValue();
message[j++]=m_wavePosition;
Message aMessage(message,j,m_destination,RAY_MPI_TAG_VERTEX_INFO,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
}else if(!m_receivedBasicInfo &&m_inbox->size()==1&&m_inbox->at(0)->getTag()==RAY_MPI_TAG_VERTEX_INFO_REPLY){
m_receivedBasicInfo=true;
@@ -88,6 +89,10 @@ void VertexMessenger::work(){
m_annotations.clear();
m_isDone=true;
+/*
+ * We fetch all the reads for nice objects. For repeated objects,
+ * we only want those for which we have a matching pair.
+ */
}else if(m_coverageValue>= 3* m_peakCoverage){
getReadsForRepeatedVertex();
}else{
@@ -107,7 +112,7 @@ void VertexMessenger::getReadsForRepeatedVertex(){
int processed=0;
while(processed<maximumMates&&m_mateIterator!=m_matesToMeet->end()){
ReadHandle mate=*m_mateIterator;
- message[j+1+processed]=mate;
+ message[j+1+processed]=mate.getValue();
processed++;
m_mateIterator++;
}
@@ -117,7 +122,7 @@ void VertexMessenger::getReadsForRepeatedVertex(){
*/
Message aMessage(message,j+1+processed,
m_destination,RAY_MPI_TAG_VERTEX_READS_FROM_LIST,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_requestedReads=true;
m_receivedReads=false;
}else if(!m_receivedReads&&m_inbox->size()==1&&m_inbox->at(0)->getTag()==RAY_MPI_TAG_VERTEX_READS_FROM_LIST_REPLY){
@@ -160,7 +165,7 @@ void VertexMessenger::getReadsForUniqueVertex(){
m_vertex.pack(message,&j);
message[j++]=(MessageUnit)m_pointer;
Message aMessage(message,j,m_destination,RAY_MPI_TAG_VERTEX_READS,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_requestedReads=true;
m_receivedReads=false;
@@ -247,5 +252,5 @@ void VertexMessenger::constructor(Kmer vertex,PathHandle wave,int pos,
m_annotations.clear();
m_isDone=false;
m_requestedBasicInfo=false;
- m_destination=m_parameters->_vertexRank(&m_vertex);
+ m_destination=m_parameters->vertexRank(&m_vertex);
}
diff --git a/code/plugin_SeedExtender/VertexMessenger.h b/code/SeedExtender/VertexMessenger.h
similarity index 88%
rename from code/plugin_SeedExtender/VertexMessenger.h
rename to code/SeedExtender/VertexMessenger.h
index 998bd0d..1526e34 100644
--- a/code/plugin_SeedExtender/VertexMessenger.h
+++ b/code/SeedExtender/VertexMessenger.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -21,12 +21,15 @@
#ifndef _VertexMessenger
#define _VertexMessenger
+#include <code/SequencesLoader/ReadHandle.h>
+#include <code/SequencesIndexer/ReadAnnotation.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/memory/RingAllocator.h>
+
#include <stdint.h>
-#include <plugin_SequencesIndexer/ReadAnnotation.h>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <structures/StaticVector.h>
-#include <memory/RingAllocator.h>
-#include <application_core/Parameters.h>
#include <vector>
#include <set>
using namespace std;
diff --git a/code/SeedingData/GraphPath.cpp b/code/SeedingData/GraphPath.cpp
new file mode 100644
index 0000000..cbf13e0
--- /dev/null
+++ b/code/SeedingData/GraphPath.cpp
@@ -0,0 +1,743 @@
+/*
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+*/
+
+#include "GraphPath.h"
+
+#include <code/Mock/constants.h>
+#include <code/Mock/common_functions.h>
+
+#include <RayPlatform/core/statistics.h>
+
+#include <iostream>
+#include <map>
+using namespace std;
+
+#include <string.h>
+
+#ifdef CONFIG_ASSERT
+#include <assert.h>
+#endif
+
+//#define CONFIG_PATH_VERBOSITY
+//#define CHECK_BUG_142
+
+GraphPath::GraphPath(){
+ m_hasPeakCoverage=false;
+
+ m_kmerLength=0;
+ m_errorRaised=false;
+
+#ifdef CONFIG_PATH_STORAGE_BLOCK
+ m_size=0;
+#endif
+
+ m_deleted = false;
+}
+
+bool GraphPath::isDeleted(){
+ return m_deleted;
+}
+
+void GraphPath::markAsDeleted(){
+ m_deleted = true;
+}
+
+int GraphPath::size()const{
+
+#ifdef CONFIG_PATH_STORAGE_DEFAULT
+ return m_vertices.size();
+#elif defined(CONFIG_PATH_STORAGE_BLOCK)
+ return m_size;
+#endif
+
+}
+
+void GraphPath::at(int i,Kmer*value)const{
+
+#ifdef CONFIG_PATH_STORAGE_DEFAULT
+ (*value)=m_vertices.at(i);
+#elif defined(CONFIG_PATH_STORAGE_BLOCK)
+
+ readObjectInBlock(i,value);
+#endif
+}
+
+CoverageDepth GraphPath::getCoverageAt(int position)const{
+
+ if(m_coverageValues.size()==0)
+ return 0;
+
+ return m_coverageValues[position];
+}
+
+bool GraphPath::canBeAdded(const Kmer*object)const{
+
+ if(size()==0)
+ return true;
+
+ Kmer lastKmer;
+ int position=size()-1;
+ at(position,&lastKmer);
+
+ return lastKmer.canHaveChild(object,m_kmerLength);
+}
+
+void GraphPath::push_back(const Kmer*a){
+
+#ifdef CONFIG_ASSERT
+ assert(m_kmerLength!=0);
+#endif
+
+ if(!canBeAdded(a)){
+ if(!m_errorRaised){
+ cout<<"Error: can not add "<<a->idToWord(m_kmerLength,false)<<endl;
+ cout<<"last objects:"<<endl;
+ int count=16;
+ int iterator=size()-count;
+ while(iterator<size()){
+ Kmer theObject;
+ at(iterator,&theObject);
+
+ cout<<" ["<<iterator<<"] ------> "<<theObject.idToWord(m_kmerLength,false)<<endl;
+
+ iterator++;
+ }
+
+ m_errorRaised=true;
+ }
+
+ return;
+ }
+
+#ifdef CONFIG_PATH_STORAGE_DEFAULT
+ m_vertices.push_back(*a);
+#elif defined(CONFIG_PATH_STORAGE_BLOCK)
+
+ writeObjectInBlock(a);
+#endif
+}
+
+void GraphPath::getVertices(vector<Kmer>*vertices)const{
+ for(int i=0;i<size();i++){
+ Kmer kmer;
+ at(i,&kmer);
+ vertices->push_back(kmer);
+ }
+}
+
+void GraphPath::clear(){
+#ifdef CONFIG_PATH_STORAGE_DEFAULT
+ m_vertices.clear();
+#elif defined(CONFIG_PATH_STORAGE_BLOCK)
+ m_blocks.clear();
+ m_size=0;
+ m_kmerLength=0;
+#endif
+}
+
+void GraphPath::resetCoverageValues(){
+ m_coverageValues.clear();
+}
+
+void GraphPath::computePeakCoverage(){
+
+ int ALGORITHM_MODE=0;
+ int ALGORITHM_MEAN=1;
+ int ALGORITHM_STAGGERED_MEAN=2;
+
+ int selectedAlgorithm=ALGORITHM_STAGGERED_MEAN;
+
+ #ifdef CONFIG_ASSERT
+ if((int)m_coverageValues.size()!=size())
+ cout<<"Error: there are "<<size()<<" objects, but only "<<m_coverageValues.size()<<" coverage values"<<endl;
+
+ assert((int)m_coverageValues.size() == size());
+
+ for(int i = 0 ; i < (int)size() ; ++i) {
+
+ CoverageDepth coverage = getCoverageAt(i);
+
+ assert(coverage > 0);
+ }
+ #endif
+
+ // the default is to use the weighted mean algorithm
+
+ if(selectedAlgorithm==ALGORITHM_MODE){
+ computePeakCoverageUsingMode();
+ }else if(selectedAlgorithm==ALGORITHM_MEAN){
+ computePeakCoverageUsingMean();
+ }else if(selectedAlgorithm==ALGORITHM_STAGGERED_MEAN){
+ computePeakCoverageUsingStaggeredMean();
+ }
+
+ m_hasPeakCoverage=true;
+}
+
+CoverageDepth GraphPath::getPeakCoverage()const{
+
+ #ifdef CONFIG_ASSERT
+ assert(m_hasPeakCoverage == true);
+ #endif
+
+ return m_peakCoverage;
+}
+
+void GraphPath::addCoverageValue(CoverageDepth value){
+ m_coverageValues.push_back(value);
+}
+
+void GraphPath::computePeakCoverageUsingMode(){
+
+ map<CoverageDepth,int> frequencies;
+
+ for(int i=0;i<(int)m_coverageValues.size();i++){
+ frequencies[m_coverageValues[i]]++;
+ }
+
+ int best=-1;
+
+ for(map<CoverageDepth,int>::iterator i=frequencies.begin();
+ i!=frequencies.end();i++){
+
+ if(frequencies.count(best)==0 || i->second > frequencies[best]){
+ best=i->first;
+ }
+
+ #ifdef CONFIG_VERBOSITY_FOR_SEEDS
+ cout<<i->first<<" "<<i->second<<endl;
+ #endif
+ }
+
+ #ifdef CONFIG_VERBOSITY_FOR_SEEDS
+ cout<<"mode= "<<best<<" length= "<<size()<<endl;
+ #endif
+
+ m_peakCoverage=best;
+
+}
+
+void GraphPath::computePeakCoverageUsingMean(){
+
+ map<CoverageDepth,int> frequencies;
+
+ for(int i=0;i<(int)m_coverageValues.size();i++){
+ frequencies[m_coverageValues[i]]++;
+ }
+
+ LargeCount sum=0;
+ LargeCount count=0;
+
+ for(map<CoverageDepth,int>::iterator i=frequencies.begin();
+ i!=frequencies.end();i++){
+
+ CoverageDepth coverage=i->first;
+ LargeCount frequency=i->second;
+
+ #ifdef CONFIG_VERBOSITY_FOR_SEEDS
+ cout<<coverage<<" "<<frequency<<endl;
+ #endif
+
+ sum+=coverage*frequency;
+ count+=frequency;
+ }
+
+ #ifdef CONFIG_ASSERT
+ assert(m_coverageValues.size()>=1);
+ assert(count!=0);
+ assert(count>0);
+ assert(sum > 0);
+ #endif
+
+ CoverageDepth mean=( sum / count );
+
+ cout<<"mean= "<<mean <<" length= "<<size()<<endl;
+
+ m_peakCoverage=mean;
+}
+
+void GraphPath::reserve(int size){
+#ifdef CONFIG_PATH_STORAGE_DEFAULT
+ m_vertices.reserve(size);
+#endif
+ m_coverageValues.reserve(size);
+
+}
+
+void GraphPath::computePeakCoverageUsingStaggeredMean(){
+
+ map<CoverageDepth,int> frequencies;
+ uint64_t totalCount=0;
+
+ for(int i=0;i<(int)m_coverageValues.size();i++){
+ frequencies[m_coverageValues[i]]++;
+
+ totalCount++;
+ }
+
+ CoverageDepth NO_VALUE=0;
+
+ CoverageDepth mean=NO_VALUE;
+ uint64_t objectsOnRight=0;
+
+/*
+ * 10% is arbitrary, anything between 5% and 45% will be fine.
+ */
+ int thresholdInPercentage=10;
+ uint64_t minimumRequired=thresholdInPercentage*totalCount/100;
+
+ while(1){
+ LargeCount sum=0;
+ LargeCount count=0;
+
+ for(map<CoverageDepth,int>::iterator i=frequencies.begin();
+ i!=frequencies.end();i++){
+
+ CoverageDepth coverage=i->first;
+ LargeCount frequency=i->second;
+
+ #ifdef CONFIG_VERBOSITY_FOR_SEEDS
+ cout<<coverage<<" "<<frequency<<endl;
+ #endif
+
+/*
+ * Skip the repeats.
+ */
+ if(mean!=NO_VALUE && coverage>=mean)
+ continue;
+
+ sum+=coverage*frequency;
+ count+=frequency;
+ }
+
+ #ifdef CONFIG_ASSERT
+ assert(m_coverageValues.size()>=1);
+ assert(count!=0);
+ assert(count>0);
+ assert(sum > 0);
+ #endif
+
+ mean=( sum / count );
+
+/*
+ * Count the number of objects that are at least the mean.
+ */
+
+ objectsOnRight=0;
+
+ for(map<CoverageDepth,int>::iterator i=frequencies.begin();
+ i!=frequencies.end();i++){
+
+ CoverageDepth coverage=i->first;
+ LargeCount frequency=i->second;
+
+ if(coverage>=mean)
+ objectsOnRight+=frequency;
+ }
+
+ if(objectsOnRight>=minimumRequired){
+ break;
+ }
+ }
+
+ #ifdef CONFIG_VERBOSITY_FOR_SEEDS
+ cout<<"mean= "<<mean <<" length= "<<size()<<" onTheRight= "<<objectsOnRight<<"/"<<totalCount<<endl;
+ #endif
+
+ m_peakCoverage=mean;
+}
+
+void GraphPath::setKmerLength(int kmerLength){
+ m_kmerLength=kmerLength;
+
+ #ifdef CONFIG_ASSERT
+ assert(kmerLength!=0);
+ assert(m_kmerLength!=0);
+ #endif
+}
+
+#ifdef CONFIG_PATH_STORAGE_BLOCK
+
+void GraphPath::writeObjectInBlock(const Kmer*a){
+
+ #ifdef CONFIG_ASSERT
+ assert(m_kmerLength!=0);
+ #endif
+
+#ifdef CHECK_BUG_142
+ string copyA="AGGAAGAACCTGCTGAGGAACAAGAAGGTCAACTGCCTGGACTGTAATACC";
+ string copyB=a->idToWord(m_kmerLength,false);
+ if(copyA==copyB)
+ cout<<"[GraphPath::writeObjectInBlock] returns "<<copyB<<endl;
+#endif
+
+ if(m_size==0){
+ #ifdef CONFIG_ASSERT
+ assert(m_blocks.size()==0);
+ #endif
+
+ addBlock();
+ string sequence=a->idToWord(m_kmerLength, false);
+
+ for(int blockPosition=0;blockPosition<m_kmerLength;blockPosition++){
+ writeSymbolInBlock(blockPosition, sequence[blockPosition]);
+ }
+ }else{
+ #ifdef CONFIG_ASSERT
+ assert(m_size>=1);
+ assert(a!=NULL);
+ assert(m_kmerLength!=0);
+ #endif
+
+ char lastSymbol=a->getLastSymbol(m_kmerLength,false);
+ int usedSymbols=size()+m_kmerLength-1;
+
+ #ifdef CONFIG_ASSERT
+ assert(usedSymbols>=m_kmerLength);
+ assert(m_blocks.size()>=1);
+ #endif
+
+ int allocatedSymbols=m_blocks.size()*getBlockSize();
+
+ #ifdef CONFIG_ASSERT
+ assert(allocatedSymbols>=getBlockSize());
+ #endif
+
+ if(usedSymbols+1>allocatedSymbols){
+ addBlock();
+ allocatedSymbols=m_blocks.size()*getBlockSize();
+ }
+
+ #ifdef CONFIG_ASSERT
+ assert(usedSymbols+1<=allocatedSymbols);
+ assert(allocatedSymbols>=getBlockSize());
+ #endif
+
+ int position=usedSymbols;
+
+ #ifdef CONFIG_ASSERT
+ assert(position<allocatedSymbols);
+ #endif
+
+ writeSymbolInBlock(position,lastSymbol);
+ }
+
+ m_size++;
+
+#ifdef CONFIG_ASSERT
+ Kmer addedObject;
+ at(size()-1,&addedObject);
+
+ if((*a)!=addedObject){
+ cout<<"Error: expected: "<<a->idToWord(m_kmerLength,false)<<endl;
+ cout<<"actual: "<<addedObject.idToWord(m_kmerLength,false)<<" at position "<<size()-1<<endl;
+ cout<<"kmerLength: "<<m_kmerLength<<" blockSize: "<<getBlockSize()<<endl;
+ int i=size()-1;
+ int j=0;
+ cout<<"dump:"<<endl;
+ while(i-j>=0 && j<10){
+ Kmer theObject;
+ at(i-j,&theObject);
+
+ cout<<" ["<<i-j<<"] ------> "<<theObject.idToWord(m_kmerLength,false)<<endl;
+
+ j++;
+ }
+ }
+
+ assert((*a)==addedObject);
+#endif
+}
+
+int GraphPath::getBlockSize()const{
+ return CONFIG_PATH_BLOCK_SIZE;
+}
+
+void GraphPath::readObjectInBlock(int position,Kmer*object)const{
+
+ #ifdef CONFIG_ASSERT
+ assert(position<size());
+ assert(position>=0);
+ assert(m_kmerLength!=0);
+ #endif
+
+ char kmer[CONFIG_MAXKMERLENGTH+1];
+
+ for(int i=0;i<m_kmerLength;i++){
+ kmer[i]=readSymbolInBlock(position+i);
+ }
+
+ kmer[m_kmerLength]='\0';
+
+#ifdef CONFIG_PATH_VERBOSITY
+ cout<<"Object: "<<kmer<<endl;
+#endif
+
+ (*object)=wordId(kmer);
+}
+
+char GraphPath::readSymbolInBlock(int position)const{
+
+ int globalPosition=position*BITS_PER_NUCLEOTIDE;
+ int numberOfBitsPerBlock=CONFIG_PATH_BLOCK_SIZE*BITS_PER_NUCLEOTIDE;
+
+/*
+ * Example:
+ *
+ * CONFIG_PATH_BLOCK_SIZE: 4096
+ * blocks: 3
+ * availableSymbols: 12288
+ * total bits: 24576
+ * bits per block: 8192
+ *
+ * position: 9999
+ * bit position: 19998
+ * block for the bit: 19998/8192 = 2
+ * uint64_t in block for the bit: 19998%8192/64 = 3614/64 = 56
+ * bit in uint64_t in block: 19998%8192%64 = (19998%8192)%64 = 30
+ *
+ * The address of position 9999 is (2,56,30).
+ */
+
+/* a block contains an array of uint64_t */
+ int blockNumber=globalPosition/numberOfBitsPerBlock;
+
+/* this is the index of the uint64_t in the block */
+ int positionInBlock=(globalPosition%numberOfBitsPerBlock)/(sizeof(uint64_t)*BITS_PER_BYTE);
+ int bitPosition=(globalPosition%numberOfBitsPerBlock)%(sizeof(uint64_t)*BITS_PER_BYTE);
+
+ uint64_t oldChunkValue=m_blocks[blockNumber].m_content[positionInBlock];
+
+ oldChunkValue<<=(sizeof(uint64_t)*BITS_PER_BYTE-BITS_PER_NUCLEOTIDE-bitPosition);
+ oldChunkValue>>=(sizeof(uint64_t)*BITS_PER_BYTE-BITS_PER_NUCLEOTIDE);
+
+ uint8_t code=oldChunkValue;
+
+#ifdef CONFIG_ASSERT
+ assert(code==RAY_NUCLEOTIDE_A||code==RAY_NUCLEOTIDE_T||code==RAY_NUCLEOTIDE_C||code==RAY_NUCLEOTIDE_G);
+#endif
+
+ char symbol=codeToChar(code,false);
+
+ return symbol;
+}
+
+void GraphPath::writeSymbolInBlock(int position,char symbol){
+
+ int globalPosition=position*BITS_PER_NUCLEOTIDE;
+ int numberOfBitsPerBlock=CONFIG_PATH_BLOCK_SIZE*BITS_PER_NUCLEOTIDE;
+
+/* a block contains an array of uint64_t */
+ int blockNumber=globalPosition/numberOfBitsPerBlock;
+
+/* this is the index of the uint64_t in the block */
+ int positionInBlock=(globalPosition%numberOfBitsPerBlock)/(sizeof(uint64_t)*BITS_PER_BYTE);
+ int bitPosition=(globalPosition%numberOfBitsPerBlock)%(sizeof(uint64_t)*BITS_PER_BYTE);
+
+ uint64_t oldChunkValue=m_blocks[blockNumber].m_content[positionInBlock];
+
+ uint64_t mask=charToCode(symbol);
+ mask<<=bitPosition;
+
+ oldChunkValue|=mask;
+
+ m_blocks[blockNumber].m_content[positionInBlock]=oldChunkValue;
+
+#ifdef CONFIG_ASSERT
+ if(readSymbolInBlock(position)!=symbol){
+ cout<<"Expected "<<symbol<<" Actual "<<readSymbolInBlock(position)<<endl;
+ }
+
+ assert(readSymbolInBlock(position)==symbol);
+#endif
+}
+
+void GraphPath::addBlock(){
+
+ GraphPathBlock block;
+ m_blocks.push_back(block);
+
+ for(int i=0;i<(int)NUMBER_OF_64_BIT_INTEGERS;i++){
+ m_blocks[m_blocks.size()-1].m_content[i]=0;
+ }
+}
+
+#endif
+
+
+int GraphPath::load(const char * buffer) {
+ int position = 0;
+
+ uint32_t elements = 0;
+ int operationSize = sizeof(uint32_t);
+
+ memcpy(&elements, buffer + position, operationSize);
+ position += operationSize;
+
+ uint32_t kmerLength = 0;
+ memcpy(&kmerLength, buffer + position, operationSize);
+ position += operationSize;
+
+ setKmerLength(kmerLength);
+ //cout << "[DEBUG] GraphPath::load kmerLength " << kmerLength << endl;
+
+ for(int i = 0 ; i < (int)elements ; i ++) {
+ Kmer value;
+ position += value.load(buffer + position);
+ push_back(&value);
+ }
+
+ //cout << "DEBUG] loaded " << size() << " items for GraphPath" << endl;
+
+ return position;
+}
+
+int GraphPath::dump(char * buffer) const {
+ int position = 0;
+
+ uint32_t elements = size();
+
+ int operationSize = sizeof(uint32_t);
+ memcpy(buffer + position, &elements, operationSize);
+ position += operationSize;
+
+ uint32_t kmerLength = getKmerLength();
+
+#ifdef CONFIG_ASSERT
+ assert(kmerLength > 0);
+#endif
+
+ memcpy(buffer + position, &kmerLength, operationSize);
+ position += operationSize;
+
+ //cout << "[DEBUG] GraphPath::dump kmerLength " << kmerLength << endl;
+
+ for(int i = 0 ; i < (int)elements ; i ++) {
+ Kmer value;
+ at(i, &value);
+ position += value.dump(buffer + position);
+ }
+
+ return position;
+}
+
+int GraphPath::getKmerLength() const {
+ return m_kmerLength;
+}
+
+void GraphPath::reverseContent(GraphPath & newPath) const {
+
+#ifdef CONFIG_ASSERT
+ assert(newPath.size() == 0);
+#endif
+
+ newPath.setKmerLength(getKmerLength());
+
+ for(int i = size() - 1 ; i >= 0 ; --i) {
+ Kmer element;
+ at(i, &element);
+
+ // the false here drops support for colored data (SOLiD)
+ // anyway who cares
+
+ Kmer newElement = element.complementVertex(getKmerLength(), false);
+ newPath.push_back(&newElement);
+ }
+
+#ifdef CONFIG_ASSERT
+ assert(size() == newPath.size());
+ assert(getKmerLength() == newPath.getKmerLength());
+#endif
+
+}
+
+void GraphPath::appendPath(const GraphPath & path) {
+
+ for(int i = 0 ; i < (int) path.size() ; ++i) {
+ Kmer element;
+ path.at(i, &element);
+
+ push_back(&element);
+ }
+
+}
+
+int GraphPath::getRequiredNumberOfBytes() const {
+
+ int position = 0;
+
+ uint32_t elements = size();
+
+ int operationSize = sizeof(uint32_t);
+ //memcpy(buffer + position, &elements, operationSize);
+ position += operationSize;
+
+
+#ifdef CONFIG_ASSERT
+ uint32_t kmerLength = getKmerLength();
+ assert(kmerLength > 0);
+#endif
+
+ //memcpy(buffer + position, &kmerLength, operationSize);
+ position += operationSize;
+
+ //cout << "[DEBUG] GraphPath::dump kmerLength " << kmerLength << endl;
+
+ for(int i = 0 ; i < (int)elements ; i ++) {
+ Kmer value;
+ at(i, &value);
+ //position += value.dump(buffer + position);
+ position += value.getRequiredNumberOfBytes();
+ }
+
+ return position;
+
+
+}
+
+bool comparePaths(const GraphPath & a,const GraphPath & b){
+ return a.size()>b.size();
+}
+
+
+void GraphPath::setCoverageValueAt(int position, CoverageDepth value) {
+
+#ifdef CONFIG_ASSERT
+ assert((int)m_coverageValues.size() == size());
+ assert(position < size());
+#endif
+
+ m_coverageValues[position] = value;
+}
+
+void GraphPath::reserveSpaceForCoverage() {
+
+ m_coverageValues.reserve(size());
+
+ for(int i = 0 ; i < (int) size() ; ++i) {
+
+ addCoverageValue(0);
+
+ }
+
+ for(int i = 0 ; i < (int)size() ; ++i) {
+ setCoverageValueAt(i, 0);
+ }
+}
diff --git a/code/SeedingData/GraphPath.h b/code/SeedingData/GraphPath.h
new file mode 100644
index 0000000..bd24ede
--- /dev/null
+++ b/code/SeedingData/GraphPath.h
@@ -0,0 +1,154 @@
+/*
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+*/
+
+#ifndef _GraphPath_h
+#define _GraphPath_h
+
+#include <code/Mock/constants.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+
+#include <RayPlatform/store/CarriageableItem.h>
+
+#include <vector>
+using namespace std;
+
+#ifdef CONFIG_PATH_STORAGE_BLOCK
+
+/*
+ * The number of symbols per block.
+ */
+#define CONFIG_PATH_BLOCK_SIZE 4096
+
+/*
+ * Each block has 128 uint64_t objects, which can store 4096 nucleotides.
+ * 4096*2 = 8192 bits
+ * 128 * 8 * 8 = 8192 bits
+ */
+
+#define NUMBER_OF_64_BIT_INTEGERS ( CONFIG_PATH_BLOCK_SIZE * BITS_PER_NUCLEOTIDE / sizeof(uint64_t) / BITS_PER_BYTE )
+
+class GraphPathBlock {
+
+public:
+ uint64_t m_content[NUMBER_OF_64_BIT_INTEGERS];
+};
+#endif
+
+/**
+ * This class describes objects representing assembly seeds.
+ * An assembly seed is a path in the de Bruijn graph.
+ * It is likely unique in the target genome (hopefully !)
+ *
+ * \author pro-grammer: Sébastien Boisvert (good with grammar)
+ * \author co-designer: The Ray committee of wise people (E. God.)
+ * \date updated 2012-06-21 for the release of v2.0.0
+ *
+ * the PathHandle should be here directly instead of being a separate instance.
+ * \date 2013-07-31 PathHandle is now a class on its own.
+ *
+ * \author Sébastien Boisvert
+ */
+class GraphPath : public CarriageableItem {
+
+ bool m_deleted;
+
+ bool m_errorRaised;
+ int m_kmerLength;
+
+#ifdef CONFIG_PATH_STORAGE_DEFAULT
+ vector<Kmer> m_vertices;
+#elif defined(CONFIG_PATH_STORAGE_BLOCK)
+ vector<GraphPathBlock> m_blocks;
+ int m_size;
+#endif
+
+ vector<CoverageDepth> m_coverageValues;
+
+ CoverageDepth m_peakCoverage;
+
+/** computes locality with a weighted mean (locality object is peak coverage) **/
+ void computePeakCoverageUsingMean();
+
+/** computes locality with a mode (locality object is peak coverage) **/
+ void computePeakCoverageUsingMode();
+
+ void computePeakCoverageUsingStaggeredMean();
+
+ bool m_hasPeakCoverage;
+
+#ifdef CONFIG_PATH_STORAGE_BLOCK
+ void readObjectInBlock(int position,Kmer*object)const;
+ void writeObjectInBlock(const Kmer*a);
+#endif
+
+ bool canBeAdded(const Kmer*value)const;
+
+ int getBlockSize()const;
+
+ char readSymbolInBlock(int position)const;
+ void writeSymbolInBlock(int position,char symbol);
+
+ void addBlock();
+
+public:
+ GraphPath();
+
+ int size()const;
+ void at(int i,Kmer*value)const;
+
+ CoverageDepth getCoverageAt(int i)const;
+
+ void push_back(const Kmer*a);
+ void getVertices(vector<Kmer>*value)const;
+ void clear();
+
+ void addCoverageValue(CoverageDepth value);
+ void setCoverageValueAt(int position, CoverageDepth value);
+ void reserveSpaceForCoverage();
+
+ CoverageDepth getPeakCoverage()const;
+ void resetCoverageValues();
+
+ void computePeakCoverage();
+ void reserve(int size);
+
+ void setKmerLength(int kmerLength);
+
+ bool isDeleted();
+ void markAsDeleted();
+
+ int load(const char * buffer);
+ int dump(char * buffer) const;
+ int getRequiredNumberOfBytes() const;
+
+ int getKmerLength() const;
+
+ /**
+ * Put the reversed content
+ * in another object.
+ */
+ void reverseContent(GraphPath & newPath) const;
+
+ void appendPath(const GraphPath & path);
+};
+
+bool comparePaths(const GraphPath & a,const GraphPath & b);
+
+#endif
diff --git a/code/SeedingData/Makefile b/code/SeedingData/Makefile
new file mode 100644
index 0000000..853ae65
--- /dev/null
+++ b/code/SeedingData/Makefile
@@ -0,0 +1,9 @@
+SeedingData-y += code/SeedingData/GraphPath.o
+SeedingData-y += code/SeedingData/SeedWorker.o
+SeedingData-y += code/SeedingData/SeedingData.o
+SeedingData-y += code/SeedingData/PathHandle.o
+
+obj-y += $(SeedingData-y)
+
+
+
diff --git a/code/SeedingData/PathHandle.cpp b/code/SeedingData/PathHandle.cpp
new file mode 100644
index 0000000..5398a55
--- /dev/null
+++ b/code/SeedingData/PathHandle.cpp
@@ -0,0 +1,128 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "PathHandle.h"
+
+#include <string.h>
+
+void PathHandle::operator=(const PathHandle &b) {
+ m_value = b.m_value;
+}
+
+bool PathHandle::operator<(const PathHandle & b)const {
+ return m_value < b.m_value;
+}
+
+bool PathHandle::operator!=(const PathHandle & b)const {
+ return m_value != b.m_value;
+}
+
+bool PathHandle::operator==(const PathHandle & b)const {
+ return m_value == b.m_value;
+}
+
+void PathHandle::operator=(const uint64_t & value) {
+ m_value = value;
+}
+
+PathHandle::PathHandle(const uint64_t & value) {
+ m_value = value;
+}
+
+const uint64_t & PathHandle::operator () () const {
+ return m_value;
+}
+
+uint64_t & PathHandle::operator () () {
+ return m_value;
+}
+
+PathHandle::PathHandle() {
+ m_value = 0;
+}
+
+uint64_t PathHandle::operator / (uint64_t value) {
+ return m_value / value;
+}
+
+ostream & operator >>(ostream & stream, const PathHandle & handle) {
+ stream >> handle.m_value;
+ return stream;
+}
+
+ostream & operator <<(ostream & stream, const PathHandle & handle) {
+ stream << handle.m_value;
+ return stream;
+}
+
+uint64_t PathHandle::operator * (uint64_t value) {
+ return m_value * value;
+}
+
+uint64_t PathHandle::operator - (uint64_t value) {
+ return m_value - value;
+}
+
+uint64_t PathHandle::operator + (uint64_t value) {
+ return m_value + value;
+}
+
+uint64_t PathHandle::operator % (uint64_t value) {
+ return m_value % value;
+}
+
+
+const uint64_t & PathHandle::getValue() const {
+ return m_value;
+}
+
+bool PathHandle::operator>(const PathHandle & b)const {
+ return m_value > b.m_value;
+}
+
+bool PathHandle::operator<=(const PathHandle & b)const {
+ return m_value <= b.m_value;
+}
+
+bool PathHandle::operator>=(const PathHandle & b)const {
+ return m_value >= b.m_value;
+}
+
+uint64_t & PathHandle::getValue() {
+ return m_value;
+}
+
+int PathHandle::load(const char * buffer) {
+ int size = sizeof(uint64_t);
+ memcpy(&m_value, buffer, size);
+ return size;
+}
+
+int PathHandle::dump(char * buffer) const {
+
+ int size = sizeof(uint64_t);
+ memcpy(buffer, &m_value, size);
+ return size;
+}
+
+
+int PathHandle::getRequiredNumberOfBytes() const {
+ return sizeof(uint64_t);
+}
diff --git a/code/SeedingData/PathHandle.h b/code/SeedingData/PathHandle.h
new file mode 100644
index 0000000..6c29b36
--- /dev/null
+++ b/code/SeedingData/PathHandle.h
@@ -0,0 +1,69 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+
+#ifndef PathHandleHeader
+#define PathHandleHeader
+
+#include <RayPlatform/store/CarriageableItem.h>
+
+#include <ostream>
+using namespace std;
+
+#include <stdint.h>
+
+/**
+ * the identifier for a path in the de Bruijn graph
+ */
+class PathHandle : public CarriageableItem {
+
+ uint64_t m_value;
+
+public:
+ PathHandle(const uint64_t & value);
+ PathHandle();
+ void operator=(const PathHandle &b);
+ bool operator<(const PathHandle & b)const;
+ bool operator>(const PathHandle & b)const;
+ bool operator<=(const PathHandle & b)const;
+ bool operator>=(const PathHandle & b)const;
+ bool operator!=(const PathHandle & b)const;
+ bool operator==(const PathHandle & b)const;
+ void operator=(const uint64_t & value);
+ const uint64_t & operator () () const;
+ uint64_t & operator () ();
+ uint64_t operator / (uint64_t value);
+ uint64_t operator - (uint64_t value);
+ uint64_t operator * (uint64_t value);
+ uint64_t operator + (uint64_t value);
+ uint64_t operator % (uint64_t value);
+
+ friend ostream & operator <<(ostream & stream, const PathHandle & handle);
+ friend ostream & operator >>(ostream & stream, const PathHandle & handle);
+
+ const uint64_t & getValue() const;
+ uint64_t & getValue();
+
+ int load(const char * buffer);
+ int dump(char * buffer) const;
+ int getRequiredNumberOfBytes() const;
+};
+
+#endif
diff --git a/code/plugin_SeedingData/SeedWorker.cpp b/code/SeedingData/SeedWorker.cpp
similarity index 65%
rename from code/plugin_SeedingData/SeedWorker.cpp
rename to code/SeedingData/SeedWorker.cpp
index ac53e7a..1dca517 100644
--- a/code/plugin_SeedingData/SeedWorker.cpp
+++ b/code/SeedingData/SeedWorker.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,15 +19,18 @@
*/
-#include <application_core/constants.h>
-#include <plugin_SeedingData/SeedWorker.h>
-#include <assert.h>
-#include <communication/Message.h>
+#include "SeedWorker.h"
+
+#include <code/Mock/constants.h>
+#include <code/Mock/common_functions.h>
+
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/communication/mpi_tags.h>
+
+#include <stdint.h>
#include <stdlib.h>
+#include <assert.h>
#include <stdio.h>
-#include <communication/mpi_tags.h>
-#include <application_core/common_functions.h>
-#include <stdint.h>
#include <iostream>
using namespace std;
@@ -61,6 +64,8 @@ void SeedWorker::work(){
m_SEEDING_firstVertexParentTestDone=true;
m_SEEDING_vertices.clear();
m_SEEDING_seed.clear();
+ m_SEEDING_seed.setKmerLength(m_wordSize);
+
// restore original starter.
m_SEEDING_currentVertex=m_SEEDING_first;
m_SEEDING_testInitiated=false;
@@ -68,7 +73,8 @@ void SeedWorker::work(){
}
}
// check if currentVertex has 1 ingoing edge and 1 outgoing edge, if yes, add it
- }else{
+ }else if(m_elongationMode){
+
// attempt to add m_SEEDING_currentVertex
if(!m_SEEDING_1_1_test_done){
do_1_1_test();
@@ -77,9 +83,8 @@ void SeedWorker::work(){
m_SEEDING_1_1_test_result=false;
}
if(!m_SEEDING_1_1_test_result){
- m_finished=true;
- if(m_parameters->debugSeeds()){
+ if(m_debugSeeds){
printf("Rank %i next vertex: Coverage= %i, ingoing coverages:",m_rank,m_cache[m_SEEDING_currentVertex]);
for(int i=0;i<(int)m_ingoingCoverages.size();i++){
printf(" %i",m_ingoingCoverages[i]);
@@ -91,23 +96,36 @@ void SeedWorker::work(){
printf("\n");
int n=100;
- if((int)m_coverages.size()<n){
- n=m_coverages.size();
+ if((int)m_SEEDING_seed.size()<n){
+ n=m_SEEDING_seed.size();
}
printf("Rank %i last %i coverage values in the seed:",m_rank,n);
for(int i=n-1;i>=0;i--){
- printf(" %i",m_coverages[m_coverages.size()-i-1]);
+ int theCoverage=m_SEEDING_seed.getCoverageAt(m_SEEDING_seed.size()-1-i);
+
+ printf(" %i",theCoverage);
}
printf("\n");
}
+
+
+ m_elongationMode=false;
+ m_endChecksMode=true;
}else{
+
+ Kmer object;
+ if(m_SEEDING_seed.size()>0)
+ m_SEEDING_seed.at(m_SEEDING_seed.size()-1,&object);
+
// we want some coherence...
if(m_SEEDING_seed.size()>0
- &&!(m_SEEDING_seed[m_SEEDING_seed.size()-1].isEqual(&m_SEEDING_currentParentVertex))){
+ &&!(object.isEqual(&m_SEEDING_currentParentVertex))){
+
m_finished=true;
}else{
- m_SEEDING_seed.push_back(m_SEEDING_currentVertex);
- m_coverages.push_back(m_cache[m_SEEDING_currentVertex]);
+ m_SEEDING_seed.push_back(&m_SEEDING_currentVertex);
+ m_SEEDING_seed.addCoverageValue(m_cache[m_SEEDING_currentVertex]);
+
m_SEEDING_vertices.insert(m_SEEDING_currentVertex);
m_SEEDING_currentVertex=m_SEEDING_currentChildVertex;
m_SEEDING_testInitiated=false;
@@ -115,10 +133,147 @@ void SeedWorker::work(){
}
}
}
+ }else if(m_endChecksMode){
+ m_endChecksMode = false;
+ return;
+
+#if 0
+ performChecksOnPathEnds();
+#endif
+ }else{
+ m_finished=true;
+ }
+}
+
+bool SeedWorker::getPathAfter(Kmer*kmer,int depth){
+
+ Kmer object=*kmer;
+
+ if(m_verticesAfter.size()>0){
+ object=m_verticesAfter[m_verticesAfter.size()-1];
+ }
+
+// we reached the desired depth
+ if((int)m_verticesAfter.size()>=depth){
+ m_verticesAfter.clear();
+ return true;
+ }
+
+// query data remotely
+ if(fetchVertexData(&object)){
+
+ m_vertexFetcherStarted=false;
+
+ // only one parent
+ if(m_vertexFetcherChildren.size()==1){
+ m_verticesAfter.push_back(m_vertexFetcherChildren[0]);
+
+ // more than 1 parent
+ }else if(m_vertexFetcherChildren.size()>1){
+ return true;
+
+ // this is a dead end
+ }else if(m_vertexFetcherChildren.size()==0){
+
+ m_tailIsDeadEnd=true;
+
+ return true;
+ }
+ }
+
+ return false;
+}
+
+bool SeedWorker::getPathBefore(Kmer*kmer,int depth){
+
+ Kmer object=*kmer;
+
+ if(m_verticesBefore.size()>0){
+ object=m_verticesBefore[m_verticesBefore.size()-1];
+ }
+
+// we reached the desired depth
+ if((int)m_verticesBefore.size()>=depth){
+ m_verticesBefore.clear();
+ return true;
+ }
+
+// query data remotely
+ if(fetchVertexData(&object)){
+
+ m_vertexFetcherStarted=false;
+
+ // only one parent
+ if(m_vertexFetcherParents.size()==1){
+ m_verticesBefore.push_back(m_vertexFetcherParents[0]);
+
+ // more than 1 parent
+ }else if(m_vertexFetcherParents.size()>1){
+ return true;
+
+ // this is a dead end
+ }else if(m_vertexFetcherParents.size()==0){
+
+ m_headIsDeadEnd=true;
+ return true;
+ }
+ }
+
+ return false;
+}
+
+void SeedWorker::performChecksOnPathEnds(){
+/*
+ * Check if it's a dead end. We don't want dead ends because they consume too
+ * much time.
+ *
+ * First, check that the first k-mer is connected to something in the graph.
+ *
+ * Then, check that the last k-mer is connected to something in the graph too.
+ */
+
+ if(!m_endChecksModeStarted){
+ m_endChecksModeStarted=true;
+ m_checkedHead=false;
+ m_vertexFetcherStarted=false;
+
+/*
+ * Check parents to see if it's a dead end.
+ */
+ }else if(!m_checkedHead){
+
+ Kmer kmer;
+ int positionInPath=0;
+ m_SEEDING_seed.at(positionInPath,&kmer);
+
+ if(getPathBefore(&kmer,10)){
+
+ m_checkedHead=true;
+ m_checkedTail=false;
+ m_vertexFetcherStarted=false;
+ }
+
+/*
+ * Check children of the last k-mer to see if there is a dead end.
+ */
+ }else if(!m_checkedTail){
+
+ Kmer kmer;
+ int positionInPath=m_SEEDING_seed.size()-1;
+ m_SEEDING_seed.at(positionInPath,&kmer);
+
+ if(getPathAfter(&kmer,10)){
+ m_checkedTail=true;
+ m_vertexFetcherStarted=false;
+ }
+
+ }else{
+ m_endChecksMode=false;
}
}
bool SeedWorker::isDone(){
+
return m_finished;
}
@@ -145,6 +300,25 @@ void SeedWorker::constructor(Kmer*key,Parameters*parameters,RingAllocator*outbox
m_SEEDING_seed.clear();
m_wordSize=parameters->getWordSize();
m_parameters=parameters;
+
+#ifdef ASSERT
+ assert(m_wordSize!=0);
+#endif
+
+ m_SEEDING_seed.setKmerLength(m_wordSize);
+
+ m_debugSeeds=false;
+
+ m_elongationMode=true;
+
+ m_endChecksMode=false;
+
+ m_headIsDeadEnd=false;
+ m_tailIsDeadEnd=false;
+}
+
+void SeedWorker::enableDebugMode(){
+ m_debugSeeds=true;
}
/*
@@ -185,7 +359,7 @@ void SeedWorker::do_1_1_test(){
int bufferPosition=0;
m_SEEDING_currentVertex.pack(message,&bufferPosition);
Message aMessage(message,m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT),
- m_parameters->_vertexRank(&m_SEEDING_currentVertex),
+ m_parameters->vertexRank(&m_SEEDING_currentVertex),
RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,getRank());
m_virtualCommunicator->pushMessage(m_workerIdentifier,&aMessage);
m_SEEDING_numberOfIngoingEdgesWithSeedCoverage=0;
@@ -206,9 +380,9 @@ void SeedWorker::do_1_1_test(){
m_cache[m_SEEDING_currentVertex]=m_mainVertexCoverage;
- m_SEEDING_receivedIngoingEdges=m_SEEDING_currentVertex._getIngoingEdges(edges,m_wordSize);
+ m_SEEDING_receivedIngoingEdges=m_SEEDING_currentVertex.getIngoingEdges(edges,m_wordSize);
- m_SEEDING_receivedOutgoingEdges=m_SEEDING_currentVertex._getOutgoingEdges(edges,m_wordSize);
+ m_SEEDING_receivedOutgoingEdges=m_SEEDING_currentVertex.getOutgoingEdges(edges,m_wordSize);
m_ingoingCoverages.clear();
m_outgoingCoverages.clear();
@@ -235,7 +409,7 @@ void SeedWorker::do_1_1_test(){
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(KMER_U64_ARRAY_SIZE*sizeof(MessageUnit));
int bufferPosition=0;
vertex.pack(message,&bufferPosition);
- int dest=m_parameters->_vertexRank(&vertex);
+ int dest=m_parameters->vertexRank(&vertex);
Message aMessage(message,bufferPosition,dest,
RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE,getRank());
@@ -260,7 +434,7 @@ void SeedWorker::do_1_1_test(){
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(KMER_U64_ARRAY_SIZE*sizeof(MessageUnit));
int bufferPosition=0;
vertex.pack(message,&bufferPosition);
- int dest=m_parameters->_vertexRank(&vertex);
+ int dest=m_parameters->vertexRank(&vertex);
Message aMessage(message,bufferPosition,
dest,
@@ -367,14 +541,70 @@ int SeedWorker::getRank(){
return m_rank;
}
-vector<Kmer>*SeedWorker::getSeed(){
+GraphPath*SeedWorker::getSeed(){
return &m_SEEDING_seed;
}
-vector<int>*SeedWorker::getCoverageVector(){
- return &m_coverages;
-}
-
WorkerHandle SeedWorker::getWorkerIdentifier(){
return m_workerIdentifier;
}
+
+bool SeedWorker::isHeadADeadEnd(){
+ return m_headIsDeadEnd;
+}
+
+bool SeedWorker::isTailADeadEnd(){
+ return m_tailIsDeadEnd;
+}
+
+/**
+ * Fetch parents and children and coverage depth.
+ */
+bool SeedWorker::fetchVertexData(Kmer*kmer){
+
+ if(!m_vertexFetcherStarted){
+
+ m_vertexFetcherRequestedData=false;
+
+ m_vertexFetcherCoverage=0;
+ m_vertexFetcherParents.clear();
+ m_vertexFetcherChildren.clear();
+
+ m_vertexFetcherStarted=true;
+
+ }else if(!m_vertexFetcherRequestedData){
+
+ MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(MessageUnit));
+ int bufferPosition=0;
+ kmer->pack(message,&bufferPosition);
+ Message aMessage(message,m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT),
+ m_parameters->vertexRank(kmer),
+ RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,getRank());
+
+ m_virtualCommunicator->pushMessage(m_workerIdentifier,&aMessage);
+
+ m_vertexFetcherRequestedData=true;
+ m_vertexFetcherReceivedData=false;
+
+ }else if(!m_vertexFetcherReceivedData && m_virtualCommunicator->isMessageProcessed(m_workerIdentifier)){
+
+ vector<MessageUnit> elements;
+ m_virtualCommunicator->getMessageResponseElements(m_workerIdentifier,&elements);
+
+ int bufferPosition=0;
+
+ uint8_t edges=elements[bufferPosition++];
+ m_vertexFetcherCoverage=elements[bufferPosition++];
+
+ m_vertexFetcherParents=kmer->getIngoingEdges(edges,m_wordSize);
+ m_vertexFetcherChildren=kmer->getOutgoingEdges(edges,m_wordSize);
+
+ return true;
+ }
+
+ return false;
+}
+
+bool SeedWorker::isBubbleWeakComponent(){
+ return false;
+}
diff --git a/code/plugin_SeedingData/SeedWorker.h b/code/SeedingData/SeedWorker.h
similarity index 70%
rename from code/plugin_SeedingData/SeedWorker.h
rename to code/SeedingData/SeedWorker.h
index b4c7966..1e30e28 100644
--- a/code/plugin_SeedingData/SeedWorker.h
+++ b/code/SeedingData/SeedWorker.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,17 +22,21 @@
#ifndef _SeedWorker
#define _SeedWorker
+#include <code/SeedingData/GraphPath.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/scheduling/Worker.h>
+
#include <stdint.h>
-#include <application_core/Parameters.h>
-#include <memory/RingAllocator.h>
-#include <communication/VirtualCommunicator.h>
-#include <scheduling/Worker.h>
#include <vector>
using namespace std;
/*
* Given a Kmer, SeedWorker determines if it spawns a seed.
* If yes, it computes the seed.
+ *
* \author Sébastien Boisvert
*/
class SeedWorker : public Worker {
@@ -42,6 +46,11 @@ class SeedWorker : public Worker {
int m_mainVertexCoverage;
+ bool m_hasDeadEnd;
+ bool m_debugSeeds;
+
+ bool m_elongationMode;
+
map<Kmer,int> m_cache;
WorkerHandle m_workerIdentifier;
bool m_finished;
@@ -72,10 +81,13 @@ class SeedWorker : public Worker {
bool m_ingoingEdgesReceived;
int m_wordSize;
- vector<Kmer> m_SEEDING_seed;
- vector<int> m_coverages;
+ GraphPath m_SEEDING_seed;
bool m_SEEDING_firstVertexParentTestDone;
+
+/*
+ * Store visited vertices.
+ */
set<Kmer> m_SEEDING_vertices;
Kmer m_SEEDING_first;
bool m_SEEDING_firstVertexTestDone;
@@ -89,6 +101,33 @@ class SeedWorker : public Worker {
int getSize();
bool m_SEEDING_1_1_test_done;
VirtualCommunicator*m_virtualCommunicator;
+
+/*
+ * Additional quality control tests.
+ */
+ bool m_checkedHead;
+ bool m_checkedTail;
+ bool m_endChecksModeStarted;
+ bool m_endChecksMode;
+ bool m_vertexFetcherStarted;
+ vector<Kmer> m_vertexFetcherParents;
+ vector<Kmer> m_vertexFetcherChildren;
+ CoverageDepth m_vertexFetcherCoverage;
+ bool m_vertexFetcherReceivedData;
+ bool m_vertexFetcherRequestedData;
+ bool m_headIsDeadEnd;
+ bool m_tailIsDeadEnd;
+
+// topological algorithm for graphs
+ vector<Kmer> m_verticesBefore;
+ vector<Kmer> m_verticesAfter;
+
+ void performChecksOnPathEnds();
+ bool fetchVertexData(Kmer*kmer);
+
+ bool getPathBefore(Kmer*kmer,int depth);
+ bool getPathAfter(Kmer*kmer,int depth);
+
public:
void constructor(Kmer*vertex,Parameters*parameters,RingAllocator*outboxAllocator,
VirtualCommunicator*vc,WorkerHandle workerId,
@@ -97,8 +136,7 @@ public:
MessageTag RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE
);
- vector<Kmer>*getSeed();
- vector<int>*getCoverageVector();
+ GraphPath*getSeed();
void do_1_1_test();
@@ -113,6 +151,10 @@ public:
/** get the worker number */
WorkerHandle getWorkerIdentifier();
+ bool isHeadADeadEnd();
+ bool isTailADeadEnd();
+ void enableDebugMode();
+ bool isBubbleWeakComponent();
};
#endif
diff --git a/code/plugin_SeedingData/SeedingData.cpp b/code/SeedingData/SeedingData.cpp
similarity index 60%
rename from code/plugin_SeedingData/SeedingData.cpp
rename to code/SeedingData/SeedingData.cpp
index d3e6bbe..884f8cc 100644
--- a/code/plugin_SeedingData/SeedingData.cpp
+++ b/code/SeedingData/SeedingData.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,31 +19,30 @@
*/
-#include <application_core/constants.h>
-#include <communication/VirtualCommunicator.h>
-#include <core/OperatingSystem.h>
+#include "SeedingData.h"
+
+#include <code/Mock/constants.h>
+#include <code/SeedingData/SeedWorker.h>
+
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/communication/mpi_tags.h>
+#include <RayPlatform/core/OperatingSystem.h>
+
#include <algorithm>
#include <fstream>
#include <sstream>
#include <assert.h>
-#include <plugin_SeedingData/SeedingData.h>
-#include <communication/Message.h>
-#include <communication/mpi_tags.h>
-#include <plugin_SeedingData/SeedWorker.h>
__CreatePlugin(SeedingData);
- /**/
- /**/
-__CreateSlaveModeAdapter(SeedingData,RAY_SLAVE_MODE_START_SEEDING); /**/
-__CreateSlaveModeAdapter(SeedingData,RAY_SLAVE_MODE_SEND_SEED_LENGTHS); /**/
- /**/
- /**/
+__CreateSlaveModeAdapter(SeedingData,RAY_SLAVE_MODE_START_SEEDING);
+__CreateSlaveModeAdapter(SeedingData,RAY_SLAVE_MODE_SEND_SEED_LENGTHS);
+/*
+ * TODO: port this with the VirtualProcessor framework.
+ */
-bool myComparator_sort(const AssemblySeed & a,const AssemblySeed & b){
- return a.size()>b.size();
-}
void SeedingData::call_RAY_SLAVE_MODE_START_SEEDING(){
if(!m_initiatedIterator){
@@ -65,11 +64,11 @@ void SeedingData::call_RAY_SLAVE_MODE_START_SEEDING(){
}
if(!m_checkedCheckpoint){
- if(m_parameters->hasCheckpoint("Seeds")){
- cout<<"Rank "<<m_parameters->getRank()<<": checkpoint Seeds exists, not computing seeds."<<endl;
+ if(m_parameters->hasCheckpoint("SimpleSeeds")){
+ cout<<"Rank "<<m_parameters->getRank()<<": checkpoint SimpleSeeds exists, not computing seeds."<<endl;
(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_SEEDING_IS_OVER,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
loadCheckpoint();
@@ -106,16 +105,11 @@ void SeedingData::call_RAY_SLAVE_MODE_START_SEEDING(){
}
if(m_aliveWorkers[workerId].isDone()){
m_workersDone.push_back(workerId);
- vector<Kmer> seed=*(m_aliveWorkers[workerId].getSeed());
- vector<int>*coverageValues=m_aliveWorkers[workerId].getCoverageVector();
+ GraphPath*seed=m_aliveWorkers[workerId].getSeed();
- #ifdef ASSERT
- assert(seed.size() == coverageValues->size());
- #endif
-
- int nucleotides=getNumberOfNucleotides(seed.size(),m_wordSize);
+ int nucleotides=getNumberOfNucleotides(seed->size(),m_wordSize);
- if(seed.size() > 0 && m_parameters->debugSeeds()){
+ if(seed->size() > 0 && m_debugSeeds){
cout<<"Raw seed length: "<<nucleotides<<" nucleotides"<<endl;
}
@@ -123,51 +117,93 @@ void SeedingData::call_RAY_SLAVE_MODE_START_SEEDING(){
assert(nucleotides==0 || nucleotides>=m_wordSize);
#endif
+ SeedWorker*worker=&(m_aliveWorkers[workerId]);
+
+ bool isSmall = nucleotides < 4 * m_parameters->getWordSize();
+
+ if(isSmall && worker->isHeadADeadEnd() && worker->isTailADeadEnd()){
+
+ m_skippedObjectsWithTwoDeadEnds++;
+
+ }else if(isSmall && worker->isHeadADeadEnd()){
+
+ m_skippedObjectsWithDeadEndForHead++;
+
+ }else if(isSmall && worker->isTailADeadEnd()){
+
+ m_skippedObjectsWithDeadEndForTail++;
+
+ }else if(isSmall && worker->isBubbleWeakComponent()){
+
+ m_skippedObjectsWithBubbleWeakComponent++;
+
// only consider the long ones.
- if(nucleotides>=m_parameters->getMinimumContigLength()){
+ }else if(nucleotides>=m_parameters->getMinimumContigLength()){
#ifdef SHOW_DISCOVERIES
printf("Rank %i discovered a seed with %i vertices\n",m_rank,(int)seed.size());
#endif
-
+
#ifdef ASSERT
- assert(seed.size()>0);
+ assert(seed->size()>0);
#endif
- Kmer firstVertex=seed[0];
- Kmer lastVertex=seed[seed.size()-1];
+ Kmer firstVertex;
+ seed->at(0,&firstVertex);
+ Kmer lastVertex;
+ seed->at(seed->size()-1,&lastVertex);
Kmer firstReverse=m_parameters->_complementVertex(&lastVertex);
- if(firstVertex<firstReverse){
- printf("Rank %i stored a seed with %i vertices\n",m_rank,(int)seed.size());
- fflush(stdout);
+ int minimumNucleotidesForVerbosity=1024;
- if(m_parameters->showMemoryUsage()){
- showMemoryUsage(m_rank);
+ bool verbose=nucleotides>=minimumNucleotidesForVerbosity;
+
+ if(m_debugSeeds){
+ verbose=true;
+ }
+
+ bool ignoreSeeds = m_parameters->hasOption("-ignore-seeds");
+
+ if(firstVertex<firstReverse && !ignoreSeeds){
+
+ if(verbose){
+ printf("Rank %i stored a seed with %i vertices\n",m_rank,(int)seed->size());
}
- AssemblySeed theSeed;
- for(int i=0;i<(int)seed.size();i++){
- theSeed.push_back(&(seed[i]));
- theSeed.addCoverageValue(coverageValues->at(i));
+ if(m_parameters->showMemoryUsage() && verbose){
+ showMemoryUsage(m_rank);
}
- theSeed.computePeakCoverage();
-
- CoverageDepth peakCoverage=theSeed.getPeakCoverage();
+ GraphPath*theSeed=seed;
+
+ theSeed->computePeakCoverage();
+
+ CoverageDepth peakCoverage=theSeed->getPeakCoverage();
+
+ if(verbose)
+ cout<<"Got a seed, peak coverage: "<<peakCoverage;
- cout<<"Got a seed, peak coverage: "<<peakCoverage;
-
/* ignore the seed if it has too much coverage. */
if(peakCoverage >= m_minimumSeedCoverageDepth
&& peakCoverage <= m_parameters->getMaximumSeedCoverage()){
- cout<<", adding seed."<<endl;
- m_SEEDING_seeds.push_back(theSeed);
+ if(verbose)
+ cout<<", adding seed."<<endl;
+
+ m_SEEDING_seeds.push_back(*theSeed);
+ m_eligiblePaths++;
}else{
- cout<<", ignoring seed."<<endl;
+
+ if(verbose)
+ cout<<", ignoring seed."<<endl;
+
+ m_skippedNotEnoughCoverage++;
}
+ }else{
+ m_skippedNotMine++;
}
+ }else{
+ m_skippedTooShort++;
}
}
m_activeWorkerIterator++;
@@ -184,7 +220,6 @@ void SeedingData::call_RAY_SLAVE_MODE_START_SEEDING(){
if(m_SEEDING_i<m_subgraph->size()&&(int)m_aliveWorkers.size()<m_maximumAliveWorkers){
if(m_SEEDING_i % 100000 ==0){
printf("Rank %i is creating seeds [%i/%i]\n",getRank(),(int)m_SEEDING_i+1,(int)m_subgraph->size());
- fflush(stdout);
if(m_parameters->showMemoryUsage()){
showMemoryUsage(m_rank);
@@ -204,6 +239,9 @@ void SeedingData::call_RAY_SLAVE_MODE_START_SEEDING(){
RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE
);
+ if(m_debugSeeds)
+ m_aliveWorkers[m_SEEDING_i].enableDebugMode();
+
m_activeWorkers.insert(m_SEEDING_i);
int population=m_aliveWorkers.size();
@@ -228,17 +266,29 @@ RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE
#endif
if((int)m_subgraph->size()==m_completedJobs){
+
printf("Rank %i has %i seeds\n",m_rank,(int)m_SEEDING_seeds.size());
- fflush(stdout);
printf("Rank %i is creating seeds [%i/%i] (completed)\n",getRank(),(int)m_SEEDING_i,(int)m_subgraph->size());
- fflush(stdout);
printf("Rank %i: peak number of workers: %i, maximum: %i\n",m_rank,m_maximumWorkers,m_maximumAliveWorkers);
- fflush(stdout);
m_virtualCommunicator->printStatistics();
+ cout<<"Rank "<<m_rank<<" runtime statistics for seeding algorithm: "<<endl;
+ cout<<"Rank "<<m_rank<<" Skipped paths because of dead end for head: "<<m_skippedObjectsWithDeadEndForHead<<endl;
+ cout<<"Rank "<<m_rank<<" Skipped paths because of dead end for tail: "<<m_skippedObjectsWithDeadEndForTail<<endl;
+ cout<<"Rank "<<m_rank<<" Skipped paths because of two dead ends: "<<m_skippedObjectsWithTwoDeadEnds<<endl;
+ cout<<"Rank "<<m_rank<<" Skipped paths because of bubble weak component: "<<m_skippedObjectsWithBubbleWeakComponent<<endl;
+ cout<<"Rank "<<m_rank<<" Skipped paths because of short length: "<<m_skippedTooShort<<endl;
+ cout<<"Rank "<<m_rank<<" Skipped paths because of bad ownership: "<<m_skippedNotMine<<endl;
+ cout<<"Rank "<<m_rank<<" Skipped paths because of low coverage: "<<m_skippedNotEnoughCoverage<<endl;
+ cout<<"Rank "<<m_rank<<" Eligible paths: "<<m_eligiblePaths<<endl;
+
+ #ifdef ASSERT
+ assert(m_eligiblePaths==(int)m_SEEDING_seeds.size());
+ #endif
+
(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_SEEDING_IS_OVER,getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
if(m_parameters->showMemoryUsage()){
showMemoryUsage(m_rank);
@@ -251,33 +301,31 @@ RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE
// sort the seeds by length
std::sort(m_SEEDING_seeds.begin(),
- m_SEEDING_seeds.end(),myComparator_sort);
-
- /** write seeds for debugging purposes */
- if(m_parameters->hasOption("-write-seeds")){
- ostringstream fileName;
- fileName<<m_parameters->getPrefix()<<"Rank"<<m_parameters->getRank()<<".RaySeeds.fasta";
- ofstream f(fileName.str().c_str());
-
- for(int i=0;i<(int)m_SEEDING_seeds.size();i++){
- PathHandle id=getPathUniqueId(m_parameters->getRank(),i);
- f<<">RaySeed-"<<id<<endl;
+ m_SEEDING_seeds.end(), comparePaths);
- f<<addLineBreaks(convertToString(m_SEEDING_seeds[i].getVertices(),
- m_parameters->getWordSize(),m_parameters->getColorSpaceMode()),
- m_parameters->getColumns());
- }
- f.close();
- }
+ /**************************************************************
+ * Write down the SimpleSeeds checkpoint now.
+ **********************************************************************/
+ writeCheckpoints();
}
}
void SeedingData::constructor(SeedExtender*seedExtender,int rank,int size,StaticVector*outbox,RingAllocator*outboxAllocator,
int*mode,
- Parameters*parameters,int*wordSize,GridTable*subgraph,StaticVector*inbox,
+ Parameters*parameters,GridTable*subgraph,StaticVector*inbox,
VirtualCommunicator*vc){
+ m_skippedObjectsWithDeadEndForHead=0;
+ m_skippedObjectsWithDeadEndForTail=0;
+ m_skippedObjectsWithTwoDeadEnds=0;
+ m_skippedObjectsWithBubbleWeakComponent=0;
+
+ m_skippedTooShort=0;
+ m_skippedNotMine=0;
+ m_skippedNotEnoughCoverage=0;
+ m_eligiblePaths=0;
+
m_checkedCheckpoint=false;
m_virtualCommunicator=vc;
m_seedExtender=seedExtender;
@@ -293,7 +341,7 @@ int*mode,
m_parameters=parameters;
m_wordSize=m_parameters->getWordSize();
#ifdef ASSERT
- assert(m_wordSize>=15&&m_wordSize<=MAXKMERLENGTH);
+ assert(m_wordSize>=15&&m_wordSize<=CONFIG_MAXKMERLENGTH);
#endif
m_subgraph=subgraph;
m_initiatedIterator=false;
@@ -304,6 +352,11 @@ int*mode,
m_minimumSeedCoverageDepth=m_parameters->getConfigurationInteger("-use-minimum-seed-coverage",0);
cout<<"[SeedingData] will use "<<m_minimumSeedCoverageDepth<<" for the minimum seed coverage"<<endl;
}
+
+ m_debugSeeds=false;
+
+ if(m_parameters->hasConfigurationOption("-debug-seeds",0))
+ m_debugSeeds=true;
}
int SeedingData::getRank(){
@@ -350,86 +403,89 @@ void SeedingData::updateStates(){
void SeedingData::call_RAY_SLAVE_MODE_SEND_SEED_LENGTHS(){
if(!m_initialized){
- for(int i=0;i<(int)m_SEEDING_seeds.size();i++){
- int length=getNumberOfNucleotides(m_SEEDING_seeds[i].size(),
- m_parameters->getWordSize());
- m_slaveSeedLengths[length]++;
- }
- m_iterator=m_slaveSeedLengths.begin();
- m_initialized=true;
- m_communicatorWasTriggered=false;
-
m_virtualCommunicator->resetCounters();
- }
-
- if(m_inbox->size()==1&&(*m_inbox)[0]->getTag()==RAY_MPI_TAG_SEND_SEED_LENGTHS_REPLY)
- m_communicatorWasTriggered=false;
-
- if(m_communicatorWasTriggered)
- return;
- if(m_iterator==m_slaveSeedLengths.end()){
- Message aMessage(NULL,0,MASTER_RANK,
- RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS,getRank());
- m_outbox->push_back(aMessage);
- (*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
- return;
+ m_initialized = true;
}
-
- MessageUnit*messageBuffer=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
- int maximumPairs=MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit)/2;
- int i=0;
- while(i<maximumPairs && m_iterator!=m_slaveSeedLengths.end()){
- int length=m_iterator->first;
- int count=m_iterator->second;
- messageBuffer[2*i]=length;
- messageBuffer[2*i+1]=count;
- i++;
- m_iterator++;
- }
-
- Message aMessage(messageBuffer,2*i,MASTER_RANK,
- RAY_MPI_TAG_SEND_SEED_LENGTHS,getRank());
- m_outbox->push_back(aMessage);
-}
-void SeedingData::writeSeedStatistics(){
- ostringstream file;
- file<<m_parameters->getPrefix();
- file<<"SeedLengthDistribution.txt";
- ofstream f(file.str().c_str());
- for(map<int,int>::iterator i=m_masterSeedLengths.begin();i!=m_masterSeedLengths.end();i++){
- int length=i->first;
- int count=i->second;
- f<<length<<"\t"<<count<<endl;
- }
- f.close();
+ Message aMessage(NULL,0,MASTER_RANK,
+ RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS,getRank());
+ m_outbox->push_back(&aMessage);
+ (*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
}
-
void SeedingData::loadCheckpoint(){
- cout<<"Rank "<<m_parameters->getRank()<<" is reading checkpoint Seeds"<<endl;
+ cout<<"Rank "<<m_parameters->getRank()<<" is reading checkpoint SimpleSeeds"<<endl;
- ifstream f(m_parameters->getCheckpointFile("Seeds").c_str());
+ ifstream f(m_parameters->getCheckpointFile("SimpleSeeds").c_str());
int n=0;
f.read((char*)&n,sizeof(int));
for(int i=0;i<n;i++){
- AssemblySeed seed;
+ GraphPath seed;
+ seed.setKmerLength(m_parameters->getWordSize());
int vertices=0;
f.read((char*)&vertices,sizeof(int));
for(int j=0;j<vertices;j++){
Kmer kmer;
kmer.read(&f);
seed.push_back(&kmer);
+
+ CoverageDepth coverageValue=0;
+
+ f.read((char*)&coverageValue,sizeof(CoverageDepth));
+ seed.addCoverageValue(coverageValue);
}
+
+ seed.computePeakCoverage();
+
m_SEEDING_seeds.push_back(seed);
}
- cout<<"Rank "<<m_parameters->getRank()<<" loaded "<<n<<" seeds from checkpoint Seeds"<<endl;
+ cout<<"Rank "<<m_parameters->getRank()<<" loaded "<<n<<" seeds from checkpoint SimpleSeeds"<<endl;
f.close();
}
+void SeedingData::writeCheckpoints(){
+
+ /* write the Seeds checkpoint */
+ if(m_parameters->writeCheckpoints() && !m_parameters->hasCheckpoint("SimpleSeeds")){
+
+ ofstream f(m_parameters->getCheckpointFile("SimpleSeeds").c_str());
+ ostringstream buffer;
+
+ cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint SimpleSeeds"<<endl;
+
+ vector<GraphPath> * seeds = & m_SEEDING_seeds;
+
+ int count=(*seeds).size();
+
+ buffer.write((char*)&count, sizeof(int));
+
+ for(int i=0;i<(int)(*seeds).size();i++){
+ int length=(*seeds)[i].size();
+ buffer.write((char*)&length, sizeof(int));
+
+ for(int j=0;j<(int)(*seeds)[i].size();j++){
+ Kmer theKmer;
+ (*seeds)[i].at(j,&theKmer);
+ theKmer.write(&buffer);
+
+ CoverageDepth coverageValue=0;
+ coverageValue=(*seeds)[i].getCoverageAt(j);
+ buffer.write((char*)&coverageValue, sizeof(CoverageDepth));
+ flushFileOperationBuffer(false, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
+ }
+ }
+ flushFileOperationBuffer(true, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
+ f.close();
+ }
+}
+
+
+
void SeedingData::registerPlugin(ComputeCore*core){
+
+ m_core=core;
PluginHandle plugin=core->allocatePluginHandle();
m_plugin=plugin;
@@ -438,17 +494,15 @@ void SeedingData::registerPlugin(ComputeCore*core){
core->setPluginAuthors(plugin,"Sébastien Boisvert");
core->setPluginLicense(plugin,"GNU General Public License version 3");
- RAY_SLAVE_MODE_START_SEEDING=core->allocateSlaveModeHandle(plugin);
- core->setSlaveModeObjectHandler(plugin,RAY_SLAVE_MODE_START_SEEDING, __GetAdapter(SeedingData,RAY_SLAVE_MODE_START_SEEDING));
- core->setSlaveModeSymbol(plugin,RAY_SLAVE_MODE_START_SEEDING,"RAY_SLAVE_MODE_START_SEEDING");
-
- RAY_SLAVE_MODE_SEND_SEED_LENGTHS=core->allocateSlaveModeHandle(plugin);
- core->setSlaveModeObjectHandler(plugin,RAY_SLAVE_MODE_SEND_SEED_LENGTHS, __GetAdapter(SeedingData,RAY_SLAVE_MODE_SEND_SEED_LENGTHS));
- core->setSlaveModeSymbol(plugin,RAY_SLAVE_MODE_SEND_SEED_LENGTHS,"RAY_SLAVE_MODE_SEND_SEED_LENGTHS");
+ __ConfigureSlaveModeHandler(SeedingData, RAY_SLAVE_MODE_START_SEEDING);
+ __ConfigureSlaveModeHandler(SeedingData, RAY_SLAVE_MODE_SEND_SEED_LENGTHS);
RAY_MPI_TAG_SEND_SEED_LENGTHS_REPLY=core->allocateMessageTagHandle(plugin);
core->setMessageTagSymbol(plugin,RAY_MPI_TAG_SEND_SEED_LENGTHS_REPLY,"RAY_MPI_TAG_SEND_SEED_LENGTHS_REPLY");
+ __BindPlugin(SeedingData);
+
+ m_core->setObjectSymbol(m_plugin, &m_SEEDING_seeds,"/RayAssembler/ObjectStore/Seeds.ray");
}
void SeedingData::resolveSymbols(ComputeCore*core){
@@ -463,6 +517,4 @@ void SeedingData::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_SEEDING_IS_OVER=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_SEEDING_IS_OVER");
RAY_MPI_TAG_SEND_SEED_LENGTHS=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_SEND_SEED_LENGTHS");
RAY_MPI_TAG_SEND_SEED_LENGTHS_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_SEND_SEED_LENGTHS_REPLY");
-
- __BindPlugin(SeedingData);
}
diff --git a/code/plugin_SeedingData/SeedingData.h b/code/SeedingData/SeedingData.h
similarity index 74%
rename from code/plugin_SeedingData/SeedingData.h
rename to code/SeedingData/SeedingData.h
index 38a8552..c120c4a 100644
--- a/code/plugin_SeedingData/SeedingData.h
+++ b/code/SeedingData/SeedingData.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -24,22 +24,27 @@
class SeedExtender;
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <communication/VirtualCommunicator.h>
-#include <plugin_SeedExtender/SeedExtender.h>
-#include <structures/SplayTreeIterator.h>
-#include <structures/SplayNode.h>
-#include <plugin_VerticesExtractor/Vertex.h>
-#include <plugin_SeedingData/AssemblySeed.h>
-#include <plugin_VerticesExtractor/GridTableIterator.h>
-#include <application_core/common_functions.h>
-#include <plugin_SeedingData/SeedWorker.h>
-#include <core/ComputeCore.h>
+#include "SeedWorker.h"
+
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/SeedExtender/SeedExtender.h>
+#include <code/SeedingData/GraphPath.h>
+#include <code/VerticesExtractor/GridTableIterator.h>
+#include <code/VerticesExtractor/Vertex.h>
+#include <code/Mock/common_functions.h>
+
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/structures/SplayTreeIterator.h>
+#include <RayPlatform/structures/SplayNode.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <set>
using namespace std;
+__DeclarePlugin(SeedingData);
+__DeclareSlaveModeAdapter(SeedingData,RAY_SLAVE_MODE_START_SEEDING);
+__DeclareSlaveModeAdapter(SeedingData,RAY_SLAVE_MODE_SEND_SEED_LENGTHS);
/*
* Computes seeds in the k-mer graph.
@@ -51,6 +56,9 @@ using namespace std;
*/
class SeedingData : public CorePlugin{
+ __AddAdapter(SeedingData,RAY_SLAVE_MODE_START_SEEDING);
+ __AddAdapter(SeedingData,RAY_SLAVE_MODE_SEND_SEED_LENGTHS);
+
/**
* the minimum seed coverage allowed
*/
@@ -69,7 +77,22 @@ class SeedingData : public CorePlugin{
SlaveMode RAY_SLAVE_MODE_START_SEEDING;
SlaveMode RAY_SLAVE_MODE_SEND_SEED_LENGTHS;
+/*
+ * Some counters for sorting objects.
+ */
+
+ int m_skippedObjectsWithDeadEndForHead;
+ int m_skippedObjectsWithDeadEndForTail;
+ int m_skippedObjectsWithTwoDeadEnds;
+ int m_skippedObjectsWithBubbleWeakComponent;
+
+ int m_skippedNotMine;
+ int m_skippedTooShort;
+ int m_skippedNotEnoughCoverage;
+
+ int m_eligiblePaths;
+ bool m_debugSeeds;
/** checkpointing */
bool m_checkedCheckpoint;
@@ -103,11 +126,13 @@ class SeedingData : public CorePlugin{
time_t m_last;
void loadCheckpoint();
+
+ void writeCheckpoints();
public:
+ // TODO: move these attributes in the private zone
map<int,int> m_masterSeedLengths;
map<int,int> m_slaveSeedLengths;
-
bool m_SEEDING_edgesRequested;
int m_SEEDING_ingoingEdgeIndex;
@@ -135,7 +160,7 @@ public:
int m_SEEDING_outgoing_index;
bool m_SEEDING_outgoing_choice_done;
int m_SEEDING_currentRank;
- vector<AssemblySeed> m_SEEDING_seeds;
+ vector<GraphPath> m_SEEDING_seeds;
vector<int> m_SEEDING_outgoingCoverages;
vector<uint64_t> m_SEEDING_outgoingKeys;
bool m_SEEDING_vertexKeyAndCoverageRequested;
@@ -147,14 +172,13 @@ public:
void call_RAY_SLAVE_MODE_START_SEEDING();
void constructor(SeedExtender*seedExtender,int rank,int size,StaticVector*outbox,RingAllocator*outboxAllocator,
- int*mode,Parameters*parameters,int*wordSize,GridTable*subgraph,
+ int*mode,Parameters*parameters,GridTable*subgraph,
StaticVector*inbox,VirtualCommunicator*vc);
void updateStates();
void call_RAY_SLAVE_MODE_SEND_SEED_LENGTHS();
bool m_initialized;
- void writeSeedStatistics();
void registerPlugin(ComputeCore*core);
void resolveSymbols(ComputeCore*core);
diff --git a/code/plugin_SequencesIndexer/DynamicVector.h b/code/SequencesIndexer/DynamicVector.h
similarity index 93%
rename from code/plugin_SequencesIndexer/DynamicVector.h
rename to code/SequencesIndexer/DynamicVector.h
index 9e40432..3bf4d24 100644
--- a/code/plugin_SequencesIndexer/DynamicVector.h
+++ b/code/SequencesIndexer/DynamicVector.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,7 +22,10 @@
#ifndef _DynamicVector_H
#define _DynamicVector_H
-#include <memory/MyAllocator.h>
+#include <RayPlatform/memory/MyAllocator.h>
+
+#include <stdlib.h>
+#include <string.h>
/**
* a dynamic vector, this is a template
diff --git a/code/plugin_SequencesIndexer/IndexerWorker.cpp b/code/SequencesIndexer/IndexerWorker.cpp
similarity index 97%
rename from code/plugin_SequencesIndexer/IndexerWorker.cpp
rename to code/SequencesIndexer/IndexerWorker.cpp
index 39b4ab6..8027d7c 100644
--- a/code/plugin_SequencesIndexer/IndexerWorker.cpp
+++ b/code/SequencesIndexer/IndexerWorker.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,9 +19,11 @@
*/
-#include <plugin_SequencesIndexer/IndexerWorker.h>
+#include "IndexerWorker.h"
+
+#include <RayPlatform/core/statistics.h>
+
#include <string.h>
-#include <core/statistics.h>
void IndexerWorker::constructor(int sequenceId,Parameters*parameters,RingAllocator*outboxAllocator,
VirtualCommunicator*vc,WorkerHandle workerId,ArrayOfReads*a,MyAllocator*allocator,
@@ -77,7 +79,7 @@ void IndexerWorker::work(){
}else if(!m_coverageRequested){
Kmer vertex=read->getVertex(m_position,m_parameters->getWordSize(),'F',m_parameters->getColorSpaceMode());
m_vertices.push_back(vertex,m_allocator);
- int sendTo=m_parameters->_vertexRank(&vertex);
+ int sendTo=m_parameters->vertexRank(&vertex);
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(MessageUnit));
int bufferPosition=0;
vertex.pack(message,&bufferPosition);
@@ -146,7 +148,7 @@ void IndexerWorker::work(){
// index it
if(selectedPosition!=-1){
Kmer vertex=(m_vertices).at(selectedPosition);
- int sendTo=m_parameters->_vertexRank(&vertex);
+ int sendTo=m_parameters->vertexRank(&vertex);
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(5*sizeof(MessageUnit));
int j=0;
vertex.pack(message,&j);
@@ -223,7 +225,7 @@ void IndexerWorker::work(){
if(selectedPosition!=-1){
Kmer tmp=m_vertices.at(selectedPosition);
Kmer vertex=m_parameters->_complementVertex(&tmp);
- int sendTo=m_parameters->_vertexRank(&vertex);
+ int sendTo=m_parameters->vertexRank(&vertex);
MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(5*sizeof(MessageUnit));
int positionOnStrand=read->length()-m_parameters->getWordSize()-selectedPosition;
int j=0;
diff --git a/code/plugin_SequencesIndexer/IndexerWorker.h b/code/SequencesIndexer/IndexerWorker.h
similarity index 86%
rename from code/plugin_SequencesIndexer/IndexerWorker.h
rename to code/SequencesIndexer/IndexerWorker.h
index bd39faa..37b489f 100644
--- a/code/plugin_SequencesIndexer/IndexerWorker.h
+++ b/code/SequencesIndexer/IndexerWorker.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,14 +22,17 @@
#ifndef _IndexerWorker
#define _IndexerWorker
-#include <application_core/Parameters.h>
-#include <memory/RingAllocator.h>
-#include <communication/VirtualCommunicator.h>
+#include "DynamicVector.h"
+
+#include <code/Mock/Parameters.h>
+#include <code/SequencesLoader/ArrayOfReads.h>
+
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/memory/MyAllocator.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/scheduling/Worker.h>
+
#include <stdint.h>
-#include <memory/MyAllocator.h>
-#include <plugin_SequencesLoader/ArrayOfReads.h>
-#include <plugin_SequencesIndexer/DynamicVector.h>
-#include <scheduling/Worker.h>
#include <fstream>
using namespace std;
diff --git a/code/SequencesIndexer/Makefile b/code/SequencesIndexer/Makefile
new file mode 100644
index 0000000..d053125
--- /dev/null
+++ b/code/SequencesIndexer/Makefile
@@ -0,0 +1,9 @@
+SequencesIndexer-y += code/SequencesIndexer/SequencesIndexer.o
+SequencesIndexer-y += code/SequencesIndexer/IndexerWorker.o
+SequencesIndexer-y += code/SequencesIndexer/PairedRead.o
+SequencesIndexer-y += code/SequencesIndexer/ReadAnnotation.o
+
+obj-y += $(SequencesIndexer-y)
+
+
+
diff --git a/code/plugin_SequencesIndexer/PairedRead.cpp b/code/SequencesIndexer/PairedRead.cpp
similarity index 81%
rename from code/plugin_SequencesIndexer/PairedRead.cpp
rename to code/SequencesIndexer/PairedRead.cpp
index ae66ce5..5175609 100644
--- a/code/plugin_SequencesIndexer/PairedRead.cpp
+++ b/code/SequencesIndexer/PairedRead.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,9 +19,11 @@
*/
-#include<plugin_SequencesIndexer/PairedRead.h>
-#include<application_core/common_functions.h>
-#include<assert.h>
+#include "PairedRead.h"
+
+#include <code/Mock/common_functions.h>
+
+#include <assert.h>
void PairedRead::constructor(int rank,int id,int library){
m_rank=rank;
@@ -39,20 +41,22 @@ uint32_t PairedRead::getId(){
/** any read has a unique distributed identifier */
ReadHandle PairedRead::getUniqueId(){
- return getPathUniqueId(m_rank,m_readIndex);
+ PathHandle handle = getPathUniqueId(m_rank,m_readIndex);
+
+ return handle.getValue();
}
int PairedRead::getLibrary(){
return m_library;
}
-void PairedRead::write(ofstream*f){
+void PairedRead::write(ostream*f){
f->write((char*)&m_readIndex,sizeof(uint32_t));
f->write((char*)&m_rank,sizeof(uint16_t));
f->write((char*)&m_library,sizeof(uint16_t));
}
-void PairedRead::read(ifstream*f){
+void PairedRead::read(istream*f){
f->read((char*)&m_readIndex,sizeof(uint32_t));
f->read((char*)&m_rank,sizeof(uint16_t));
f->read((char*)&m_library,sizeof(uint16_t));
diff --git a/code/plugin_SequencesIndexer/PairedRead.h b/code/SequencesIndexer/PairedRead.h
similarity index 85%
rename from code/plugin_SequencesIndexer/PairedRead.h
rename to code/SequencesIndexer/PairedRead.h
index 38405c7..49d35da 100644
--- a/code/plugin_SequencesIndexer/PairedRead.h
+++ b/code/SequencesIndexer/PairedRead.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,7 +22,9 @@
#ifndef _PairedRead
#define _PairedRead
-#include<application_core/common_functions.h>
+#include <code/SequencesLoader/ReadHandle.h>
+#include <code/Mock/common_functions.h>
+
#include <fstream>
using namespace std;
@@ -38,7 +40,7 @@ using namespace std;
class PairedRead{
uint32_t m_readIndex;
uint16_t m_rank; // should be Rank
- uint16_t m_library;
+ LibraryHandle m_library;
public:
void constructor(int rank,int id,int library);
Rank getRank();
@@ -46,8 +48,8 @@ public:
ReadHandle getUniqueId();
int getLibrary();
- void read(ifstream*f);
- void write(ofstream*f);
+ void read(istream*f);
+ void write(ostream*f);
} ATTRIBUTE_PACKED;
#endif
diff --git a/code/plugin_SequencesIndexer/ReadAnnotation.cpp b/code/SequencesIndexer/ReadAnnotation.cpp
similarity index 85%
rename from code/plugin_SequencesIndexer/ReadAnnotation.cpp
rename to code/SequencesIndexer/ReadAnnotation.cpp
index 40adf58..3ac8032 100644
--- a/code/plugin_SequencesIndexer/ReadAnnotation.cpp
+++ b/code/SequencesIndexer/ReadAnnotation.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,10 +19,12 @@
*/
-#include<assert.h>
-#include<plugin_SequencesIndexer/ReadAnnotation.h>
-#include<stdlib.h>
-#include<application_core/common_functions.h>
+#include "ReadAnnotation.h"
+
+#include <code/Mock/common_functions.h>
+
+#include <assert.h>
+#include <stdlib.h>
void ReadAnnotation::constructor(int rank,int readIndex,int positionOnStrand,char c,bool lower){
m_lower=lower;
@@ -58,14 +60,16 @@ ReadAnnotation*ReadAnnotation::getNext()const{
}
ReadHandle ReadAnnotation::getUniqueId()const{
- return getPathUniqueId(m_rank,m_readIndex);
+ PathHandle handle = getPathUniqueId(m_rank,m_readIndex);
+
+ return handle.getValue();
}
bool ReadAnnotation::isLower(){
return m_lower;
}
-void ReadAnnotation::write(ofstream*f){
+void ReadAnnotation::write(ostream*f){
int rank=getRank();
int readIndex=getReadIndex();
int positionOnStrand=getPositionOnStrand();
@@ -76,7 +80,7 @@ void ReadAnnotation::write(ofstream*f){
f->write((char*)&strand,sizeof(char));
}
-void ReadAnnotation::read(ifstream*f,bool isLower){
+void ReadAnnotation::read(istream*f,bool isLower){
int rank=0;
int readIndex=0;
int positionOnStrand=0;
diff --git a/code/plugin_SequencesIndexer/ReadAnnotation.h b/code/SequencesIndexer/ReadAnnotation.h
similarity index 87%
rename from code/plugin_SequencesIndexer/ReadAnnotation.h
rename to code/SequencesIndexer/ReadAnnotation.h
index 32b94eb..f232b88 100644
--- a/code/plugin_SequencesIndexer/ReadAnnotation.h
+++ b/code/SequencesIndexer/ReadAnnotation.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,7 +22,9 @@
#ifndef _ReadAnnotation
#define _ReadAnnotation
-#include <application_core/common_functions.h>
+#include <code/SequencesLoader/ReadHandle.h>
+#include <code/Mock/common_functions.h>
+
#include <fstream>
using namespace std;
@@ -50,8 +52,8 @@ public:
void setNext(ReadAnnotation*a);
ReadHandle getUniqueId() const;
- void read(ifstream*f,bool isLower);
- void write(ofstream*f);
+ void read(istream*f,bool isLower);
+ void write(ostream*f);
} ATTRIBUTE_PACKED;
#endif
diff --git a/code/plugin_SequencesIndexer/SequencesIndexer.cpp b/code/SequencesIndexer/SequencesIndexer.cpp
similarity index 95%
rename from code/plugin_SequencesIndexer/SequencesIndexer.cpp
rename to code/SequencesIndexer/SequencesIndexer.cpp
index 608cecb..a72b67c 100644
--- a/code/plugin_SequencesIndexer/SequencesIndexer.cpp
+++ b/code/SequencesIndexer/SequencesIndexer.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -18,24 +18,22 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_SequencesIndexer/SequencesIndexer.h>
-#include <string.h>
-#include <core/OperatingSystem.h>
+#include "SequencesIndexer.h"
+
+#include <code/SequencesLoader/Loader.h>
+#include <code/Mock/Parameters.h>
+#include <code/Mock/common_functions.h>
+
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/communication/Message.h>
+
#include <stdlib.h>
#include <assert.h>
-#include <application_core/Parameters.h>
-#include <plugin_SequencesLoader/Loader.h>
-#include <application_core/common_functions.h>
-#include <communication/Message.h>
+#include <string.h>
__CreatePlugin(SequencesIndexer);
- /**/
- /**/
-__CreateSlaveModeAdapter(SequencesIndexer,RAY_SLAVE_MODE_INDEX_SEQUENCES); /**/
- /**/
- /**/
-
+__CreateSlaveModeAdapter(SequencesIndexer,RAY_SLAVE_MODE_INDEX_SEQUENCES);
void SequencesIndexer::call_RAY_SLAVE_MODE_INDEX_SEQUENCES(){
if(!m_initiatedIterator){
@@ -54,7 +52,7 @@ void SequencesIndexer::call_RAY_SLAVE_MODE_INDEX_SEQUENCES(){
cout<<"Rank "<<m_parameters->getRank()<<": checkpoint OptimalMarkers exists, not selecting markers."<<endl;
(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
return;
}
m_checkedCheckpoint=true;
@@ -97,9 +95,8 @@ void SequencesIndexer::call_RAY_SLAVE_MODE_INDEX_SEQUENCES(){
// AND
// the number of alive workers is below the maximum
if(m_theSequenceId<(int)m_myReads->size()&&(int)m_aliveWorkers.size()<m_maximumAliveWorkers){
- if(m_theSequenceId%10000==0){
+ if(m_theSequenceId%100000==0){
printf("Rank %i is selecting optimal read markers [%i/%i]\n",m_rank,m_theSequenceId+1,(int)m_myReads->size());
- fflush(stdout);
m_derivative.addX(m_theSequenceId);
m_derivative.printStatus(SLAVE_MODES[RAY_SLAVE_MODE_INDEX_SEQUENCES],RAY_SLAVE_MODE_INDEX_SEQUENCES);
@@ -146,12 +143,10 @@ void SequencesIndexer::call_RAY_SLAVE_MODE_INDEX_SEQUENCES(){
if((int)m_myReads->size()==m_completedJobs){
printf("Rank %i is selecting optimal read markers [%i/%i] (completed)\n",m_rank,(int)m_myReads->size(),(int)m_myReads->size());
- fflush(stdout);
printf("Rank %i: peak number of workers: %i, maximum: %i\n",m_rank,m_maximumWorkers,m_maximumAliveWorkers);
- fflush(stdout);
(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_derivative.writeFile(&cout);
@@ -342,4 +337,7 @@ void SequencesIndexer::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_VERTEX_READS_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_VERTEX_READS_REPLY");
__BindPlugin(SequencesIndexer);
+
+ __BindAdapter(SequencesIndexer,RAY_SLAVE_MODE_INDEX_SEQUENCES);
+
}
diff --git a/code/plugin_SequencesIndexer/SequencesIndexer.h b/code/SequencesIndexer/SequencesIndexer.h
similarity index 75%
rename from code/plugin_SequencesIndexer/SequencesIndexer.h
rename to code/SequencesIndexer/SequencesIndexer.h
index 9a908e2..895142f 100644
--- a/code/plugin_SequencesIndexer/SequencesIndexer.h
+++ b/code/SequencesIndexer/SequencesIndexer.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,27 +22,32 @@
#ifndef _SequencesIndexer
#define _SequencesIndexer
-#include <plugin_SequencesIndexer/IndexerWorker.h>
-#include <communication/Message.h>
-#include <communication/BufferedData.h>
-#include <communication/VirtualCommunicator.h>
-#include <application_core/Parameters.h>
-#include <application_core/common_functions.h>
-#include <memory/MyAllocator.h>
-#include <memory/RingAllocator.h>
-#include <structures/SplayTree.h>
-#include <structures/SplayTreeIterator.h>
-#include <structures/StaticVector.h>
-#include <plugin_SequencesLoader/ArrayOfReads.h>
-#include <profiling/Derivative.h>
-#include <plugin_SequencesLoader/Read.h>
-#include <core/ComputeCore.h>
+#include "IndexerWorker.h"
+
+#include <code/Mock/Parameters.h>
+#include <code/Mock/common_functions.h>
+#include <code/SequencesLoader/ArrayOfReads.h>
+#include <code/SequencesLoader/Read.h>
+
+#include <RayPlatform/profiling/Derivative.h>
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/memory/MyAllocator.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/structures/SplayTree.h>
+#include <RayPlatform/structures/SplayTreeIterator.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/communication/BufferedData.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
#include <map>
#include <vector>
#include <fstream>
using namespace std;
+__DeclarePlugin(SequencesIndexer);
+
+__DeclareSlaveModeAdapter(SequencesIndexer,RAY_SLAVE_MODE_INDEX_SEQUENCES);
/*
* Computes optimal read markers using workers.
@@ -50,6 +55,8 @@ using namespace std;
*/
class SequencesIndexer: public CorePlugin{
+ __AddAdapter(SequencesIndexer,RAY_SLAVE_MODE_INDEX_SEQUENCES);
+
MessageTag RAY_MPI_TAG_GET_READ_MARKERS_REPLY;
MessageTag RAY_MPI_TAG_GET_READ_MATE_REPLY;
MessageTag RAY_MPI_TAG_REQUEST_VERTEX_READS_REPLY;
diff --git a/code/plugin_SequencesLoader/ArrayOfReads.cpp b/code/SequencesLoader/ArrayOfReads.cpp
similarity index 96%
rename from code/plugin_SequencesLoader/ArrayOfReads.cpp
rename to code/SequencesLoader/ArrayOfReads.cpp
index 3461dc7..830cc86 100644
--- a/code/plugin_SequencesLoader/ArrayOfReads.cpp
+++ b/code/SequencesLoader/ArrayOfReads.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,12 +14,13 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_SequencesLoader/ArrayOfReads.h>
+#include "ArrayOfReads.h"
+
#include <stdlib.h>
#include <assert.h>
#include <iostream>
diff --git a/code/plugin_SequencesLoader/ArrayOfReads.h b/code/SequencesLoader/ArrayOfReads.h
similarity index 90%
rename from code/plugin_SequencesLoader/ArrayOfReads.h
rename to code/SequencesLoader/ArrayOfReads.h
index a8c6149..f758404 100644
--- a/code/plugin_SequencesLoader/ArrayOfReads.h
+++ b/code/SequencesLoader/ArrayOfReads.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,7 +14,7 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
@@ -22,7 +22,7 @@
#ifndef _ArrayOfReads
#define _ArrayOfReads
-#include<plugin_SequencesLoader/Read.h>
+#include "Read.h"
/**
* This class holds reads. These are stored in chunks, which are utterly linked in chains.
diff --git a/code/SequencesLoader/BufferedReader.cpp b/code/SequencesLoader/BufferedReader.cpp
new file mode 100644
index 0000000..e710c2e
--- /dev/null
+++ b/code/SequencesLoader/BufferedReader.cpp
@@ -0,0 +1,206 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "BufferedReader.h"
+
+#include <code/Mock/constants.h>
+
+#include <iostream>
+using namespace std;
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#ifdef CONFIG_ASSERT
+#include <assert.h>
+#endif
+
+//#define CONFIG_FASTQ_DEBUG_MESSAGE
+#define CONFIG_FASTQ_BUFFER_SIZE SIZE_4M
+
+void BufferedReader::initialize() {
+ m_assetCacheMaximumLength = CONFIG_FASTQ_BUFFER_SIZE;
+ m_assetCacheContent = (char*) malloc(m_assetCacheMaximumLength * sizeof(char));
+
+ reset();
+}
+
+void BufferedReader::reset() {
+ m_assetCacheLength = 0;
+ m_assetCacheOffset = 0;
+}
+
+/**
+ * This is a drop-in replacement for fgets with buffering features.
+ *
+ * \see http://www.cplusplus.com/reference/cstdio/fgets/
+ * \see http://www.cplusplus.com/reference/cstdio/feof/
+ * \see http://www.cplusplus.com/reference/cstdio/fread/
+ *
+ * TODO This method will not work properly if numberOfBytes is > CONFIG_FASTQ_BUFFER_SIZE
+ * TODO Also, it will not work if lines from the content stream are longer than CONFIG_FASTQ_BUFFER_SIZE.
+ *
+ * \author Sébastien Boisvert
+ * \see https://github.com/sebhtml/ray/issues/137
+ *
+ *
+ * m_assetCacheContent -> the content
+ * m_assetCacheLength -> the size of the content
+ * m_assetCacheOffset -> the current offset in the content
+ * m_assetCacheMaximumLength -> the maximum size
+ */
+char * BufferedReader::readLine(char * content, int numberOfBytes, FILE * endpoint){
+
+ //cout << "[DEBUG] readLine" << endl;
+
+ //return fgets(content, numberOfBytes, endpoint);
+
+ return readLineFromBuffer(content, numberOfBytes, endpoint, true);
+}
+
+/**
+ *
+ */
+char * BufferedReader::readLineFromBuffer(char * content, int numberOfBytes, FILE * endpoint, bool retry) {
+
+ if(copyWithNewLine(content))
+ return content;
+
+ if(retry) {
+ fillBuffer(endpoint);
+ return readLineFromBuffer(content, numberOfBytes, endpoint, false);
+ }
+
+ if(copyWithMaximumNumberOfBytes(content, numberOfBytes))
+ return content;
+
+ return NULL;
+}
+
+void BufferedReader::fillBuffer(FILE * source) {
+
+ //cout << "[DEBUG] fillBuffer " << endl;
+ moveBuffer();
+
+ char * buffer = getBuffer();
+ int bufferSize = getBufferSize();
+ int maximumBufferSize = getMaximumBufferSize();
+ int available = maximumBufferSize - bufferSize;
+
+ char * destination = buffer + bufferSize;
+
+ int elements = fread(destination, sizeof(char), available, source);
+
+ m_assetCacheLength += elements;
+}
+
+int BufferedReader::getMaximumBufferSize() {
+ return m_assetCacheMaximumLength;
+}
+
+void BufferedReader::moveBuffer() {
+ int source = m_assetCacheOffset;
+ int destination = 0;
+
+ while(source < m_assetCacheLength) {
+ m_assetCacheContent[destination] = m_assetCacheContent[source];
+ destination++;
+ source++;
+ }
+
+ m_assetCacheLength -= m_assetCacheOffset;
+ m_assetCacheOffset = 0;
+}
+
+char * BufferedReader::getBuffer() {
+ return m_assetCacheContent + m_assetCacheOffset;
+}
+
+int BufferedReader::getBufferSize() {
+ return m_assetCacheLength - m_assetCacheOffset;
+}
+
+int BufferedReader::findNewLine(char * sequence, int length) {
+
+ int i = 0;
+
+ while(i < length) {
+ if(sequence[i] == '\n')
+ return i;
+ i++;
+ }
+
+ return -1;
+}
+
+bool BufferedReader::copyWithNewLine(char * content) {
+
+ char * buffer = getBuffer();
+ int bufferSize = getBufferSize();
+
+ int newLinePosition = findNewLine(buffer, bufferSize);
+
+ if(newLinePosition < 0)
+ return false;
+
+ int count = newLinePosition + 1;
+
+ consumeContent(buffer, content, count);
+
+ return true;
+}
+
+bool BufferedReader::copyWithMaximumNumberOfBytes(char * content, int numberOfBytes) {
+
+ int availableBytes = getBufferSize();
+
+ if(availableBytes == 0)
+ return false;
+
+ char * buffer = getBuffer();
+
+ int count = numberOfBytes;
+
+ if(availableBytes < count)
+ count = availableBytes;
+
+ consumeContent(buffer, content, count);
+
+ return true;
+
+}
+
+void BufferedReader::consumeContent(char * buffer, char * content, int count) {
+ memcpy(content, buffer, count);
+ content[count] = '\0';
+
+ m_assetCacheOffset += count;
+}
+
+void BufferedReader::destroy() {
+ if(m_assetCacheContent != NULL) {
+ free(m_assetCacheContent);
+ }
+ m_assetCacheContent = NULL;
+ m_assetCacheLength = 0;
+ m_assetCacheMaximumLength = 0;
+
+}
diff --git a/code/SequencesLoader/BufferedReader.h b/code/SequencesLoader/BufferedReader.h
new file mode 100644
index 0000000..93a60b6
--- /dev/null
+++ b/code/SequencesLoader/BufferedReader.h
@@ -0,0 +1,66 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+
+
+#ifndef BufferedReaderHeader
+#define BufferedReaderHeader
+
+#include <stdio.h>
+
+/**
+ * This is a buffered reader for files.
+ *
+ * readLine takes the same arguments than fgets
+ *
+ * \author Sébastien Boisvert
+ */
+class BufferedReader {
+
+/**
+ * This is the buffer for storing asset cache streamed from the endpoint.
+ */
+ char * m_assetCacheContent;
+ int m_assetCacheLength;
+ int m_assetCacheMaximumLength;
+ int m_assetCacheOffset;
+
+ char * readLineFromBuffer(char * content, int numberOfBytes, FILE * endpoint, bool retry);
+ void fillBuffer(FILE * source);
+ int getMaximumBufferSize();
+ void moveBuffer();
+ char * getBuffer();
+ int getBufferSize();
+ int findNewLine(char * sequence, int length);
+ bool copyWithNewLine(char * content);
+ bool copyWithMaximumNumberOfBytes(char * content, int numberOfBytes);
+
+ void consumeContent(char * buffer, char * content, int count);
+
+public:
+
+ void reset();
+ void initialize();
+ void destroy();
+ char * readLine(char * content, int numberOfBytes, FILE * endpoint);
+
+};
+
+#endif
diff --git a/code/plugin_SequencesLoader/BzReader.cpp b/code/SequencesLoader/BzReader.cpp
similarity index 93%
rename from code/plugin_SequencesLoader/BzReader.cpp
rename to code/SequencesLoader/BzReader.cpp
index 03e4d59..66474f9 100644
--- a/code/plugin_SequencesLoader/BzReader.cpp
+++ b/code/SequencesLoader/BzReader.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,19 +14,21 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#ifdef HAVE_LIBBZ2
+#ifdef CONFIG_HAVE_LIBBZ2
-#define __BzReader_MAXIMUM_LENGTH 2*4096
+#define __BzReader_MAXIMUM_LENGTH RAY_MAXIMUM_READ_LENGTH*2
+
+#include "BzReader.h"
+
+#include <code/Mock/common_functions.h>
-#include <plugin_SequencesLoader/BzReader.h>
#include <stdlib.h>
#include <assert.h>
-#include <application_core/common_functions.h>
void BzReader::open(const char*file){
m_file=fopen(file,"r");
@@ -61,6 +63,9 @@ char*BzReader::readLine(char*s, int n){
//cout<<"[BzReader::readLine]"<<endl;
#ifdef ASSERT
+ if(!(n<=__BzReader_MAXIMUM_LENGTH)){
+ cout<<"Expected: "<<__BzReader_MAXIMUM_LENGTH<<" Actual: "<<n<<endl;
+ }
assert(n<=__BzReader_MAXIMUM_LENGTH);
#endif
diff --git a/code/plugin_SequencesLoader/BzReader.h b/code/SequencesLoader/BzReader.h
similarity index 88%
rename from code/plugin_SequencesLoader/BzReader.h
rename to code/SequencesLoader/BzReader.h
index f52f0ec..060a371 100644
--- a/code/plugin_SequencesLoader/BzReader.h
+++ b/code/SequencesLoader/BzReader.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,7 +14,7 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
@@ -22,7 +22,7 @@
#ifndef _BzReader
#define _BzReader
-#ifdef HAVE_LIBBZ2
+#ifdef CONFIG_HAVE_LIBBZ2
#include <bzlib.h>
#include <stdint.h>
@@ -51,6 +51,6 @@ public:
void close();
};
-#endif /* HAVE_LIBBZ2 */
+#endif
#endif /* _BzReader */
diff --git a/code/plugin_SequencesLoader/ColorSpaceDecoder.cpp b/code/SequencesLoader/ColorSpaceDecoder.cpp
similarity index 76%
rename from code/plugin_SequencesLoader/ColorSpaceDecoder.cpp
rename to code/SequencesLoader/ColorSpaceDecoder.cpp
index 2582f15..fa8ce49 100644
--- a/code/plugin_SequencesLoader/ColorSpaceDecoder.cpp
+++ b/code/SequencesLoader/ColorSpaceDecoder.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,15 +14,18 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include<iostream>
-#include<string.h>
-#include<plugin_SequencesLoader/ColorSpaceDecoder.h>
-#include<stdlib.h>
+#include "ColorSpaceDecoder.h"
+
+#include <code/Mock/constants.h>
+
+#include <iostream>
+#include <string.h>
+#include <stdlib.h>
using namespace std;
#define _COLOR_SPACE_DECODER_CODE_A 0
@@ -33,14 +36,10 @@ using namespace std;
#define _COLOR_SPACE_DECODER_COLOR_GREEN '1'
#define _COLOR_SPACE_DECODER_COLOR_ORANGE '2'
#define _COLOR_SPACE_DECODER_COLOR_RED '3'
-#define _COLOR_SPACE_DECODER_LETTER_A 'A'
-#define _COLOR_SPACE_DECODER_LETTER_T 'T'
-#define _COLOR_SPACE_DECODER_LETTER_C 'C'
-#define _COLOR_SPACE_DECODER_LETTER_G 'G'
#define _COLOR_SPACE_DECODER_NUMBER_OF_COLORS 4
/*
- * see http://www.ploscompbiol.org/article/slideshow.action?uri=info:doi/10.1371/journal.pcbi.1000386&imageURI=info:doi/10.1371/journal.pcbi.1000386.g002
+ * \see http://www.ploscompbiol.org/article/slideshow.action?uri=info:doi/10.1371/journal.pcbi.1000386&imageURI=info:doi/10.1371/journal.pcbi.1000386.g002
*/
ColorSpaceDecoder::ColorSpaceDecoder(){
int i=0;
@@ -78,13 +77,13 @@ string ColorSpaceDecoder::decode(char*x){
char v[1024];
int lastCode=0;
char boot=x[0];
- if(boot==_COLOR_SPACE_DECODER_LETTER_A){
+ if(boot==SYMBOL_A){
lastCode=_COLOR_SPACE_DECODER_CODE_A;
- }else if(boot==_COLOR_SPACE_DECODER_LETTER_T){
+ }else if(boot==SYMBOL_T){
lastCode=_COLOR_SPACE_DECODER_CODE_T;
- }else if(boot==_COLOR_SPACE_DECODER_LETTER_C){
+ }else if(boot==SYMBOL_C){
lastCode=_COLOR_SPACE_DECODER_CODE_C;
- }else if(boot==_COLOR_SPACE_DECODER_LETTER_G){
+ }else if(boot==SYMBOL_G){
lastCode=_COLOR_SPACE_DECODER_CODE_G;
}
int i=1;
@@ -98,13 +97,13 @@ string ColorSpaceDecoder::decode(char*x){
currentCode++;
}
if(currentCode==_COLOR_SPACE_DECODER_CODE_A){
- v[i-1]=_COLOR_SPACE_DECODER_LETTER_A;
+ v[i-1]=SYMBOL_A;
}else if(currentCode==_COLOR_SPACE_DECODER_CODE_C){
- v[i-1]=_COLOR_SPACE_DECODER_LETTER_C;
+ v[i-1]=SYMBOL_C;
}else if(currentCode==_COLOR_SPACE_DECODER_CODE_G){
- v[i-1]=_COLOR_SPACE_DECODER_LETTER_G;
+ v[i-1]=SYMBOL_G;
}else if(currentCode==_COLOR_SPACE_DECODER_CODE_T){
- v[i-1]=_COLOR_SPACE_DECODER_LETTER_T;
+ v[i-1]=SYMBOL_T;
}
lastCode=currentCode;
i++;
diff --git a/code/plugin_SequencesLoader/ColorSpaceDecoder.h b/code/SequencesLoader/ColorSpaceDecoder.h
similarity index 81%
rename from code/plugin_SequencesLoader/ColorSpaceDecoder.h
rename to code/SequencesLoader/ColorSpaceDecoder.h
index d5f6e87..cb1459f 100644
--- a/code/plugin_SequencesLoader/ColorSpaceDecoder.h
+++ b/code/SequencesLoader/ColorSpaceDecoder.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,7 +14,7 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
@@ -22,7 +22,7 @@
#ifndef _ColorSpaceDecoder
#define _ColorSpaceDecoder
-#include<string>
+#include <string>
using namespace std;
/**
diff --git a/code/plugin_SequencesLoader/ColorSpaceLoader.cpp b/code/SequencesLoader/ColorSpaceLoader.cpp
similarity index 79%
rename from code/plugin_SequencesLoader/ColorSpaceLoader.cpp
rename to code/SequencesLoader/ColorSpaceLoader.cpp
index 289c99c..bb87278 100644
--- a/code/plugin_SequencesLoader/ColorSpaceLoader.cpp
+++ b/code/SequencesLoader/ColorSpaceLoader.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,20 +14,27 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include<stdlib.h>
-#include<plugin_SequencesLoader/ColorSpaceLoader.h>
-#include<fstream>
-#include<application_core/common_functions.h>
-#include<iostream>
-#include<string.h>
+#include "ColorSpaceLoader.h"
+
+#include <code/Mock/common_functions.h>
+
+#include <stdlib.h>
+#include <fstream>
+#include <iostream>
+#include <string.h>
#include <assert.h>
using namespace std;
+ColorSpaceLoader::ColorSpaceLoader() {
+ addExtension(".csfasta");
+ addExtension(".csfa");
+}
+
int ColorSpaceLoader::open(string file){
m_f=fopen(file.c_str(),"r");
m_size=0;
@@ -68,13 +75,13 @@ void ColorSpaceLoader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAl
for(int j=0;j<(int)strlen(bufferForLine);j++){
if(bufferForLine[j]==DOUBLE_ENCODING_A_COLOR){
- bufferForLine[j]='A';
+ bufferForLine[j]=SYMBOL_A;
}else if(bufferForLine[j]==DOUBLE_ENCODING_T_COLOR){
- bufferForLine[j]='T';
+ bufferForLine[j]=SYMBOL_T;
}else if(bufferForLine[j]==DOUBLE_ENCODING_C_COLOR){
- bufferForLine[j]='C';
+ bufferForLine[j]=SYMBOL_C;
}else if(bufferForLine[j]==DOUBLE_ENCODING_G_COLOR){
- bufferForLine[j]='G';
+ bufferForLine[j]=SYMBOL_G;
}
}
Read t;
@@ -93,3 +100,6 @@ void ColorSpaceLoader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAl
int ColorSpaceLoader::getSize(){
return m_size;
}
+
+void ColorSpaceLoader::close(){
+}
diff --git a/code/plugin_SequencesLoader/ColorSpaceLoader.h b/code/SequencesLoader/ColorSpaceLoader.h
similarity index 71%
copy from code/plugin_SequencesLoader/ColorSpaceLoader.h
copy to code/SequencesLoader/ColorSpaceLoader.h
index 97fb696..d6580e8 100644
--- a/code/plugin_SequencesLoader/ColorSpaceLoader.h
+++ b/code/SequencesLoader/ColorSpaceLoader.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,7 +14,7 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
@@ -22,29 +22,35 @@
#ifndef _ColorSpaceLoader
#define _ColorSpaceLoader
+#include "LoaderInterface.h"
+#include "ArrayOfReads.h"
+#include "ColorSpaceDecoder.h"
+#include "Read.h"
+
+#include <RayPlatform/memory/MyAllocator.h>
+
+#include <stdio.h>
+#include <fstream>
#include <string>
#include <vector>
-#include <plugin_SequencesLoader/ArrayOfReads.h>
-#include <fstream>
-#include <memory/MyAllocator.h>
-#include <stdio.h>
-#include <plugin_SequencesLoader/ColorSpaceDecoder.h>
-#include <plugin_SequencesLoader/Read.h>
using namespace std;
/**
*
* \author Sébastien Boisvert
*/
-class ColorSpaceLoader{
+class ColorSpaceLoader : public LoaderInterface{
ColorSpaceDecoder m_decoder;
FILE*m_f;
int m_size;
int m_loaded;
public:
+
+ ColorSpaceLoader();
void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator);
int open(string file);
int getSize();
+ void close();
};
diff --git a/code/SequencesLoader/ExportLoader.cpp b/code/SequencesLoader/ExportLoader.cpp
new file mode 100644
index 0000000..201b334
--- /dev/null
+++ b/code/SequencesLoader/ExportLoader.cpp
@@ -0,0 +1,108 @@
+/*
+ Ray
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#include "ExportLoader.h"
+
+#include <code/Mock/constants.h>
+
+#include <fstream>
+#include <stdlib.h>
+using namespace std;
+
+ExportLoader::ExportLoader() {
+ addExtension("export.txt");
+ addExtension("qseq.txt");
+}
+
+int ExportLoader::open(string file){
+ m_f=fopen(file.c_str(),"r");
+ m_size=0;
+ m_loaded=0;
+ char buffer[RAY_MAXIMUM_READ_LENGTH];
+
+ while(NULL!=fgets(buffer,RAY_MAXIMUM_READ_LENGTH,m_f)){
+ m_size++;
+ }
+
+ fclose(m_f);
+ m_f=fopen(file.c_str(),"r");
+ return EXIT_SUCCESS;
+}
+
+void ExportLoader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator){
+ char buffer[RAY_MAXIMUM_READ_LENGTH];
+ int loadedSequences=0;
+
+ while(loadedSequences<maxToLoad && NULL!=fgets(buffer,RAY_MAXIMUM_READ_LENGTH,m_f)){
+
+// find the 9th column
+// index 8 if 0-based
+// 1 -> 1 \t
+// 2 -> 2 \t
+// 8 -> 8 \t
+//
+// find the 8th \t
+// then find the 9th \t
+
+ int theLength=strlen(buffer);
+ int positionFor8thTabulation=0;
+ int positionFor9thTabulation=0;
+ int tabulations=0;
+ int position=0;
+
+ while(position<theLength){
+ if(buffer[position]=='\t')
+ tabulations++;
+
+ if(tabulations==8 && positionFor8thTabulation==0)
+ positionFor8thTabulation=position;
+ if(tabulations==9 && positionFor9thTabulation==0)
+ positionFor9thTabulation=position;
+
+ position++;
+ }
+
+ buffer[positionFor9thTabulation]='\0';
+ char*dnaSequence=buffer+positionFor8thTabulation+1;
+
+#ifdef DEBUG_EXPORT_FORMAT
+ cout<<" ExportLoader -> <sequence>"<<dnaSequence<<"</sequence>"<<endl;
+#endif
+
+ Read t;
+ t.constructor(dnaSequence,seqMyAllocator,true);
+ reads->push_back(&t);
+
+ loadedSequences++;
+ m_loaded++;
+ }
+
+ if(m_loaded==m_size){
+ fclose(m_f);
+ }
+}
+
+int ExportLoader::getSize(){
+ return m_size;
+}
+
+void ExportLoader::close(){
+}
diff --git a/code/plugin_SequencesLoader/FastqLoader.h b/code/SequencesLoader/ExportLoader.h
similarity index 55%
rename from code/plugin_SequencesLoader/FastqLoader.h
rename to code/SequencesLoader/ExportLoader.h
index 6b370b1..e84e2d1 100644
--- a/code/plugin_SequencesLoader/FastqLoader.h
+++ b/code/SequencesLoader/ExportLoader.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,35 +14,46 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#ifndef _FastqLoader
-#define _FastqLoader
-
-#include<stdio.h>
-#include<string>
-#include<memory/MyAllocator.h>
-#include<fstream>
-#include<plugin_SequencesLoader/ArrayOfReads.h>
-#include<vector>
-#include<sstream>
-#include<plugin_SequencesLoader/Read.h>
+#ifndef _ExportLoader
+#define _ExportLoader
+
+#include "LoaderInterface.h"
+#include "ArrayOfReads.h"
+#include "Read.h"
+
+#include <RayPlatform/memory/MyAllocator.h>
+
+#include <fstream>
+#include <vector>
+#include <sstream>
+#include <stdio.h>
+#include <string>
using namespace std;
/**
+ *
+ * A loader for export files.
+ *
+ * This gene was created by a gene duplication of FastqLoader.h.
+ *
* \author Sébastien Boisvert
*/
-class FastqLoader{
+class ExportLoader: public LoaderInterface{
+
int m_loaded;
int m_size;
FILE*m_f;
public:
- int open(string file,int period);
+ ExportLoader();
+ int open(string file);
int getSize();
- void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator,int period);
+ void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator);
+ void close();
};
#endif
diff --git a/code/SequencesLoader/FastaBz2Loader.cpp b/code/SequencesLoader/FastaBz2Loader.cpp
new file mode 100644
index 0000000..a7e460b
--- /dev/null
+++ b/code/SequencesLoader/FastaBz2Loader.cpp
@@ -0,0 +1,51 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#ifdef CONFIG_HAVE_LIBBZ2
+
+#include "FastaBz2Loader.h"
+
+#include <stdlib.h>
+#include <fstream>
+using namespace std;
+
+FastaBz2Loader::FastaBz2Loader() {
+ addExtension(".fa.bz2");
+ addExtension(".fasta.bz2");
+}
+
+int FastaBz2Loader::getSize(){
+ return m_fastqBz2Loader.getSize();
+}
+
+int FastaBz2Loader::open(string file){
+ return m_fastqBz2Loader.openWithPeriod(file,2);
+}
+
+void FastaBz2Loader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator){
+ m_fastqBz2Loader.loadWithPeriod(maxToLoad,reads,seqMyAllocator,2);
+}
+
+void FastaBz2Loader::close(){
+ m_fastqBz2Loader.close();
+}
+
+#endif
diff --git a/code/plugin_SequencesLoader/FastqGzLoader.h b/code/SequencesLoader/FastaBz2Loader.h
similarity index 60%
rename from code/plugin_SequencesLoader/FastqGzLoader.h
rename to code/SequencesLoader/FastaBz2Loader.h
index 46dd8c3..66fe80b 100644
--- a/code/plugin_SequencesLoader/FastqGzLoader.h
+++ b/code/SequencesLoader/FastaBz2Loader.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,35 +14,40 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#ifndef _FastqGzLoader
-#define _FastqGzLoader
-#ifdef HAVE_LIBZ
+#ifndef _FastaBz2Loader
+#define _FastaBz2Loader
+
+#ifdef CONFIG_HAVE_LIBBZ2
+
+#include "FastqBz2Loader.h"
+#include "LoaderInterface.h"
+#include "ArrayOfReads.h"
+#include "Read.h"
+
+#include <RayPlatform/memory/MyAllocator.h>
#include <string>
#include <vector>
-#include <plugin_SequencesLoader/Read.h>
-#include <plugin_SequencesLoader/ArrayOfReads.h>
-#include <zlib.h>
-#include <memory/MyAllocator.h>
using namespace std;
/**
* \author Sébastien Boisvert
*/
-class FastqGzLoader{
- gzFile m_f;
- int m_size;
- int m_loaded;
+class FastaBz2Loader: public LoaderInterface{
+ FastqBz2Loader m_fastqBz2Loader;
public:
- int open(string file,int period);
+ FastaBz2Loader();
+ int open(string file);
int getSize();
- void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator,int period);
+ void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator);
+ void close();
};
#endif
#endif
+
diff --git a/code/SequencesLoader/FastaGzLoader.cpp b/code/SequencesLoader/FastaGzLoader.cpp
new file mode 100644
index 0000000..7cb8d56
--- /dev/null
+++ b/code/SequencesLoader/FastaGzLoader.cpp
@@ -0,0 +1,53 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#ifdef CONFIG_HAVE_LIBZ
+
+#include "FastaGzLoader.h"
+
+#include <fstream>
+#include <stdlib.h>
+using namespace std;
+
+FastaGzLoader::FastaGzLoader() {
+ addExtension("fasta.gz");
+ addExtension("fa.gz");
+}
+
+int FastaGzLoader::open(string file){
+ return m_fastqGzLoader.openWithPeriod(file,2);
+}
+
+void FastaGzLoader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator){
+ m_fastqGzLoader.loadWithPeriod(maxToLoad,reads,seqMyAllocator,2);
+}
+
+int FastaGzLoader::getSize(){
+ return m_fastqGzLoader.getSize();
+}
+
+void FastaGzLoader::close(){
+ m_fastqGzLoader.close();
+}
+
+#endif
+
diff --git a/code/plugin_SequencesLoader/ColorSpaceLoader.h b/code/SequencesLoader/FastaGzLoader.h
similarity index 59%
copy from code/plugin_SequencesLoader/ColorSpaceLoader.h
copy to code/SequencesLoader/FastaGzLoader.h
index 97fb696..920e423 100644
--- a/code/plugin_SequencesLoader/ColorSpaceLoader.h
+++ b/code/SequencesLoader/FastaGzLoader.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,38 +14,42 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#ifndef _ColorSpaceLoader
-#define _ColorSpaceLoader
+#ifndef _FastaGzLoader
+#define _FastaGzLoader
+
+#ifdef CONFIG_HAVE_LIBZ
+
+#include "FastqGzLoader.h"
+#include "LoaderInterface.h"
+#include "Read.h"
+#include "ArrayOfReads.h"
+
+#include <RayPlatform/memory/MyAllocator.h>
#include <string>
#include <vector>
-#include <plugin_SequencesLoader/ArrayOfReads.h>
-#include <fstream>
-#include <memory/MyAllocator.h>
-#include <stdio.h>
-#include <plugin_SequencesLoader/ColorSpaceDecoder.h>
-#include <plugin_SequencesLoader/Read.h>
using namespace std;
+
/**
- *
+ * This class is responsible for reading .fasta.gz files.
* \author Sébastien Boisvert
*/
-class ColorSpaceLoader{
- ColorSpaceDecoder m_decoder;
- FILE*m_f;
- int m_size;
- int m_loaded;
+class FastaGzLoader: public LoaderInterface{
+
+ FastqGzLoader m_fastqGzLoader;
public:
- void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator);
+ FastaGzLoader();
int open(string file);
int getSize();
+ void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator);
+ void close();
};
-
+#endif
#endif
diff --git a/code/plugin_SequencesLoader/FastaLoader.cpp b/code/SequencesLoader/FastaLoader.cpp
similarity index 89%
rename from code/plugin_SequencesLoader/FastaLoader.cpp
rename to code/SequencesLoader/FastaLoader.cpp
index 22cbd83..dc1726f 100644
--- a/code/plugin_SequencesLoader/FastaLoader.cpp
+++ b/code/SequencesLoader/FastaLoader.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,14 +14,16 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include<plugin_SequencesLoader/FastaLoader.h>
-#include<fstream>
+#include "FastaLoader.h"
+
+#include <fstream>
#include <stdlib.h>
+using namespace std;
/** load sequences */
int FastaLoader::load(string file,ArrayOfReads*reads,MyAllocator*seqMyAllocator){
diff --git a/code/plugin_SequencesLoader/FastaLoader.h b/code/SequencesLoader/FastaLoader.h
similarity index 69%
rename from code/plugin_SequencesLoader/FastaLoader.h
rename to code/SequencesLoader/FastaLoader.h
index 1297151..36433eb 100644
--- a/code/plugin_SequencesLoader/FastaLoader.h
+++ b/code/SequencesLoader/FastaLoader.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,7 +14,7 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
@@ -22,13 +22,16 @@
#ifndef _FastaLoader
#define _FastaLoader
-#include<plugin_SequencesLoader/ArrayOfReads.h>
-#include<stdio.h>
-#include<string>
-#include<memory/MyAllocator.h>
-#include<vector>
-#include<sstream>
-#include<plugin_SequencesLoader/Read.h>
+#include "LoaderInterface.h"
+#include "ArrayOfReads.h"
+#include "Read.h"
+
+#include <RayPlatform/memory/MyAllocator.h>
+
+#include <vector>
+#include <sstream>
+#include <stdio.h>
+#include <string>
using namespace std;
/**
diff --git a/code/SequencesLoader/FastaLoaderForReads.cpp b/code/SequencesLoader/FastaLoaderForReads.cpp
new file mode 100644
index 0000000..72396d7
--- /dev/null
+++ b/code/SequencesLoader/FastaLoaderForReads.cpp
@@ -0,0 +1,49 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#include "FastaLoaderForReads.h"
+
+#include <code/Mock/constants.h>
+
+#include <fstream>
+#include <stdlib.h>
+using namespace std;
+
+FastaLoaderForReads::FastaLoaderForReads() {
+ addExtension(".fasta");
+ addExtension(".fa");
+}
+
+int FastaLoaderForReads::open(string file){
+ return m_fastqLoader.openWithPeriod(file,2);
+}
+
+void FastaLoaderForReads::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator){
+ m_fastqLoader.loadWithPeriod(maxToLoad,reads,seqMyAllocator,2);
+}
+
+int FastaLoaderForReads::getSize(){
+ return m_fastqLoader.getSize();
+}
+
+void FastaLoaderForReads::close(){
+ m_fastqLoader.close();
+}
diff --git a/code/plugin_SequencesLoader/ColorSpaceLoader.h b/code/SequencesLoader/FastaLoaderForReads.h
similarity index 63%
copy from code/plugin_SequencesLoader/ColorSpaceLoader.h
copy to code/SequencesLoader/FastaLoaderForReads.h
index 97fb696..85d4150 100644
--- a/code/plugin_SequencesLoader/ColorSpaceLoader.h
+++ b/code/SequencesLoader/FastaLoaderForReads.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,38 +14,41 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#ifndef _ColorSpaceLoader
-#define _ColorSpaceLoader
+#ifndef _FastaLoaderForReads
+#define _FastaLoaderForReads
+
+#include "FastqLoader.h"
+#include "LoaderInterface.h"
+#include "ArrayOfReads.h"
+#include "Read.h"
+
+#include <RayPlatform/memory/MyAllocator.h>
-#include <string>
-#include <vector>
-#include <plugin_SequencesLoader/ArrayOfReads.h>
#include <fstream>
-#include <memory/MyAllocator.h>
+#include <vector>
+#include <sstream>
#include <stdio.h>
-#include <plugin_SequencesLoader/ColorSpaceDecoder.h>
-#include <plugin_SequencesLoader/Read.h>
+#include <string>
using namespace std;
/**
- *
* \author Sébastien Boisvert
*/
-class ColorSpaceLoader{
- ColorSpaceDecoder m_decoder;
- FILE*m_f;
- int m_size;
- int m_loaded;
+class FastaLoaderForReads: public LoaderInterface{
+
+ FastqLoader m_fastqLoader;
public:
- void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator);
+ FastaLoaderForReads();
int open(string file);
int getSize();
+ void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator);
+ void close();
};
-
#endif
+
diff --git a/code/plugin_SequencesLoader/FastqBz2Loader.cpp b/code/SequencesLoader/FastqBz2Loader.cpp
similarity index 62%
rename from code/plugin_SequencesLoader/FastqBz2Loader.cpp
rename to code/SequencesLoader/FastqBz2Loader.cpp
index 958b06d..0715b3d 100644
--- a/code/plugin_SequencesLoader/FastqBz2Loader.cpp
+++ b/code/SequencesLoader/FastqBz2Loader.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,30 +14,41 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#ifdef HAVE_LIBBZ2
+#ifdef CONFIG_HAVE_LIBBZ2
+
+#include "FastqBz2Loader.h"
+#include "BzReader.h"
-#include<plugin_SequencesLoader/FastqBz2Loader.h>
#include <stdlib.h>
-#include<fstream>
-#include<plugin_SequencesLoader/BzReader.h>
+#include <fstream>
+using namespace std;
+
+FastqBz2Loader::FastqBz2Loader() {
+ addExtension(".fq.bz2");
+ addExtension(".fastq.bz2");
+}
int FastqBz2Loader::getSize(){
return m_size;
}
-int FastqBz2Loader::open(string file,int period){
+int FastqBz2Loader::open(string file){
+ return openWithPeriod(file,4);
+}
+
+int FastqBz2Loader::openWithPeriod(string file,int period){
m_reader.open(file.c_str());
- char buffer[4096];
+ char buffer[RAY_MAXIMUM_READ_LENGTH];
m_loaded=0;
m_size=0;
int rotatingVariable=0;
- while(NULL!=m_reader.readLine(buffer,4096)){
+ while(NULL!=m_reader.readLine(buffer,RAY_MAXIMUM_READ_LENGTH)){
if(rotatingVariable==1){
m_size++;
}
@@ -52,12 +63,16 @@ int FastqBz2Loader::open(string file,int period){
return EXIT_SUCCESS;
}
+void FastqBz2Loader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator){
+ loadWithPeriod(maxToLoad,reads,seqMyAllocator,4);
+}
+
// a very simple and compact fastq.gz reader
-void FastqBz2Loader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator,int period){
- char buffer[4096];
+void FastqBz2Loader::loadWithPeriod(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator,int period){
+ char buffer[RAY_MAXIMUM_READ_LENGTH];
int rotatingVariable=0;
int loadedSequences=0;
- while(loadedSequences<maxToLoad&&NULL!=m_reader.readLine(buffer,4096)){
+ while(loadedSequences<maxToLoad&&NULL!=m_reader.readLine(buffer,RAY_MAXIMUM_READ_LENGTH)){
if(rotatingVariable==1){
Read t;
t.constructor(buffer,seqMyAllocator,true);
@@ -78,4 +93,7 @@ void FastqBz2Loader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllo
}
}
+void FastqBz2Loader::close(){
+}
+
#endif
diff --git a/code/plugin_SequencesLoader/FastqBz2Loader.h b/code/SequencesLoader/FastqBz2Loader.h
similarity index 58%
rename from code/plugin_SequencesLoader/FastqBz2Loader.h
rename to code/SequencesLoader/FastqBz2Loader.h
index 46ef362..8d3b71e 100644
--- a/code/plugin_SequencesLoader/FastqBz2Loader.h
+++ b/code/SequencesLoader/FastqBz2Loader.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,34 +14,42 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
#ifndef _FastqBz2Loader
#define _FastqBz2Loader
-#ifdef HAVE_LIBBZ2
-
-#include<string>
-#include<vector>
-#include<plugin_SequencesLoader/ArrayOfReads.h>
-#include<plugin_SequencesLoader/Read.h>
-#include<plugin_SequencesLoader/BzReader.h>
-#include<memory/MyAllocator.h>
+
+#ifdef CONFIG_HAVE_LIBBZ2
+
+#include "LoaderInterface.h"
+#include "ArrayOfReads.h"
+#include "Read.h"
+#include "BzReader.h"
+
+#include <RayPlatform/memory/MyAllocator.h>
+
+#include <string>
+#include <vector>
using namespace std;
/**
* \author Sébastien Boisvert
*/
-class FastqBz2Loader{
+class FastqBz2Loader: public LoaderInterface{
int m_loaded;
int m_size;
BzReader m_reader;
public:
- int open(string file,int period);
+ FastqBz2Loader();
+ int openWithPeriod(string file,int period);
+ int open(string file);
int getSize();
- void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator,int period);
+ void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator);
+ void loadWithPeriod(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator,int period);
+ void close();
};
#endif
diff --git a/code/SequencesLoader/FastqGzLoader.cpp b/code/SequencesLoader/FastqGzLoader.cpp
new file mode 100644
index 0000000..793b363
--- /dev/null
+++ b/code/SequencesLoader/FastqGzLoader.cpp
@@ -0,0 +1,305 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#ifdef CONFIG_HAVE_LIBZ
+
+#include "FastqGzLoader.h"
+
+#define CONFIG_ZLIB_USE_READAHEAD
+
+#define CONFIG_ZLIB_MAXIMUM_READ_LENGTH 4096
+#define CONFIG_ZLIB_READAHEAD_SIZE SIZE_4M
+
+#include <fstream>
+#include <zlib.h>
+#include <stdlib.h>
+using namespace std;
+
+FastqGzLoader::FastqGzLoader() {
+ addExtension(".fastq.gz");
+ addExtension(".fq.gz");
+}
+
+int FastqGzLoader::open(string file){
+ return openWithPeriod(file,4);
+}
+
+int FastqGzLoader::openWithPeriod(string file,int period){
+
+ m_debug=false;
+
+ m_completed=false;
+
+#ifdef CONFIG_ZLIB_USE_READAHEAD
+ m_bufferedBytes=0;
+ m_readaheadBuffer=NULL;
+ m_noMoreBytes=false;
+#endif
+
+ m_f=gzopen(file.c_str(),"r");
+ char buffer[CONFIG_ZLIB_MAXIMUM_READ_LENGTH];
+ m_size=0;
+ m_loaded=0;
+
+ int rotatingVariable=0;
+ while(readOneSingleLine(buffer,CONFIG_ZLIB_MAXIMUM_READ_LENGTH)){
+ if(rotatingVariable==1){
+ m_size++;
+ }
+ rotatingVariable++;
+ if(rotatingVariable==period){
+ rotatingVariable=0;
+ }
+ }
+ gzclose(m_f);
+ m_f=gzopen(file.c_str(),"r");
+
+#ifdef CONFIG_ZLIB_USE_READAHEAD
+ m_bufferedBytes=0;
+ m_readaheadBuffer=NULL;
+ m_noMoreBytes=false;
+ m_completed=false;
+#endif
+
+ return EXIT_SUCCESS;
+}
+
+void FastqGzLoader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator){
+ loadWithPeriod(maxToLoad,reads,seqMyAllocator,4);
+}
+
+// a very simple and compact fastq.gz reader
+void FastqGzLoader::loadWithPeriod(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator,int period){
+ char buffer[CONFIG_ZLIB_MAXIMUM_READ_LENGTH];
+ int rotatingVariable=0;
+ int loadedSequences=0;
+
+ while(loadedSequences<maxToLoad && readOneSingleLine(buffer,CONFIG_ZLIB_MAXIMUM_READ_LENGTH)){
+ if(rotatingVariable==1){
+ Read t;
+ t.constructor(buffer,seqMyAllocator,true);
+ reads->push_back(&t);
+ }
+ rotatingVariable++;
+
+ // a period is reached for each read.
+ if(rotatingVariable==period){
+ rotatingVariable=0;
+ loadedSequences++;
+ m_loaded++;
+ }
+ }
+ if(m_loaded==m_size){
+ gzclose(m_f);
+ }
+}
+
+int FastqGzLoader::getSize(){
+ return m_size;
+}
+
+/**
+ * The initial implementation was using gzgets.
+ * But tests on the IBM Blue Gene/Q at SciNet in Toronto
+ * indicated that gzgets is not a good idea in the end.
+ *
+ * The new implementation does its own read-ahead code.
+ */
+bool FastqGzLoader::readOneSingleLine(char*buffer,int maximumLength){
+
+#ifdef CONFIG_ZLIB_USE_READAHEAD
+
+ return pullLineWithReadaheadTechnology(buffer,maximumLength);
+
+#else
+ char*returnedValue=gzgets(m_f,buffer,maximumLength);
+
+ return returnedValue!=Z_NULL;
+#endif
+
+}
+
+
+bool FastqGzLoader::pullLineWithReadaheadTechnology(char*buffer,int maximumLength){
+
+ if(m_readaheadBuffer==NULL && !m_completed){
+
+ if(m_debug)
+ cout<<"Starting system."<<endl;
+
+ m_readaheadBuffer=(char*)malloc(2*CONFIG_ZLIB_READAHEAD_SIZE);
+ m_bufferedBytes=0;
+ m_currentStart=0;
+ m_firstNewLine=0;
+
+ #ifdef ASSERT
+ assert(m_readaheadBuffer!=NULL);
+ #endif
+ }
+
+ #ifdef ASSERT
+ assert(m_readaheadBuffer!=NULL);
+ #endif
+
+ int availableBytes=2*CONFIG_ZLIB_READAHEAD_SIZE-m_bufferedBytes+m_currentStart;
+
+/* check for available bytes */
+ if(!m_noMoreBytes && availableBytes>=CONFIG_ZLIB_READAHEAD_SIZE){
+
+/* make sure there is space available */
+
+ if(m_debug)
+ cout<<"Moving data around"<<endl;
+
+ int destination=0;
+
+/* move data on the left to make room */
+ for(int i=m_currentStart;i<m_bufferedBytes;i++){
+ m_readaheadBuffer[destination++]=m_readaheadBuffer[i];
+ }
+
+ m_bufferedBytes-=m_currentStart;
+ m_firstNewLine-=m_currentStart;
+ m_currentStart-=m_currentStart;
+
+ if(m_debug)
+ cout<<"Reading data from disk"<<endl;
+
+ #ifdef ASSERT
+ assert(m_bufferedBytes<=2*CONFIG_ZLIB_READAHEAD_SIZE);
+
+ if(m_bufferedBytes>CONFIG_ZLIB_READAHEAD_SIZE){
+ cout<<"Error: there is too much data: m_bufferedBytes: "<<m_bufferedBytes<<" CONFIG_ZLIB_READAHEAD_SIZE:";
+ cout<<CONFIG_ZLIB_READAHEAD_SIZE<<endl;
+ }
+
+ assert(m_bufferedBytes<=CONFIG_ZLIB_READAHEAD_SIZE);
+
+ assert(m_f!=NULL);
+ assert(m_readaheadBuffer!=NULL);
+
+ #endif
+
+ int bytes=gzread(m_f,m_readaheadBuffer+m_bufferedBytes,CONFIG_ZLIB_READAHEAD_SIZE);
+ m_bufferedBytes+=bytes;
+
+ if(bytes==0)
+ m_noMoreBytes=true;
+
+ #ifdef ASSERT
+ assert(m_bufferedBytes<=2*CONFIG_ZLIB_READAHEAD_SIZE);
+ #endif
+ }
+
+
+ if(m_debug)
+ cout<<"Searching for new line at m_firstNewLine "<<m_firstNewLine<<endl;
+
+/* find the next new line */
+ while(m_firstNewLine<m_bufferedBytes && m_readaheadBuffer[m_firstNewLine]!='\n')
+ m_firstNewLine++;
+
+ if(m_firstNewLine==m_bufferedBytes)
+ m_firstNewLine--;
+
+ if(m_debug)
+ cout<<"seek offset if at m_firstNewLine "<<m_firstNewLine<<endl;
+
+ #ifdef ASSERT
+ assert(m_firstNewLine<m_bufferedBytes);
+ assert(m_readaheadBuffer[m_firstNewLine]=='\n' || m_firstNewLine==m_bufferedBytes-1);
+ #endif
+
+ #ifdef ASSERT
+ assert(m_bufferedBytes<=2*CONFIG_ZLIB_READAHEAD_SIZE);
+ #endif
+
+ if(m_debug){
+ cout<<"Copying data m_currentStart="<<m_currentStart<<" m_firstNewLine="<<m_firstNewLine;
+ cout<<" m_bufferedBytes: "<<m_bufferedBytes<<endl;
+ }
+
+/* copy the line */
+ int bytesToCopy=m_firstNewLine-m_currentStart+1;
+
+/* nothing more to read */
+ if(m_bufferedBytes==0 || bytesToCopy==0){
+
+ if(m_debug){
+ cout<<" m_bufferedBytes: "<<m_bufferedBytes<<" m_firstNewLine= ";
+ cout<<m_firstNewLine<<" m_currentStart: "<<m_currentStart;
+ cout<<endl;
+ }
+
+ free(m_readaheadBuffer);
+ m_readaheadBuffer=NULL;
+ m_bufferedBytes=0;
+ m_completed=true;
+
+ return false;
+ }
+
+
+ char*source=m_readaheadBuffer+m_currentStart;
+
+ #ifdef ASSERT
+ if(bytesToCopy==0){
+ cout<<"Error: nothing to copy, m_bufferedBytes= "<<m_bufferedBytes;
+ cout<<" m_firstNewLine "<<m_firstNewLine<<" m_currentStart "<<m_currentStart;
+ cout<<endl;
+ }
+
+ assert(bytesToCopy>0);
+ assert(m_currentStart<m_bufferedBytes);
+
+ int last=m_currentStart+bytesToCopy-1;
+
+ if(last>=m_bufferedBytes){
+ cout<<"Error: start="<<m_currentStart<<" bytes="<<bytesToCopy<<" last= "<<last<<" buffered: "<<m_bufferedBytes<<endl;
+ }
+
+ assert(m_currentStart+bytesToCopy-1<m_bufferedBytes);
+ #endif /* ASSERT */
+
+ memcpy(buffer,source,bytesToCopy);
+ buffer[bytesToCopy]='\0';
+
+ if(m_debug)
+ cout<<"Line @"<<m_currentStart<<"= "<< buffer<<endl;
+
+/* advance the current start and find the new new line */
+ if(m_debug)
+ cout<<"Start + "<<bytesToCopy<<endl;
+
+ m_currentStart+=bytesToCopy;
+ m_firstNewLine++; // the next call will find the new line position
+
+ if(m_debug)
+ cout<<"Finding the next new line in "<<m_bufferedBytes<<" bytes starting at "<<m_firstNewLine<<""<<endl;
+
+ return true;
+}
+
+void FastqGzLoader::close(){
+}
+
+#endif
+
diff --git a/code/SequencesLoader/FastqGzLoader.h b/code/SequencesLoader/FastqGzLoader.h
new file mode 100644
index 0000000..cd1d08d
--- /dev/null
+++ b/code/SequencesLoader/FastqGzLoader.h
@@ -0,0 +1,75 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#ifndef _FastqGzLoader
+#define _FastqGzLoader
+
+#ifdef CONFIG_HAVE_LIBZ
+
+#include "LoaderInterface.h"
+#include "Read.h"
+#include "ArrayOfReads.h"
+
+#include <RayPlatform/memory/MyAllocator.h>
+
+#include <zlib.h>
+#include <string>
+#include <vector>
+using namespace std;
+
+
+/**
+ * This class is responsible for reading .fastq.gz files.
+ * \author Sébastien Boisvert
+ */
+class FastqGzLoader: public LoaderInterface{
+
+ bool m_completed;
+ bool m_noMoreBytes;
+ bool m_debug;
+
+/* stuff for readahead */
+
+ int m_bufferedBytes;
+ int m_currentStart;
+ int m_firstNewLine;
+ char*m_readaheadBuffer;
+
+ gzFile m_f;
+ int m_size;
+ int m_loaded;
+
+ bool readOneSingleLine(char*buffer,int maximumLength);
+ bool pullLineWithReadaheadTechnology(char*buffer,int maximumLength);
+
+public:
+ FastqGzLoader();
+ int openWithPeriod(string file,int period);
+ void loadWithPeriod(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator,int period);
+
+ int open(string file);
+ int getSize();
+ void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator);
+ void close();
+};
+
+#endif
+#endif
diff --git a/code/SequencesLoader/FastqLoader.cpp b/code/SequencesLoader/FastqLoader.cpp
new file mode 100644
index 0000000..1f26db8
--- /dev/null
+++ b/code/SequencesLoader/FastqLoader.cpp
@@ -0,0 +1,134 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#include "FastqLoader.h"
+
+#include <code/Mock/constants.h>
+
+#include <fstream>
+#include <stdlib.h>
+using namespace std;
+
+FastqLoader::FastqLoader() {
+ addExtension(".fastq");
+ addExtension(".fq");
+
+ m_f = NULL;
+}
+
+int FastqLoader::open(string file){
+ return openWithPeriod(file,4);
+}
+
+int FastqLoader::openWithPeriod(string file,int period){
+ m_f=fopen(file.c_str(),"r");
+
+ //cout << "[DEBUG] counting entries" << endl;
+
+ m_lineReader.initialize();
+
+ m_size=0;
+ m_loaded=0;
+ int rotatingVariable=0;
+ char buffer[RAY_MAXIMUM_READ_LENGTH];
+
+ while(NULL!= m_lineReader.readLine(buffer,RAY_MAXIMUM_READ_LENGTH,m_f)){
+
+ /*
+ if(m_size > 7000000)
+ //cout << "[DEBUG] buffer= " << buffer << endl;
+ */
+
+ if(rotatingVariable==1){
+ m_size++;
+ }
+ rotatingVariable++;
+ if(rotatingVariable==period){
+ rotatingVariable=0;
+ }
+
+ /*
+ if(m_size % 1000 == 0)
+ //cout << "[DEBUG] m_size= " << m_size << endl;
+ */
+ }
+
+ //cout << "[DEBUG] m_size= " << m_size << endl;
+
+ fclose(m_f);
+ m_f=fopen(file.c_str(),"r");
+
+ m_lineReader.reset();
+
+ return EXIT_SUCCESS;
+}
+
+void FastqLoader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator){
+
+ //cout << "[DEBUG] loading fastq file maxToLoad= " << maxToLoad << endl;
+
+ loadWithPeriod(maxToLoad,reads,seqMyAllocator,4);
+}
+
+void FastqLoader::loadWithPeriod(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator,int period){
+ char buffer[RAY_MAXIMUM_READ_LENGTH];
+ int rotatingVariable=0;
+ int loadedSequences=0;
+
+ //cout << "[DEBUG] loadWithPeriod m_loaded= " << m_loaded << " m_size= " << m_size << endl;
+
+ while(loadedSequences<maxToLoad && NULL != m_lineReader.readLine(buffer,RAY_MAXIMUM_READ_LENGTH,m_f)){
+
+ // pick up the sequence data
+ if(rotatingVariable == 1){
+ Read t;
+ t.constructor(buffer,seqMyAllocator,true);
+ reads->push_back(&t);
+ }
+ rotatingVariable++;
+
+ // a period is reached for each read.
+ if(rotatingVariable==period){
+ rotatingVariable=0;
+ loadedSequences++;
+ m_loaded++;
+ }
+ }
+
+ if(m_loaded==m_size){
+ close();
+ }
+}
+
+int FastqLoader::getSize(){
+ return m_size;
+}
+
+void FastqLoader::close(){
+ if(m_f != NULL) {
+ fclose(m_f);
+ m_f = NULL;
+ }
+
+ m_lineReader.destroy();
+}
+
+
diff --git a/code/plugin_SequencesLoader/ColorSpaceLoader.h b/code/SequencesLoader/FastqLoader.h
similarity index 59%
rename from code/plugin_SequencesLoader/ColorSpaceLoader.h
rename to code/SequencesLoader/FastqLoader.h
index 97fb696..4e2734c 100644
--- a/code/plugin_SequencesLoader/ColorSpaceLoader.h
+++ b/code/SequencesLoader/FastqLoader.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,38 +14,48 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#ifndef _ColorSpaceLoader
-#define _ColorSpaceLoader
+#ifndef _FastqLoader
+#define _FastqLoader
+
+#include "LoaderInterface.h"
+#include "ArrayOfReads.h"
+#include "Read.h"
+#include "BufferedReader.h"
+
+#include <RayPlatform/memory/MyAllocator.h>
-#include <string>
-#include <vector>
-#include <plugin_SequencesLoader/ArrayOfReads.h>
#include <fstream>
-#include <memory/MyAllocator.h>
+#include <vector>
+#include <sstream>
#include <stdio.h>
-#include <plugin_SequencesLoader/ColorSpaceDecoder.h>
-#include <plugin_SequencesLoader/Read.h>
+#include <string>
using namespace std;
/**
- *
* \author Sébastien Boisvert
*/
-class ColorSpaceLoader{
- ColorSpaceDecoder m_decoder;
- FILE*m_f;
- int m_size;
+class FastqLoader: public LoaderInterface{
+
+ BufferedReader m_lineReader;
int m_loaded;
+ int m_size;
+ FILE*m_f;
+
public:
- void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator);
+ FastqLoader();
+ void loadWithPeriod(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator,int period);
+ int openWithPeriod(string file,int period);
+
int open(string file);
int getSize();
+ void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator);
+ void close();
};
-
#endif
+
diff --git a/code/SequencesLoader/Loader.cpp b/code/SequencesLoader/Loader.cpp
new file mode 100644
index 0000000..2aff6e6
--- /dev/null
+++ b/code/SequencesLoader/Loader.cpp
@@ -0,0 +1,154 @@
+
+/*
+ Ray
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#include "Loader.h"
+#include "Read.h"
+
+#include <sstream>
+#include <iostream>
+#include <assert.h>
+#include <string>
+#include <vector>
+#include <stdlib.h>
+using namespace std;
+
+void Loader::constructor(const char*prefix,bool show,Rank rank){
+
+ m_rank=rank;
+
+ m_maxToLoad=500000;
+ m_currentOffset=0;
+ //m_type=FORMAT_NULL;
+ ostringstream prefixFull;
+ prefixFull<<prefix<<"_Loader";
+ m_show=show;
+ m_allocator.constructor(4194304,"RAY_MALLOC_TYPE_LOADER_ALLOCATOR",m_show);
+ m_reads.constructor(&m_allocator);
+}
+
+int Loader::load(string file,bool isGenome){
+ ifstream f(file.c_str());
+ bool exists=f;
+ f.close();
+ if(!exists){
+ cout<<"Ray: cannot access '"<<file<<"': No such file or directory"<<endl;
+ return EXIT_FAILURE;// ERROR
+ }
+
+ if(file.length()<4){
+ (cout)<<"Error: "<<file<<endl;
+ return EXIT_FAILURE;
+ }
+
+#if 0
+ if(isGenome){
+ m_fasta.load(file,&m_reads,&m_allocator);
+ m_size=m_reads.size();
+ return EXIT_SUCCESS;
+ }
+#endif
+
+ cout<<"Rank "<<m_rank<<" is fetching file "<<file<<" with lazy loading (please wait...)"<<endl;
+
+ m_interface=m_factory.makeLoader(file);
+
+ if(m_interface!=NULL){
+ m_interface->open(file);
+ m_size=m_interface->getSize();
+ return EXIT_SUCCESS;
+ }
+
+ cout<<"Error: "<<file<<": unknown extension, exiting. (see Ray --help for valid extensions)"<<endl;
+
+ return EXIT_FAILURE;
+}
+
+Read*Loader::at(LargeIndex i){
+ #ifdef ASSERT
+ assert(i<m_size);
+ #endif
+
+ if(i>=m_currentOffset+m_reads.size()){
+ loadSequences();
+ }
+
+ #ifdef ASSERT
+ if(i>=m_currentOffset+m_reads.size()){
+ cout<<"i= "<<i<<" m_currentOffset= " << m_currentOffset<<" m_reads.size: " << m_reads.size()<<endl;
+ }
+ assert(i<m_currentOffset+m_reads.size());
+ #endif
+
+ return m_reads.at(i-m_currentOffset);
+}
+
+LargeCount Loader::size(){
+ return m_size;
+}
+
+void Loader::clear(){
+ m_reads.clear();
+ m_allocator.clear();
+ m_size=0;
+ m_currentOffset=0;
+ //m_type=FORMAT_NULL;
+
+ #ifdef ASSERT
+ assert(m_reads.size()==0);
+ #endif
+}
+
+void Loader::loadSequences(){
+ //cout << "[DEBUG] loadSequences." << endl;
+
+ m_currentOffset += m_reads.size();
+ m_allocator.reset();
+ m_reads.reset();
+
+ if(m_interface==NULL) {
+
+ //cout << "[DEBUG] no interface found" << endl;
+ return;
+ }
+
+ m_interface->load(m_maxToLoad,&m_reads,&m_allocator);
+
+ //cout << "[DEBUG] loaded " << m_reads.size() << " entries into memory" << endl;
+}
+
+void Loader::reset(){
+ m_allocator.reset();
+ m_reads.reset();
+ m_size=0;
+ m_currentOffset=0;
+
+ if(m_interface!=NULL){
+ m_interface->close();
+ m_interface=NULL;
+ }
+
+ //m_type=FORMAT_NULL;
+
+ #ifdef ASSERT
+ assert(m_reads.size()==0);
+ #endif
+}
diff --git a/code/plugin_SequencesLoader/Loader.h b/code/SequencesLoader/Loader.h
similarity index 55%
rename from code/plugin_SequencesLoader/Loader.h
rename to code/SequencesLoader/Loader.h
index d79bb96..01c6616 100644
--- a/code/plugin_SequencesLoader/Loader.h
+++ b/code/SequencesLoader/Loader.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010 Sébastien Boisvert
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,7 +14,7 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
@@ -22,49 +22,33 @@
#ifndef _Loader
#define _Loader
-#include <application_core/common_functions.h>
-#include <vector>
-#include <memory/MyAllocator.h>
-#include <plugin_SequencesLoader/Read.h>
-#include <fstream>
-#include <plugin_SequencesLoader/ArrayOfReads.h>
-#include <string>
+#include "LoaderInterface.h"
+#include "LoaderFactory.h"
+#include "Read.h"
+#include "ArrayOfReads.h"
-#ifdef HAVE_LIBZ
-#include<plugin_SequencesLoader/FastqGzLoader.h>
-#endif
+#include <code/Mock/common_functions.h>
-#ifdef HAVE_LIBBZ2
-#include<plugin_SequencesLoader/FastqBz2Loader.h>
-#endif
+#include <RayPlatform/memory/MyAllocator.h>
-#include<plugin_SequencesLoader/FastaLoader.h>
-#include<plugin_SequencesLoader/FastqLoader.h>
-#include<plugin_SequencesLoader/ColorSpaceLoader.h>
-#include<plugin_SequencesLoader/SffLoader.h>
+#include <string>
+#include <fstream>
+#include <vector>
using namespace std;
-enum{
-FORMAT_NULL,
-FORMAT_CSFASTA,
-FORMAT_SFF,
-FORMAT_FASTA,
-FORMAT_FASTQ,
-FORMAT_FASTA_GZ,
-FORMAT_FASTQ_GZ,
-FORMAT_FASTA_BZ2,
-FORMAT_FASTQ_BZ2
-};
-
/*
* Loader loads data files. Data can be formated as SFF, FASTA, and FASTQ.
* Ray makes no use of quality values, so Their encoding is irrelevant.
* \author Sébastien Boisvert
*/
class Loader{
+
+ LoaderFactory m_factory;
+ LoaderInterface*m_interface;
+ Rank m_rank;
+
bool m_show;
- int m_type;
int DISTRIBUTION_ALLOCATOR_CHUNK_SIZE;
ArrayOfReads m_reads;
MyAllocator m_allocator;
@@ -73,23 +57,11 @@ class Loader{
int m_maxToLoad;
LargeCount m_size;
- SffLoader m_sff;
- ColorSpaceLoader m_color;
- FastqLoader m_fastq;
- FastaLoader m_fasta;
-
- #ifdef HAVE_LIBZ
- FastqGzLoader m_fastqgz;
- #endif
-
- #ifdef HAVE_LIBBZ2
- FastqBz2Loader m_fastqbz2;
- #endif
void loadSequences();
public:
- void constructor(const char*prefix,bool show);
+ void constructor(const char*prefix,bool show,Rank rank);
int load(string file,bool isGenome);
LargeCount size();
Read*at(LargeIndex i);
diff --git a/code/SequencesLoader/LoaderFactory.cpp b/code/SequencesLoader/LoaderFactory.cpp
new file mode 100644
index 0000000..4bb7baa
--- /dev/null
+++ b/code/SequencesLoader/LoaderFactory.cpp
@@ -0,0 +1,50 @@
+/*
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+*/
+
+#include "LoaderFactory.h"
+
+LoaderFactory::LoaderFactory() {
+
+ m_loaders.push_back(&m_color);
+ m_loaders.push_back(&m_sff);
+ m_loaders.push_back(&m_fastaLoader);
+ m_loaders.push_back(&m_export);
+ m_loaders.push_back(&m_fastq);
+
+ #ifdef CONFIG_HAVE_LIBZ
+ m_loaders.push_back(&m_fastqgz);
+ m_loaders.push_back(&m_fastagz);
+ #endif
+
+ #ifdef CONFIG_HAVE_LIBBZ2
+ m_loaders.push_back(&m_fastqbz2);
+ m_loaders.push_back(&m_fastabz2);
+ #endif
+}
+
+LoaderInterface* LoaderFactory::makeLoader(string fileName) {
+ vector<LoaderInterface*>::iterator iterator;
+ for(iterator = m_loaders.begin(); iterator != m_loaders.end(); ++iterator) {
+ if((*iterator)->checkFileType(fileName.c_str())) {
+ return *iterator;
+ }
+ }
+ return NULL;
+}
diff --git a/code/SequencesLoader/LoaderFactory.h b/code/SequencesLoader/LoaderFactory.h
new file mode 100644
index 0000000..6e7e298
--- /dev/null
+++ b/code/SequencesLoader/LoaderFactory.h
@@ -0,0 +1,77 @@
+/*
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#ifndef _LoaderFactory_h
+#define _LoaderFactory_h
+
+#include "FastaLoader.h"
+#include "FastaLoaderForReads.h"
+#include "ExportLoader.h"
+#include "FastqLoader.h"
+#include "ColorSpaceLoader.h"
+#include "SffLoader.h"
+
+#ifdef CONFIG_HAVE_LIBZ
+#include "FastqGzLoader.h"
+#include "FastaGzLoader.h"
+#endif
+
+#ifdef CONFIG_HAVE_LIBBZ2
+#include "FastqBz2Loader.h"
+#include "FastaBz2Loader.h"
+#endif
+
+
+/**
+ *
+ * This is a factory that produces products.
+ * The product is a singleton at the moment.
+ *
+ * \author Sébastien Boisvert
+ */
+class LoaderFactory{
+
+ SffLoader m_sff;
+ ColorSpaceLoader m_color;
+ FastqLoader m_fastq;
+ FastaLoader m_fasta;
+ FastaLoaderForReads m_fastaLoader;
+ ExportLoader m_export;
+
+ vector<LoaderInterface*> m_loaders;
+ vector<LoaderInterface*>::iterator m_it;
+
+ #ifdef CONFIG_HAVE_LIBZ
+ FastqGzLoader m_fastqgz;
+ FastaGzLoader m_fastagz;
+ #endif
+
+ #ifdef CONFIG_HAVE_LIBBZ2
+ FastqBz2Loader m_fastqbz2;
+ FastaBz2Loader m_fastabz2;
+ #endif
+
+public:
+ LoaderFactory();
+ LoaderInterface* makeLoader(string file);
+};
+
+#endif
diff --git a/code/SequencesLoader/LoaderInterface.cpp b/code/SequencesLoader/LoaderInterface.cpp
new file mode 100644
index 0000000..51eb94e
--- /dev/null
+++ b/code/SequencesLoader/LoaderInterface.cpp
@@ -0,0 +1,54 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+
+ http://DeNovoAssembler.SourceForge.Net/
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You have received a copy of the GNU General Public License
+ along with this program (gpl-3.0.txt).
+ see <http://www.gnu.org/licenses/>
+
+*/
+
+#include "LoaderInterface.h"
+
+bool LoaderInterface::hasSuffix(const char* fileName,const char*suffix) {
+ int fileNameLength=strlen(fileName);
+ int suffixLength=strlen(suffix);
+
+ if(suffixLength>fileNameLength)
+ return false;
+
+ int delta=0;
+
+ while(delta<suffixLength) {
+ if(suffix[suffixLength-1-delta] !=
+ fileName[fileNameLength-1-delta])
+ return false;
+ delta++;
+ }
+ return true;
+}
+
+void LoaderInterface::addExtension(const char* extension) {
+ m_extensions.push_back(extension);
+}
+
+bool LoaderInterface::checkFileType(const char* fileName) {
+ vector<string>::iterator iterator;
+ for(iterator = m_extensions.begin(); iterator != m_extensions.end(); ++iterator) {
+ if(hasSuffix(fileName, iterator->c_str())) {
+ return true;
+ }
+ }
+ return false;
+}
diff --git a/code/plugin_PhylogenyViewer/PhylogeneticTreeLoader.h b/code/SequencesLoader/LoaderInterface.h
similarity index 50%
copy from code/plugin_PhylogenyViewer/PhylogeneticTreeLoader.h
copy to code/SequencesLoader/LoaderInterface.h
index 3b9d27e..6917f92 100644
--- a/code/plugin_PhylogenyViewer/PhylogeneticTreeLoader.h
+++ b/code/SequencesLoader/LoaderInterface.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,39 +14,38 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
-*/
-#ifndef _PhylogeneticTreeLoader_h
-#define _PhylogeneticTreeLoader_h
+*/
+#ifndef _LoaderInterface_h
+#define _LoaderInterface_h
-#include <application_core/constants.h>
-#include <core/types.h>
-#include <plugin_PhylogenyViewer/types.h>
+#include "Read.h"
+#include "ArrayOfReads.h"
#include <string>
-#include <stdint.h>
-#include <fstream>
using namespace std;
/**
- * A loader for tree files
+ * This is a interface for implementing new file formats.
*
* \author Sébastien Boisvert
*/
-class PhylogeneticTreeLoader{
-
- ifstream m_stream;
-
- LargeCount m_size;
- LargeIndex m_current;
+class LoaderInterface {
+private:
+ vector<string> m_extensions;
+ bool hasSuffix(const char* fileName,const char* suffix);
public:
- void load(string file);
- bool hasNext();
- void getNext(TaxonIdentifier*parent,TaxonIdentifier*child);
+ virtual int open(string file) = 0;
+ virtual int getSize() = 0;
+ virtual void load(int maxToLoad,ArrayOfReads*reads,
+ MyAllocator*seqMyAllocator) = 0;
+ virtual void close() = 0;
+ bool checkFileType(const char* fileName);
+ void addExtension(const char* fileName);
};
#endif
diff --git a/code/SequencesLoader/Makefile b/code/SequencesLoader/Makefile
new file mode 100644
index 0000000..974deb2
--- /dev/null
+++ b/code/SequencesLoader/Makefile
@@ -0,0 +1,25 @@
+SequencesLoader-y += code/SequencesLoader/SequencesLoader.o
+SequencesLoader-y += code/SequencesLoader/Read.o
+SequencesLoader-y += code/SequencesLoader/ArrayOfReads.o
+SequencesLoader-y += code/SequencesLoader/ColorSpaceDecoder.o
+SequencesLoader-y += code/SequencesLoader/ColorSpaceLoader.o
+SequencesLoader-y += code/SequencesLoader/FastaLoader.o
+SequencesLoader-y += code/SequencesLoader/FastaLoaderForReads.o
+SequencesLoader-y += code/SequencesLoader/FastqLoader.o
+SequencesLoader-y += code/SequencesLoader/ExportLoader.o
+SequencesLoader-y += code/SequencesLoader/LoaderInterface.o
+SequencesLoader-y += code/SequencesLoader/LoaderFactory.o
+SequencesLoader-y += code/SequencesLoader/SffLoader.o
+SequencesLoader-y += code/SequencesLoader/Loader.o
+SequencesLoader-y += code/SequencesLoader/BufferedReader.o
+SequencesLoader-y += code/SequencesLoader/ReadHandle.o
+
+SequencesLoader-$(CONFIG_HAVE_LIBBZ2) += code/SequencesLoader/BzReader.o
+SequencesLoader-$(CONFIG_HAVE_LIBBZ2) += code/SequencesLoader/FastqBz2Loader.o
+SequencesLoader-$(CONFIG_HAVE_LIBBZ2) += code/SequencesLoader/FastaBz2Loader.o
+SequencesLoader-$(CONFIG_HAVE_LIBZ) += code/SequencesLoader/FastqGzLoader.o
+SequencesLoader-$(CONFIG_HAVE_LIBZ) += code/SequencesLoader/FastaGzLoader.o
+
+SequencesLoader-y += code/SequencesLoader/SequenceFileDetector.o
+
+obj-y += $(SequencesLoader-y)
diff --git a/code/plugin_SequencesLoader/Read.cpp b/code/SequencesLoader/Read.cpp
similarity index 85%
rename from code/plugin_SequencesLoader/Read.cpp
rename to code/SequencesLoader/Read.cpp
index 1ea1722..0bc76c6 100644
--- a/code/plugin_SequencesLoader/Read.cpp
+++ b/code/SequencesLoader/Read.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,16 +14,18 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include<assert.h>
-#include<application_core/common_functions.h>
-#include<plugin_SequencesLoader/Read.h>
-#include<cstdlib>
-#include<iostream>
-#include<cstring>
+#include "Read.h"
+
+#include <code/Mock/common_functions.h>
+
+#include <assert.h>
+#include <cstdlib>
+#include <iostream>
+#include <cstring>
using namespace std;
char*Read::trim(char*buffer,const char*sequence){
@@ -31,19 +33,19 @@ char*Read::trim(char*buffer,const char*sequence){
int theLen=strlen(sequence);
strcpy(buffer,sequence);
for(int i=0;i<theLen;i++){
- if(buffer[i]=='a')
- buffer[i]='A';
- else if(buffer[i]=='t')
- buffer[i]='T';
- else if(buffer[i]=='c')
- buffer[i]='C';
- else if(buffer[i]=='g')
- buffer[i]='G';
+ if(buffer[i]==SYMBOL_LOWER_A)
+ buffer[i]=SYMBOL_A;
+ else if(buffer[i]==SYMBOL_LOWER_T)
+ buffer[i]=SYMBOL_T;
+ else if(buffer[i]==SYMBOL_LOWER_C)
+ buffer[i]=SYMBOL_C;
+ else if(buffer[i]==SYMBOL_LOWER_G)
+ buffer[i]=SYMBOL_G;
}
// discard N at the beginning and end of the read.
// find the first symbol that is a A,T,C or G
int first=0;
- while(buffer[first]!='A' && buffer[first]!='T' &&buffer[first]!='C' &&buffer[first]!='G' &&first<theLen){
+ while(buffer[first]!=SYMBOL_A && buffer[first]!=SYMBOL_T &&buffer[first]!=SYMBOL_C &&buffer[first]!=SYMBOL_G &&first<theLen){
first++;
}
char*corrected=buffer+first;
@@ -52,7 +54,7 @@ char*Read::trim(char*buffer,const char*sequence){
int last=0;
int len=strlen(corrected);
for(int i=0;i<len;i++){
- if(corrected[i]=='A' || corrected[i]=='T' || corrected[i]=='C' || corrected[i]=='G'){
+ if(corrected[i]==SYMBOL_A || corrected[i]==SYMBOL_T || corrected[i]==SYMBOL_C || corrected[i]==SYMBOL_G){
last=i;
}
}
@@ -112,13 +114,13 @@ void Read::constructor(const char*sequence,MyAllocator*seqMyAllocator,bool trimF
for(int position=0;position<length;position++){
char nucleotide=sequence[position];
- if(nucleotide!='A'&&nucleotide!='T'&&nucleotide!='C'&&nucleotide!='G'){
+ if(nucleotide!=SYMBOL_A&&nucleotide!=SYMBOL_T&&nucleotide!=SYMBOL_C&&nucleotide!=SYMBOL_G){
#ifdef DEBUG_GCC_4_7_2
cout<<"[DEBUG_GCC_4_7_2] nucleotide "<<nucleotide<<" is not in {A,T,C,G}, position "<<position<<" in "<<sequence<<", length is "<<length<<endl;
#endif
- nucleotide='A';
+ nucleotide=SYMBOL_A;
}
uint8_t code=charToCode(nucleotide);
@@ -254,14 +256,14 @@ int Read::getReverseOffset(){
return m_reverseOffset;
}
-void Read::writeOffsets(ofstream*f){
+void Read::writeOffsets(ostream*f){
int forwardOffset=getForwardOffset();
int reverseOffset=getReverseOffset();
f->write((char*)&forwardOffset,sizeof(int));
f->write((char*)&reverseOffset,sizeof(int));
}
-void Read::readOffsets(ifstream*f){
+void Read::readOffsets(istream*f){
int forwardOffset=0;
int reverseOffset=0;
f->read((char*)&forwardOffset,sizeof(int));
@@ -270,7 +272,7 @@ void Read::readOffsets(ifstream*f){
setReverseOffset(reverseOffset);
}
-void Read::write(ofstream*f){
+void Read::write(ostream*f){
m_pairedRead.write(f);
f->write((char*)&m_type,sizeof(uint8_t));
f->write((char*)&m_length,sizeof(uint16_t));
@@ -278,7 +280,7 @@ void Read::write(ofstream*f){
f->write((char*)m_sequence,getRequiredBytes());
}
-void Read::read(ifstream*f,MyAllocator*seqMyAllocator){
+void Read::read(istream*f,MyAllocator*seqMyAllocator){
m_pairedRead.read(f);
f->read((char*)&m_type,sizeof(uint8_t));
f->read((char*)&m_length,sizeof(uint16_t));
diff --git a/code/plugin_SequencesLoader/Read.h b/code/SequencesLoader/Read.h
similarity index 85%
rename from code/plugin_SequencesLoader/Read.h
rename to code/SequencesLoader/Read.h
index b02a493..95863d5 100644
--- a/code/plugin_SequencesLoader/Read.h
+++ b/code/SequencesLoader/Read.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,7 +14,7 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
@@ -22,12 +22,14 @@
#ifndef _Read
#define _Read
+#include <code/SequencesIndexer/PairedRead.h>
+
+#include <RayPlatform/memory/MyAllocator.h>
+
#include <string>
#include <stdint.h>
#include <vector>
#include <fstream>
-#include <memory/MyAllocator.h>
-#include <plugin_SequencesIndexer/PairedRead.h>
using namespace std;
#define TYPE_SINGLE_END 0
@@ -73,11 +75,11 @@ public:
int getForwardOffset();
int getReverseOffset();
- void writeOffsets(ofstream*f);
- void readOffsets(ifstream*f);
+ void writeOffsets(ostream*f);
+ void readOffsets(istream*f);
- void write(ofstream*f);
- void read(ifstream*f,MyAllocator*seqMyAllocator);
+ void write(ostream*f);
+ void read(istream*f,MyAllocator*seqMyAllocator);
} ATTRIBUTE_PACKED;
#endif
diff --git a/code/SequencesLoader/ReadHandle.cpp b/code/SequencesLoader/ReadHandle.cpp
new file mode 100644
index 0000000..f7eae2a
--- /dev/null
+++ b/code/SequencesLoader/ReadHandle.cpp
@@ -0,0 +1,100 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "ReadHandle.h"
+
+void ReadHandle::operator=(const ReadHandle &b) {
+ m_value = b.m_value;
+}
+
+bool ReadHandle::operator<(const ReadHandle & b)const {
+ return m_value < b.m_value;
+}
+
+bool ReadHandle::operator!=(const ReadHandle & b)const {
+ return m_value != b.m_value;
+}
+
+bool ReadHandle::operator==(const ReadHandle & b)const {
+ return m_value == b.m_value;
+}
+
+void ReadHandle::operator=(const uint64_t & value) {
+ m_value = value;
+}
+
+ReadHandle::ReadHandle(const uint64_t & value) {
+ m_value = value;
+}
+
+const uint64_t & ReadHandle::operator () () const {
+ return m_value;
+}
+
+uint64_t & ReadHandle::operator () () {
+ return m_value;
+}
+
+ReadHandle::ReadHandle() {
+ m_value = 0;
+}
+
+uint64_t ReadHandle::operator / (uint64_t value) {
+ return m_value / value;
+}
+
+ostream & operator <<(ostream & stream, const ReadHandle & handle) {
+ stream << handle.m_value;
+ return stream;
+}
+
+uint64_t ReadHandle::operator * (uint64_t value) {
+ return m_value * value;
+}
+
+uint64_t ReadHandle::operator - (uint64_t value) {
+ return m_value - value;
+}
+
+uint64_t ReadHandle::operator + (uint64_t value) {
+ return m_value + value;
+}
+
+const uint64_t & ReadHandle::getValue() const {
+ return m_value;
+}
+
+bool ReadHandle::operator>(const ReadHandle & b)const {
+ return m_value > b.m_value;
+}
+
+bool ReadHandle::operator<=(const ReadHandle & b)const {
+ return m_value <= b.m_value;
+}
+
+bool ReadHandle::operator>=(const ReadHandle & b)const {
+ return m_value >= b.m_value;
+}
+
+uint64_t & ReadHandle::getValue() {
+ return m_value;
+}
+
+
diff --git a/code/SequencesLoader/ReadHandle.h b/code/SequencesLoader/ReadHandle.h
new file mode 100644
index 0000000..8cfb8c9
--- /dev/null
+++ b/code/SequencesLoader/ReadHandle.h
@@ -0,0 +1,58 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+
+#ifndef ReadHandleHeader
+#define ReadHandleHeader
+
+#include <stdint.h>
+
+#include <ostream>
+using namespace std;
+
+class ReadHandle {
+
+ uint64_t m_value;
+
+public:
+ ReadHandle(const uint64_t & value);
+ ReadHandle();
+ void operator=(const ReadHandle &b);
+ bool operator<(const ReadHandle & b)const;
+ bool operator>(const ReadHandle & b)const;
+ bool operator<=(const ReadHandle & b)const;
+ bool operator>=(const ReadHandle & b)const;
+ bool operator!=(const ReadHandle & b)const;
+ bool operator==(const ReadHandle & b)const;
+ void operator=(const uint64_t & value);
+ const uint64_t & operator () () const;
+ uint64_t & operator () ();
+ uint64_t operator / (uint64_t value);
+ uint64_t operator - (uint64_t value);
+ uint64_t operator * (uint64_t value);
+ uint64_t operator + (uint64_t value);
+
+ friend ostream & operator <<(ostream & stream, const ReadHandle & handle);
+
+ const uint64_t & getValue() const;
+ uint64_t & getValue();
+};
+
+#endif
diff --git a/code/SequencesLoader/SequenceFileDetector.cpp b/code/SequencesLoader/SequenceFileDetector.cpp
new file mode 100644
index 0000000..7172aa0
--- /dev/null
+++ b/code/SequencesLoader/SequenceFileDetector.cpp
@@ -0,0 +1,380 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "SequenceFileDetector.h"
+#include "LoaderFactory.h"
+
+#include <RayPlatform/core/OperatingSystem.h>
+
+#include <vector>
+#include <string>
+#include <set>
+using namespace std;
+
+SequenceFileDetector::SequenceFileDetector() {
+
+ m_directorySeparator = '/';
+
+#ifdef _WIN32
+ m_directorySeparator = '\\';
+#endif
+
+}
+
+// TODO: this code does not detect loops...
+void SequenceFileDetector::gatherAllFiles(string & root, vector<string> & rawFiles) {
+
+#if 0
+ cout << "DEBUG SequenceFileDetector::gatherAllFiles ";
+ cout << root << endl;
+#endif
+
+ // explore the directory
+ if(isDirectory(root)) {
+
+ vector<string> entries;
+ getDirectoryFiles(root, entries);
+
+ for(vector<string>::iterator i = entries.begin() ;
+ i != entries.end() ; ++i) {
+
+ string & entry = *i;
+
+ // skip self and parent
+ if(entry == "." || entry == "..")
+ continue;
+
+ ostringstream path;
+ path << root;
+
+ if(root[root.length()-1] != m_directorySeparator)
+ path << m_directorySeparator;
+
+ path << entry;
+
+ string aPath = path.str();
+
+ gatherAllFiles(aPath, rawFiles);
+ }
+
+ // add the file directly...
+ } else {
+
+ rawFiles.push_back(root);
+ }
+}
+
+string SequenceFileDetector::replaceString(const string & templateString, const string & oldString,
+ const string & newString) {
+
+ int position = templateString.find(oldString);
+
+ if(position < 0)
+ return templateString;
+
+ string mutableString = templateString;
+
+ string result = mutableString.replace(position, oldString.length(), newString);
+
+ return result;
+}
+
+void SequenceFileDetector::detectSequenceFiles(string & directory) {
+
+ // read files
+ vector<string> rawFiles;
+ gatherAllFiles(directory, rawFiles);
+
+ vector<string> files;
+
+ // keep supported files
+ LoaderFactory factory;
+
+ for(int i = 0 ; i < (int) rawFiles.size() ; ++ i) {
+
+ string & file = rawFiles[i];
+ if(factory.makeLoader(file) != NULL)
+ files.push_back(file);
+ }
+
+ // build an index
+
+ map<string, int> fileIndex;
+
+ for(int i = 0 ; i < (int) files.size() ; ++i) {
+
+ string & fileName = files[i];
+
+ fileIndex[fileName] = i;
+ }
+
+ set<int> consumedFiles;
+
+ /**
+ * Options for the matching algorithm
+ */
+ bool enableSmartMatchingMode = true;
+ bool enableBestHitMatchingMode = true;
+ int configurationMaximumHits = 1;
+ int configurationMinimumScore = 0;
+ int configurationMaximumScore = 2;
+
+ // detect pairs
+ for(int i = 0 ; i < (int) files.size() ; ++i){
+
+ if(consumedFiles.count(i) > 0)
+ continue;
+
+ // First, try to match the files using pairing rules.
+ //
+ // _R1_ + _R2_
+ // _R2_ + _R1_
+ // _1.fa + _2.fa
+ // _2.fa + _1.fa
+
+ // first, replace the _R1_ with _R2_
+ string & file1 = files[i];
+ string newFile = replaceString(file1, "_R1_", "_R2_");
+
+ if(enableSmartMatchingMode && fileIndex.count(newFile) > 0
+ && newFile != file1) {
+
+ int index2 = fileIndex[newFile];
+
+#ifdef CONFIG_ASSERT
+ assert(i != index2);
+#endif // CONFIG_ASSERT
+
+ if(consumedFiles.count(index2) == 0) {
+ m_leftFiles.push_back(file1);
+ m_rightFiles.push_back(newFile);
+ consumedFiles.insert(i);
+ consumedFiles.insert(index2);
+
+ continue;
+ }
+ }
+
+ // try to replace the _R2_ with _R1_
+ file1 = files[i];
+ newFile = replaceString(file1, "_R2_", "_R1_");
+
+ if(enableSmartMatchingMode && fileIndex.count(newFile) > 0
+ && newFile != file1) {
+
+ int index2 = fileIndex[newFile];
+
+#ifdef CONFIG_ASSERT
+ assert(i != index2);
+#endif // CONFIG_ASSERT
+
+ if(consumedFiles.count(index2) == 0) {
+ m_leftFiles.push_back(newFile);
+ m_rightFiles.push_back(file1);
+ consumedFiles.insert(i);
+ consumedFiles.insert(index2);
+
+ continue;
+ }
+ }
+
+ // replace the _1.fa with _2.fa
+ file1 = files[i];
+ newFile = replaceString(file1, "_1.f", "_2.f");
+
+ if(enableSmartMatchingMode && fileIndex.count(newFile) > 0
+ && newFile != file1) {
+
+ int index2 = fileIndex[newFile];
+
+#ifdef CONFIG_ASSERT
+ assert(i != index2);
+#endif // CONFIG_ASSERT
+
+ if(consumedFiles.count(index2) == 0) {
+ m_leftFiles.push_back(file1);
+ m_rightFiles.push_back(newFile);
+ consumedFiles.insert(i);
+ consumedFiles.insert(index2);
+
+ continue;
+ }
+ }
+
+ // try to replace the _2.fa with _1.fa
+ file1 = files[i];
+ newFile = replaceString(file1, "_2.f", "_1.f");
+
+ if(enableSmartMatchingMode && fileIndex.count(newFile) > 0
+ && newFile != file1) {
+
+ int index2 = fileIndex[newFile];
+
+#ifdef CONFIG_ASSERT
+ assert(i != index2);
+#endif // CONFIG_ASSERT
+
+ if(consumedFiles.count(index2) == 0) {
+
+ m_leftFiles.push_back(newFile);
+ m_rightFiles.push_back(file1);
+ consumedFiles.insert(i);
+ consumedFiles.insert(index2);
+
+ continue;
+ }
+ }
+
+ int bestMatch = -1;
+ int bestScore = 999;
+ int withBestMatch = 0;
+
+ for(int j = 0 ; j < (int) files.size() ; ++j) {
+
+ if(!enableBestHitMatchingMode)
+ break;
+
+ if(consumedFiles.count(j) > 0)
+ continue;
+
+ if(i == j)
+ continue;
+
+ string & file2 = files[j];
+
+ int position1 = 0;
+ int position2 = 0;
+
+ // find the last m_directorySeparator
+
+ int positionForFile1 = 0;
+ while(positionForFile1 < (int)file1.length()) {
+ if(file1[positionForFile1] == m_directorySeparator)
+ position1 = positionForFile1;
+
+ positionForFile1 ++;
+ }
+
+ int positionForFile2 = 0;
+ while(positionForFile2 < (int)file2.length()) {
+ if(file2[positionForFile2] == m_directorySeparator)
+ position2 = positionForFile2;
+
+ positionForFile2 ++;
+ }
+
+ // the files must be in the same directory
+ if(position1 != position2)
+ continue;
+
+ // compute mismatches before the last separator
+
+ int mismatches = 0;
+
+ int positionBefore1 = 0;
+ int positionBefore2 = 0;
+
+ while(positionBefore1 < position1 && positionBefore2 < position2) {
+
+ if(file1[positionBefore1] != file2[positionBefore2])
+ mismatches++;
+
+ positionBefore1 ++;
+ positionBefore2 ++;
+ }
+
+ // skip because they are not in the same directory
+ if(mismatches != 0)
+ continue;
+#if 0
+ cout << "DEBUG position1 " << position1 << " position2 " << position2 << endl;
+#endif
+ mismatches = 0;
+
+ while(position1 < (int)file1.length() && position2 < (int)file2.length()) {
+
+ if(file1[position1++] != file2[position2++])
+ mismatches++;
+ }
+
+ if(bestMatch < 0) {
+ bestMatch = j;
+ bestScore = mismatches;
+
+ withBestMatch = 1;
+ }
+
+ if(mismatches < bestScore) {
+ bestMatch = j;
+ bestScore = mismatches;
+ withBestMatch = 1;
+ } else if(mismatches == bestScore) {
+
+ withBestMatch ++;
+ }
+ }
+
+ if(bestScore > configurationMinimumScore
+ && bestScore <= configurationMaximumScore
+ && withBestMatch <= configurationMaximumHits) {
+
+ string & file1 = files[i];
+ string & file2 = files[bestMatch];
+
+#if 0
+ cout << "DEBUG matching files " << file1 << " and " << file2 << endl;
+#endif
+
+ m_leftFiles.push_back(file1);
+ m_rightFiles.push_back(file2);
+
+ consumedFiles.insert(i);
+ consumedFiles.insert(bestMatch);
+ }
+
+ }
+
+ // detect single files
+ for(int i = 0 ; i < (int) files.size() ; ++i){
+
+ if(consumedFiles.count(i) > 0)
+ continue;
+
+ string & file = files[i];
+
+ m_singleFiles.push_back(file);
+ consumedFiles.insert(i);
+ }
+}
+
+vector<string> & SequenceFileDetector::getLeftFiles() {
+ return m_leftFiles;
+}
+
+vector<string> & SequenceFileDetector::getRightFiles() {
+
+ return m_rightFiles;
+}
+
+vector<string> & SequenceFileDetector::getSingleFiles() {
+
+ return m_singleFiles;
+}
+
diff --git a/code/SequencesLoader/SequenceFileDetector.h b/code/SequencesLoader/SequenceFileDetector.h
new file mode 100644
index 0000000..a29aba2
--- /dev/null
+++ b/code/SequencesLoader/SequenceFileDetector.h
@@ -0,0 +1,59 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef SequenceFileDetectorHeader
+#define SequenceFileDetectorHeader
+
+#include <RayPlatform/core/OperatingSystem.h>
+
+/**
+ *
+ * This class receives a directory
+ * and returns the valid genomic files.
+ *
+ * \author Sébastien Boisvert
+ */
+class SequenceFileDetector {
+
+ char m_directorySeparator;
+
+ vector<string> m_leftFiles;
+ vector<string> m_rightFiles;
+ vector<string> m_singleFiles;
+
+ void gatherAllFiles(string & root, vector<string> & rawFiles);
+
+ string replaceString(const string & templateString, const string & oldString,
+ const string & newString);
+
+public:
+
+ SequenceFileDetector();
+
+ /**
+ * Detect all supported files in a directory, recursively.
+ */
+ void detectSequenceFiles(string & directory);
+ vector<string> & getLeftFiles();
+ vector<string> & getRightFiles();
+ vector<string> & getSingleFiles();
+};
+
+#endif
diff --git a/code/plugin_SequencesLoader/SequencesLoader.cpp b/code/SequencesLoader/SequencesLoader.cpp
similarity index 70%
rename from code/plugin_SequencesLoader/SequencesLoader.cpp
rename to code/SequencesLoader/SequencesLoader.cpp
index 85aa353..80299bb 100644
--- a/code/plugin_SequencesLoader/SequencesLoader.cpp
+++ b/code/SequencesLoader/SequencesLoader.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,34 +14,36 @@ Ray
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include<string.h>
-#include<plugin_SequencesLoader/SequencesLoader.h>
-#include <core/OperatingSystem.h>
-#include<communication/Message.h>
-#include<plugin_SeedExtender/BubbleData.h>
-#include<assert.h>
-#include<vector>
-#include<application_core/common_functions.h>
-#include<plugin_SequencesLoader/Loader.h>
-#include<plugin_SequencesLoader/Read.h>
-#include<iostream>
+#include "SequencesLoader.h"
+#include "Loader.h"
+#include "Read.h"
+
+#include <code/SeedExtender/BubbleData.h>
+#include <code/Mock/common_functions.h>
+
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/communication/Message.h>
+
+#include <iostream>
+#include <string.h>
+#include <assert.h>
+#include <vector>
__CreatePlugin(SequencesLoader);
- /**/
- /**/
-__CreateSlaveModeAdapter(SequencesLoader,RAY_SLAVE_MODE_LOAD_SEQUENCES); /**/
- /**/
- /**/
+__CreateSlaveModeAdapter(SequencesLoader,RAY_SLAVE_MODE_LOAD_SEQUENCES);
+
+__CreateMessageTagAdapter(SequencesLoader,RAY_MPI_TAG_LOAD_SEQUENCES);
+__CreateMessageTagAdapter(SequencesLoader,RAY_MPI_TAG_SET_FILE_ENTRIES);
using namespace std;
-#define NUMBER_OF_SEQUENCES_PERIOD 10000
+#define NUMBER_OF_SEQUENCES_PERIOD 100000
void SequencesLoader::registerSequence(){
if(m_myReads->size()% NUMBER_OF_SEQUENCES_PERIOD ==0){
@@ -60,7 +62,7 @@ void SequencesLoader::registerSequence(){
Read*theRead=m_loader.at(m_distribution_sequence_id);
char read[RAY_MAXIMUM_READ_LENGTH];
theRead->getSeq(read,m_parameters->getColorSpaceMode(),false);
-
+
//cout<<"DEBUG2 Read="<<m_distribution_sequence_id<<" color="<<m_parameters->getColorSpaceMode()<<" Seq= "<<read<<endl;
Read myRead;
@@ -79,6 +81,10 @@ void SequencesLoader::registerSequence(){
#ifdef ASSERT
assert(leftSequenceGlobalId<rightSequenceGlobalId);
+ assert(leftSequenceGlobalId>=0);
+ assert(leftSequenceGlobalId<m_totalNumberOfSequences);
+ assert(rightSequenceGlobalId>=0);
+ assert(rightSequenceGlobalId<m_totalNumberOfSequences);
#endif
int rightSequenceRank=m_parameters->getRankFromGlobalId(rightSequenceGlobalId);
@@ -93,7 +99,7 @@ void SequencesLoader::registerSequence(){
LargeIndex rightSequenceIdOnRank=m_parameters->getIdFromGlobalId(rightSequenceGlobalId);
int library=m_parameters->getLibrary(m_distribution_file_id);
-
+
(*m_myReads)[leftSequenceIdOnRank]->setLeftType();
(*m_myReads)[leftSequenceIdOnRank]->getPairedRead()->constructor(rightSequenceRank,rightSequenceIdOnRank,library);
}else if(m_LOADER_isRightFile){
@@ -106,6 +112,24 @@ void SequencesLoader::registerSequence(){
LargeIndex rightSequenceIdOnRank=m_myReads->size()-1;
ReadHandle leftSequenceGlobalId=rightSequenceGlobalId-m_loader.size();
+ #ifdef ASSERT
+ assert(leftSequenceGlobalId>=0);
+
+ if(leftSequenceGlobalId>=m_totalNumberOfSequences){
+ cout<<"Error: invalid ReadHandle object, leftSequenceGlobalId: "<<leftSequenceGlobalId;
+ cout<<" m_totalNumberOfSequences: "<<m_totalNumberOfSequences;
+ cout<<" rightSequenceGlobalId: "<<rightSequenceGlobalId<<endl;
+ cout<<" m_distribution_currentSequenceId "<<m_distribution_currentSequenceId;
+ cout<<" m_loader.size: "<<m_loader.size();
+ cout<<" rightSequenceIdOnRank: "<<rightSequenceIdOnRank<<" m_myReads->size: "<<m_myReads->size();
+ cout<<endl;
+ }
+
+ assert(leftSequenceGlobalId<m_totalNumberOfSequences);
+ assert(rightSequenceGlobalId>=0);
+ assert(rightSequenceGlobalId<m_totalNumberOfSequences);
+ #endif
+
Rank leftSequenceRank=m_parameters->getRankFromGlobalId(leftSequenceGlobalId);
#ifdef ASSERT
if(leftSequenceRank>=m_size){
@@ -121,13 +145,19 @@ void SequencesLoader::registerSequence(){
// left sequence in interleaved file
}else if(m_isInterleavedFile && ((m_distribution_sequence_id)%2)==0){
ReadHandle rightSequenceGlobalId=(m_distribution_currentSequenceId)+1;
+
+ #ifdef ASSERT
+ assert(rightSequenceGlobalId>=0);
+ assert(rightSequenceGlobalId<m_totalNumberOfSequences);
+ #endif
+
Rank rightSequenceRank=m_parameters->getRankFromGlobalId(rightSequenceGlobalId);
LargeIndex rightSequenceIdOnRank=m_parameters->getIdFromGlobalId(rightSequenceGlobalId);
LargeIndex leftSequenceIdOnRank=m_myReads->size()-1;
int library=m_parameters->getLibrary(m_distribution_file_id);
-
+
(*m_myReads)[leftSequenceIdOnRank]->setLeftType();
(*m_myReads)[leftSequenceIdOnRank]->getPairedRead()->constructor(rightSequenceRank,rightSequenceIdOnRank,library);
@@ -136,10 +166,18 @@ void SequencesLoader::registerSequence(){
ReadHandle rightSequenceGlobalId=(m_distribution_currentSequenceId);
LargeIndex rightSequenceIdOnRank=m_myReads->size()-1;
ReadHandle leftSequenceGlobalId=rightSequenceGlobalId-1;
+
+ #ifdef ASSERT
+ assert(leftSequenceGlobalId>=0);
+ assert(leftSequenceGlobalId<m_totalNumberOfSequences);
+ assert(rightSequenceGlobalId>=0);
+ assert(rightSequenceGlobalId<m_totalNumberOfSequences);
+ #endif
+
Rank leftSequenceRank=m_parameters->getRankFromGlobalId(leftSequenceGlobalId);
LargeIndex leftSequenceIdOnRank=m_parameters->getIdFromGlobalId(leftSequenceGlobalId);
int library=m_parameters->getLibrary(m_distribution_file_id);
-
+
(*m_myReads)[rightSequenceIdOnRank]->setRightType();
(*m_myReads)[rightSequenceIdOnRank]->getPairedRead()->constructor(leftSequenceRank,leftSequenceIdOnRank,library);
}
@@ -160,7 +198,12 @@ bool SequencesLoader::writeSequencesToAMOSFile(int rank,int size,
}
m_distribution_sequence_id=0;
vector<string> allFiles=(*m_parameters).getAllFiles();
- m_loader.constructor(m_parameters->getMemoryPrefix().c_str(),m_parameters->showMemoryAllocations());
+ m_loader.constructor(m_parameters->getMemoryPrefix().c_str(),m_parameters->showMemoryAllocations(),
+ m_rank);
+
+ char * seq = (char *) __Malloc(RAY_MAXIMUM_READ_LENGTH * sizeof(char), "CodePath/-amos", false);
+ char * qlt = (char *) __Malloc(RAY_MAXIMUM_READ_LENGTH * sizeof(char), "CodePath/-amos", false);
+
for(m_distribution_file_id=0;m_distribution_file_id<(int)allFiles.size();
m_distribution_file_id++){
@@ -170,14 +213,11 @@ bool SequencesLoader::writeSequencesToAMOSFile(int rank,int size,
return false;
}
- fflush(stdout);
// write Reads in AMOS format.
if(rank==MASTER_RANK&&m_parameters->useAmos()){
- char qlt[20000];
for(LargeIndex i=0;i<m_loader.size();i++){
ReadHandle iid=m_distribution_currentSequenceId;
m_distribution_currentSequenceId++;
- char seq[4000];
m_loader.at(i)->getSeq(seq,m_parameters->getColorSpaceMode(),true);
#ifdef ASSERT
assert(seq!=NULL);
@@ -198,6 +238,10 @@ bool SequencesLoader::writeSequencesToAMOSFile(int rank,int size,
}
m_loader.reset();
}
+
+ __Free(qlt, "/CodePath/-amos", false);
+ __Free(seq, "/CodePath/-amos", false);
+
m_loader.clear();
if(m_parameters->useAmos()){
fclose(fp);
@@ -208,12 +252,10 @@ bool SequencesLoader::writeSequencesToAMOSFile(int rank,int size,
bool SequencesLoader::call_RAY_SLAVE_MODE_LOAD_SEQUENCES(){
printf("Rank %i is loading sequence reads\n",m_rank);
- fflush(stdout);
/* check if the checkpoint exists */
if(m_parameters->hasCheckpoint("Sequences")){
cout<<"Rank "<<m_parameters->getRank()<<" is reading checkpoint Sequences"<<endl;
- cout.flush();
ifstream f(m_parameters->getCheckpointFile("Sequences").c_str());
LargeCount count=0;
@@ -225,7 +267,7 @@ bool SequencesLoader::call_RAY_SLAVE_MODE_LOAD_SEQUENCES(){
}
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_SEQUENCES_READY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
cout<<"Rank "<<m_parameters->getRank()<<" loaded "<<count<<" sequences from checkpoint Sequences"<<endl;
@@ -236,20 +278,20 @@ bool SequencesLoader::call_RAY_SLAVE_MODE_LOAD_SEQUENCES(){
// count the number of sequences in all files.
vector<string> allFiles=(*m_parameters).getAllFiles();
-
- LargeCount totalNumberOfSequences=0;
+
+ m_totalNumberOfSequences=0;
for(int i=0;i<(int)m_parameters->getNumberOfFiles();i++){
- totalNumberOfSequences+=m_parameters->getNumberOfSequences(i);
+ m_totalNumberOfSequences+=m_parameters->getNumberOfSequences(i);
}
- LargeCount sequencesPerRank=totalNumberOfSequences/m_size;
+ LargeCount sequencesPerRank=m_totalNumberOfSequences/m_size;
LargeIndex sequencesOnRanksBeforeThisOne=m_rank*sequencesPerRank;
-
+
LargeIndex startingSequenceId=sequencesOnRanksBeforeThisOne;
LargeIndex endingSequenceId=startingSequenceId+sequencesPerRank-1;
if(m_rank==m_size-1){
- endingSequenceId=totalNumberOfSequences-1;
+ endingSequenceId=m_totalNumberOfSequences-1;
}
LargeCount sequences=endingSequenceId-startingSequenceId+1;
@@ -258,7 +300,8 @@ bool SequencesLoader::call_RAY_SLAVE_MODE_LOAD_SEQUENCES(){
cout<<";"<<endingSequenceId<<"], "<<sequences<<" sequence reads"<<endl;
m_distribution_currentSequenceId=0;
- m_loader.constructor(m_parameters->getMemoryPrefix().c_str(),m_parameters->showMemoryAllocations());
+ m_loader.constructor(m_parameters->getMemoryPrefix().c_str(),m_parameters->showMemoryAllocations(),
+ m_rank);
for(m_distribution_file_id=0;m_distribution_file_id<(int)allFiles.size();
m_distribution_file_id++){
@@ -314,10 +357,10 @@ bool SequencesLoader::call_RAY_SLAVE_MODE_LOAD_SEQUENCES(){
m_loader.reset();
}
-
+
m_loader.clear();
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_SEQUENCES_READY,m_rank);
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
LargeCount amount=m_myReads->size();
@@ -327,16 +370,17 @@ bool SequencesLoader::call_RAY_SLAVE_MODE_LOAD_SEQUENCES(){
if(m_parameters->writeCheckpoints() && !m_parameters->hasCheckpoint("Sequences")){
/* announce the user that we are writing a checkpoint */
cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint Sequences"<<endl;
- cout.flush();
ofstream f(m_parameters->getCheckpointFile("Sequences").c_str());
+ ostringstream buffer;
LargeCount count=m_myReads->size();
- f.write((char*)&count,sizeof(LargeCount));
+ buffer.write((char*)&count, sizeof(LargeCount));
for(LargeIndex i=0;i<count;i++){
- m_myReads->at(i)->write(&f);
+ m_myReads->at(i)->write(&buffer);
+ flushFileOperationBuffer(false, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
}
-
+ flushFileOperationBuffer(true, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
f.close();
}
@@ -356,6 +400,39 @@ void SequencesLoader::constructor(int size,MyAllocator*allocator,ArrayOfReads*re
m_outbox=outbox;
}
+void SequencesLoader::call_RAY_MPI_TAG_SET_FILE_ENTRIES(Message*message){
+
+ MessageUnit*incoming=(MessageUnit*)message->getBuffer();
+
+ #ifdef ASSERT
+ assert(message->getCount()>=2);
+ assert(message->getCount()%2==0);
+ #endif
+
+ int input=0;
+
+/*
+ * The source can multiplex the body.
+ */
+ while(input<message->getCount()){
+ int file=incoming[input++];
+ LargeCount count=incoming[input++];
+
+ if(m_parameters->hasOption("-debug-partitioner"))
+ cout<<"Rank "<<m_parameters->getRank()<<" RAY_MPI_TAG_SET_FILE_ENTRIES File "<<file<<" "<<count<<endl;
+
+ m_parameters->setNumberOfSequences(file,count);
+ }
+
+ Message aMessage(NULL,0,message->getSource(),RAY_MPI_TAG_SET_FILE_ENTRIES_REPLY,m_rank);
+ m_outbox->push_back(&aMessage);
+}
+
+void SequencesLoader::call_RAY_MPI_TAG_LOAD_SEQUENCES(Message*message){
+}
+
+
+
void SequencesLoader::registerPlugin(ComputeCore*core){
PluginHandle plugin=core->allocatePluginHandle();
@@ -370,6 +447,19 @@ void SequencesLoader::registerPlugin(ComputeCore*core){
RAY_SLAVE_MODE_LOAD_SEQUENCES=core->allocateSlaveModeHandle(plugin);
core->setSlaveModeObjectHandler(plugin,RAY_SLAVE_MODE_LOAD_SEQUENCES, __GetAdapter(SequencesLoader,RAY_SLAVE_MODE_LOAD_SEQUENCES));
core->setSlaveModeSymbol(plugin,RAY_SLAVE_MODE_LOAD_SEQUENCES,"RAY_SLAVE_MODE_LOAD_SEQUENCES");
+
+ RAY_MPI_TAG_LOAD_SEQUENCES=core->allocateMessageTagHandle(plugin);
+ core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_LOAD_SEQUENCES, __GetAdapter(SequencesLoader,RAY_MPI_TAG_LOAD_SEQUENCES));
+ core->setMessageTagSymbol(plugin,RAY_MPI_TAG_LOAD_SEQUENCES,"RAY_MPI_TAG_LOAD_SEQUENCES");
+
+ RAY_MPI_TAG_SET_FILE_ENTRIES=core->allocateMessageTagHandle(plugin);
+ core->setMessageTagObjectHandler(plugin,RAY_MPI_TAG_SET_FILE_ENTRIES, __GetAdapter(SequencesLoader,RAY_MPI_TAG_SET_FILE_ENTRIES));
+ core->setMessageTagSymbol(plugin,RAY_MPI_TAG_SET_FILE_ENTRIES,"RAY_MPI_TAG_SET_FILE_ENTRIES");
+
+ RAY_MPI_TAG_SET_FILE_ENTRIES_REPLY=core->allocateMessageTagHandle(plugin);
+ core->setMessageTagSymbol(plugin,RAY_MPI_TAG_SET_FILE_ENTRIES_REPLY,"RAY_MPI_TAG_SET_FILE_ENTRIES_REPLY");
+
+ core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_LOAD_SEQUENCES, RAY_SLAVE_MODE_LOAD_SEQUENCES);
}
void SequencesLoader::resolveSymbols(ComputeCore*core){
@@ -378,5 +468,14 @@ void SequencesLoader::resolveSymbols(ComputeCore*core){
RAY_MPI_TAG_SEQUENCES_READY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_SEQUENCES_READY");
+ RAY_MPI_TAG_LOAD_SEQUENCES=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_LOAD_SEQUENCES");
+ RAY_MPI_TAG_SET_FILE_ENTRIES=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_SET_FILE_ENTRIES");
+ RAY_MPI_TAG_SET_FILE_ENTRIES_REPLY=core->getMessageTagFromSymbol(m_plugin,"RAY_MPI_TAG_SET_FILE_ENTRIES_REPLY");
+
__BindPlugin(SequencesLoader);
+
+ __BindAdapter(SequencesLoader,RAY_MPI_TAG_LOAD_SEQUENCES);
+ __BindAdapter(SequencesLoader,RAY_MPI_TAG_SET_FILE_ENTRIES);
+ __BindAdapter(SequencesLoader,RAY_SLAVE_MODE_LOAD_SEQUENCES);
+
}
diff --git a/code/plugin_SequencesLoader/SequencesLoader.h b/code/SequencesLoader/SequencesLoader.h
similarity index 62%
rename from code/plugin_SequencesLoader/SequencesLoader.h
rename to code/SequencesLoader/SequencesLoader.h
index 06c8ac3..1c94b33 100644
--- a/code/plugin_SequencesLoader/SequencesLoader.h
+++ b/code/SequencesLoader/SequencesLoader.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,7 +14,7 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
@@ -22,20 +22,28 @@
#ifndef _SequencesLoader
#define _SequencesLoader
-#include <application_core/Parameters.h>
-#include <memory/RingAllocator.h>
-#include <plugin_SequencesLoader/Loader.h>
-#include <memory/MyAllocator.h>
-#include <structures/StaticVector.h>
-#include <communication/Message.h>
-#include <plugin_SequencesLoader/Read.h>
-#include <plugin_SeedExtender/BubbleData.h>
-#include <core/ComputeCore.h>
+#include "Read.h"
+
+#include <code/Mock/Parameters.h>
+#include <code/SequencesLoader/Loader.h>
+#include <code/SeedExtender/BubbleData.h>
+
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/memory/MyAllocator.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/core/ComputeCore.h>
#include <time.h>
#include <vector>
using namespace std;
+__DeclarePlugin(SequencesLoader);
+
+__DeclareSlaveModeAdapter(SequencesLoader,RAY_SLAVE_MODE_LOAD_SEQUENCES);
+
+__DeclareMessageTagAdapter(SequencesLoader,RAY_MPI_TAG_LOAD_SEQUENCES);
+__DeclareMessageTagAdapter(SequencesLoader,RAY_MPI_TAG_SET_FILE_ENTRIES);
/*
* Computes the partition on reads (MASTER_RANK).
@@ -45,16 +53,23 @@ using namespace std;
*/
class SequencesLoader : public CorePlugin{
+ __AddAdapter(SequencesLoader,RAY_SLAVE_MODE_LOAD_SEQUENCES);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_LOAD_SEQUENCES);
+ __AddAdapter(MessageProcessor,RAY_MPI_TAG_SET_FILE_ENTRIES);
+
MessageTag RAY_MPI_TAG_SEQUENCES_READY;
+ MessageTag RAY_MPI_TAG_SET_FILE_ENTRIES_REPLY;
+ MessageTag RAY_MPI_TAG_LOAD_SEQUENCES;
+ MessageTag RAY_MPI_TAG_SET_FILE_ENTRIES;
SlaveMode RAY_SLAVE_MODE_LOAD_SEQUENCES;
SlaveMode RAY_SLAVE_MODE_DO_NOTHING;
-
MyAllocator*m_persistentAllocator;
ArrayOfReads*m_myReads;
Parameters*m_parameters;
LargeIndex m_distribution_currentSequenceId;
+ LargeCount m_totalNumberOfSequences;
int m_distribution_file_id;
LargeIndex m_distribution_sequence_id;
bool m_LOADER_isLeftFile;
@@ -71,6 +86,8 @@ class SequencesLoader : public CorePlugin{
public:
bool call_RAY_SLAVE_MODE_LOAD_SEQUENCES();
+ void call_RAY_MPI_TAG_LOAD_SEQUENCES(Message*message);
+ void call_RAY_MPI_TAG_SET_FILE_ENTRIES(Message*message);
bool writeSequencesToAMOSFile(int rank,int size,StaticVector*m_outbox,
RingAllocator*m_outboxAllocator,
diff --git a/code/plugin_SequencesLoader/SffLoader.cpp b/code/SequencesLoader/SffLoader.cpp
similarity index 88%
rename from code/plugin_SequencesLoader/SffLoader.cpp
rename to code/SequencesLoader/SffLoader.cpp
index 95d9ffc..8ab333b 100644
--- a/code/plugin_SequencesLoader/SffLoader.cpp
+++ b/code/SequencesLoader/SffLoader.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,24 +14,26 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
-#include <application_core/common_functions.h>
+#include "SffLoader.h"
+
+#include <code/Mock/common_functions.h>
+
#include <cstring>
#include <fstream>
#include <string>
#include <vector>
-#include <plugin_SequencesLoader/SffLoader.h>
#include <iostream>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
using namespace std;
-/** TODO: this code will fail on big-endian systems
+/** TODO: this code will fail on big-endian systems
*
* SFF files are stored as big-endian
*
@@ -42,6 +44,10 @@ using namespace std;
* That is why here byte order is inverted.
* */
+SffLoader::SffLoader() {
+ addExtension(".ssf");
+}
+
int max(int a,int b){
if(a>b)
return a;
@@ -144,10 +150,18 @@ int SffLoader::openSff(string file){
return EXIT_SUCCESS;
}
-// see manual http://sequence.otago.ac.nz/download/GS_FLX_Software_Manual.pdf,
-// page 445-448
-// or
-// http://blog.malde.org/index.php/2008/11/14/454-sequencing-and-parsing-the-sff-binary-format/
+
+/**
+ * SFF specification:
+ *
+ * \see manual http://sequence.otago.ac.nz/download/GS_FLX_Software_Manual.pdf, page 445-448
+ *
+ * \see http://blog.malde.org/index.php/2008/11/14/454-sequencing-and-parsing-the-sff-binary-format/
+ *
+ * \see Section "13.3.8 Standard Flowgram Files (.sff)"
+ * Genome Sequencer, Data Analysis Software Manual, Software Version 2.0.00, October 2008, page 528
+ * http://sequence.otago.ac.nz/download/GS_FLX_Software_Manual.pdf
+ */
void SffLoader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator){
int loadedSequences=0;
@@ -231,3 +245,6 @@ void SffLoader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator
int SffLoader::getSize(){
return m_size;
}
+
+void SffLoader::close(){
+}
diff --git a/code/plugin_SequencesLoader/SffLoader.h b/code/SequencesLoader/SffLoader.h
similarity index 70%
rename from code/plugin_SequencesLoader/SffLoader.h
rename to code/SequencesLoader/SffLoader.h
index 04ff928..e3f9695 100644
--- a/code/plugin_SequencesLoader/SffLoader.h
+++ b/code/SequencesLoader/SffLoader.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010 Sébastien Boisvert
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -14,7 +14,7 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt).
see <http://www.gnu.org/licenses/>
*/
@@ -22,21 +22,24 @@
#ifndef _SffLoader
#define _SffLoader
-#include<string>
-#include<vector>
-#include<plugin_SequencesLoader/Read.h>
-#include<stdio.h>
-#include<fstream>
-#include<plugin_SequencesLoader/ArrayOfReads.h>
-#include<memory/MyAllocator.h>
+#include "LoaderInterface.h"
+#include "Read.h"
+#include "ArrayOfReads.h"
+
+#include <RayPlatform/memory/MyAllocator.h>
+
+#include <string>
+#include <stdio.h>
+#include <fstream>
+#include <vector>
using namespace std;
/**
* This class allows one to use SFF file directly.
- * see http://454.com
+ * \see http://454.com
* \author Sébastien Boisvert
*/
-class SffLoader{
+class SffLoader: public LoaderInterface{
int m_size;
int m_loaded;
FILE*m_fp;
@@ -48,9 +51,11 @@ class SffLoader{
int openSff(string file);
public:
+ SffLoader();
int open(string file);
int getSize();
void load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator);
+ void close();
};
#endif
diff --git a/code/SpuriousSeedAnnihilator/AnnihilationWorker.cpp b/code/SpuriousSeedAnnihilator/AnnihilationWorker.cpp
new file mode 100644
index 0000000..88f2898
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/AnnihilationWorker.cpp
@@ -0,0 +1,614 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "AnnihilationWorker.h"
+
+//#define DEBUG_CODE_PATH
+
+#include <stack>
+using namespace std;
+
+/**
+ *
+ * This is a bubble caused by a polymorphism (a SNP).
+ *
+ * For sequencing error, one of the branches will be weak.
+ *
+ *
+ * 32 31 29 28 27 31 33 34 27
+ *
+ * (x)--->(x)--->(x)--->(x)--->(x)--->(x)--->(x)--->(x)--->(x)
+ * 63 64 61 /
+ * / =====================================================>
+ * (x)---(x)--->(x)
+ * \ =====================================================>
+ * | \
+ * | (x)--->(x)--->(x)--->(x)--->(x)--->(x)--->(x)--->(x)--->(x)
+ * |
+ * | 25 26 27 29 30 31 32 26 29
+ * |
+ * | |
+ * | |
+ * | |
+ * | v
+ * | STEP_FETCH_FIRST_PARENT operation
+ * |
+ * v
+ * STEP_FETCH_SECOND_PARENT operation
+ * |
+ * |
+ * |
+ * |
+ * v
+ * STEP_DOWNLOAD_ORIGINAL_ANNOTATIONS operation
+ *
+ * Algorithm:
+ *
+ * See the initialize method for steps.
+ *
+ * \author Sébastien Boisvert
+ */
+void AnnihilationWorker::work(){
+
+/*
+ * This is only useful for bubbles I think. -Séb
+ */
+ if(m_step == STEP_CHECK_LENGTH){
+
+#ifdef DEBUG_CODE_PATH
+ cout<<"Worker " << m_identifier << " STEP_CHECK_LENGTH"<<endl;
+#endif
+
+ if(m_seed->size() > 3 * m_parameters->getWordSize()){
+ m_done = true;
+
+ }
+
+ m_step++;
+
+ }else if(m_step == STEP_CHECK_DEAD_END_ON_THE_LEFT){
+
+ if(checkDeadEndOnTheLeft())
+ m_step++;
+
+ }else if(m_step == STEP_CHECK_DEAD_END_ON_THE_RIGHT){
+
+ if(checkDeadEndOnTheRight())
+ m_step++;
+
+ }else if(m_step == STEP_CHECK_BUBBLE_PATTERNS){
+
+ if(checkBubblePatterns())
+ m_done = true;
+ }
+}
+
+bool AnnihilationWorker::searchGraphForNiceThings(int direction){
+
+ if(!m_searchIsStarted) {
+
+#ifdef ASSERT
+ assert(direction == DIRECTION_PARENTS || direction == DIRECTION_CHILDREN);
+#endif
+
+ while(!m_depths.empty())
+ m_depths.pop();
+
+ while(!m_vertices.empty())
+ m_vertices.pop();
+
+ int depth=0;
+ m_actualMaximumDepth=0;
+
+ Kmer startingPoint;
+ int index = -1;
+
+ if(direction == DIRECTION_PARENTS)
+ index = 0;
+ else if(direction == DIRECTION_CHILDREN)
+ index = m_seed->size() -1;
+
+#ifdef ASSERT
+ assert(index == 0 || index == m_seed->size()-1);
+#endif
+ m_seed->at(index, &startingPoint);
+
+ m_vertices.push(startingPoint);
+ m_depths.push(depth);
+ m_visited.clear();
+
+ m_attributeFetcher.reset();
+
+ m_searchIsStarted = true;
+
+ }else if(!m_vertices.empty()){
+
+ Kmer kmer = m_vertices.top();
+ int depth = m_depths.top();
+
+#ifdef DEBUG_LEFT_EXPLORATION
+ //cout<<"Stack is not empty"<<endl;
+#endif
+
+ if(depth > m_actualMaximumDepth)
+ m_actualMaximumDepth = depth;
+
+// too deep
+ if(depth == m_maximumAllowedDepth){
+
+#ifdef DEBUG_LEFT_EXPLORATION
+ cout<<"Reached maximum"<<endl;
+#endif
+
+ m_vertices.pop();
+ m_depths.pop();
+
+// working ...
+ }else if(!m_attributeFetcher.fetchObjectMetaData(&kmer)){
+
+ }else if((int)m_attributeFetcher.getDepth() > m_maximumDepthForExploration) {
+
+ return true;
+ }else{
+
+// need to pop the thing now !
+ m_vertices.pop();
+ m_depths.pop();
+
+#ifdef DEBUG_LEFT_EXPLORATION
+ cout<<"fetchObjectMetaData is done... " << m_attributeFetcher.getParents()->size() << " links "<<endl;
+#endif
+
+ m_visited.insert(kmer);
+// explore links
+
+ vector<Kmer> * links = NULL;
+
+ if(direction == DIRECTION_PARENTS)
+ links = m_attributeFetcher.getParents();
+ else if(direction == DIRECTION_CHILDREN)
+ links = m_attributeFetcher.getChildren();
+
+#ifdef ASSERT
+ assert(links != NULL);
+#endif
+
+ for(int i = 0 ; i < (int)links->size() ; i++){
+
+ Kmer parent = links->at(i);
+
+ if(m_visited.count(parent)>0)
+ continue;
+
+ m_vertices.push(parent);
+ m_depths.push( depth + 1 );
+ }
+
+
+// prepare the system for the next wave.
+
+ m_attributeFetcher.reset();
+ }
+
+// the exploration is finished
+// and we did not go far.
+ }else if(m_actualMaximumDepth < m_maximumAllowedDepth){
+
+ m_valid = false;
+
+ return true;
+ }else{
+ return true;
+ }
+
+ return false;
+}
+
+// #define DEBUG_LEFT_EXPLORATION
+
+bool AnnihilationWorker::checkDeadEndOnTheLeft(){
+
+ if(!m_startedToCheckDeadEndOnTheLeft){
+
+#ifdef DEBUG_CODE_PATH
+ cout<<"Worker " << m_identifier << " STEP_CHECK_DEAD_END_ON_THE_LEFT"<<endl;
+#endif
+#ifdef DEBUG_LEFT_EXPLORATION
+ cout<<"Starting checkDeadEndOnTheLeft"<<endl;
+#endif
+
+ m_searchIsStarted = false;
+ m_startedToCheckDeadEndOnTheLeft=true;
+
+ }else if(!searchGraphForNiceThings(DIRECTION_PARENTS)){
+
+ // wait a little bit now
+
+ }else if(!m_valid){
+
+ m_done = true;
+
+ return true;
+
+ }else{
+ return true;
+ }
+
+ return false;
+}
+
+bool AnnihilationWorker::checkDeadEndOnTheRight(){
+
+ if(!m_startedToCheckDeadEndOnTheRight){
+
+#ifdef DEBUG_CODE_PATH
+ cout<<"Worker " << m_identifier << " STEP_CHECK_DEAD_END_ON_THE_RIGHT"<<endl;
+#endif
+#ifdef DEBUG_LEFT_EXPLORATION
+ cout<<"Starting checkDeadEndOnTheRight"<<endl;
+#endif
+
+ m_searchIsStarted = false;
+ m_startedToCheckDeadEndOnTheRight = true;
+
+ }else if(!searchGraphForNiceThings(DIRECTION_CHILDREN)){
+
+ // wait a little bit now
+
+ }else if(!m_valid){
+
+ m_done = true;
+
+ return true;
+ }else{
+
+#ifdef DEBUG_LEFT_EXPLORATION
+ cout << "Next is bubble check"<<endl;
+#endif
+ return true;
+
+ }
+
+ return false;
+}
+
+bool AnnihilationWorker::isDone(){
+
+ return m_done;
+}
+
+WorkerHandle AnnihilationWorker::getWorkerIdentifier(){
+
+ return m_identifier;
+}
+
+// TODO: skip this if the length is too short.
+bool AnnihilationWorker::checkBubblePatterns(){
+
+ if(!m_fetchedFirstParent){
+
+ Kmer startingPoint;
+ int index = 0;
+ m_seed->at(index, &startingPoint);
+
+ if(!m_attributeFetcher.fetchObjectMetaData(&startingPoint)){
+
+ }else{
+
+#ifdef DEBUG_CODE_PATH
+ cout<<"Worker " << m_identifier << " STEP_CHECK_BUBBLE_PATTERNS"<<endl;
+#endif
+ if(m_attributeFetcher.getParents()->size() != 1){
+
+ return true;
+ }else{
+
+ m_parent=m_attributeFetcher.getParents()->at(0);
+ }
+
+ m_fetchedFirstParent = true;
+ m_fetchedSecondParent = false;
+ m_attributeFetcher.reset();
+ }
+
+ }else if(!m_fetchedSecondParent){
+
+ if(!m_attributeFetcher.fetchObjectMetaData(&m_parent)){
+
+ }else{
+ if(m_attributeFetcher.getParents()->size() != 1){
+
+ return true;
+ }else{
+
+ m_grandparent=m_attributeFetcher.getParents()->at(0);
+ }
+
+ m_fetchedSecondParent = true;
+ m_fetchedGrandparentDirections = false;
+ m_annotationFetcher.reset();
+
+#ifdef DEBUG_LEFT_EXPLORATION
+ cout << "Next is to fetch directions " <<endl;
+#endif
+ }
+ }else if(!m_fetchedGrandparentDirections){
+
+ if(!m_annotationFetcher.fetchDirections(&m_grandparent)){
+
+ // work a bit here
+ }else{
+ m_leftDirections = *(m_annotationFetcher.getDirections() );
+
+ m_fetchedGrandparentDirections = true;
+ m_fetchedGrandparentReverseDirections = false;
+ m_annotationFetcher.reset();
+ }
+ }else if(!m_fetchedGrandparentReverseDirections){
+
+ Kmer theKmer;
+ theKmer = m_grandparent.complementVertex(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
+
+ if(!m_annotationFetcher.fetchDirections(&theKmer)){
+
+ // work a bit here
+ }else{
+ for(int i=0;i< (int) m_annotationFetcher.getDirections()->size(); i++){
+ Direction direction;
+ direction.constructor(m_annotationFetcher.getDirections()->at(i).getWave(),
+ m_annotationFetcher.getDirections()->at(i).getProgression(),
+ true);
+
+ m_leftDirections.push_back(direction);
+ }
+
+ m_fetchedGrandparentReverseDirections= true;
+
+ m_fetchedFirstChild = false;
+ m_attributeFetcher.reset();
+ }
+
+ }else if(!m_fetchedFirstChild){
+
+ Kmer startingPoint;
+ int index = m_seed->size()-1;
+ m_seed->at(index, &startingPoint);
+
+ if(!m_attributeFetcher.fetchObjectMetaData(&startingPoint)){
+
+ }else{
+ if(m_attributeFetcher.getChildren()->size() != 1){
+
+ return true;
+ }else{
+
+ m_child = m_attributeFetcher.getChildren()->at(0);
+ }
+
+ m_fetchedFirstChild = true;
+ m_fetchedSecondChild = false;
+ m_attributeFetcher.reset();
+ }
+
+ }else if(!m_fetchedSecondChild){
+
+ if(!m_attributeFetcher.fetchObjectMetaData(&m_child)){
+ }else{
+ if(m_attributeFetcher.getChildren()->size() != 1){
+
+ return true;
+ }else{
+
+ m_grandchild = m_attributeFetcher.getChildren()->at(0);
+ }
+
+ m_fetchedSecondChild= true;
+ m_fetchedGrandchildDirections = false;
+ m_annotationFetcher.reset();
+
+#ifdef DEBUG_LEFT_EXPLORATION
+ cout << "Next is to fetch directions " <<endl;
+#endif
+ }
+ }else if(!m_fetchedGrandchildDirections){
+
+ if(!m_annotationFetcher.fetchDirections(&m_grandchild)){
+
+ }else{
+ m_rightDirections = *(m_annotationFetcher.getDirections() );
+
+ m_fetchedGrandchildDirections= true;
+
+ m_fetchedGrandchildReverseDirections = false;
+ m_annotationFetcher.reset();
+ }
+ }else if(!m_fetchedGrandchildReverseDirections){
+
+ Kmer theKmer;
+ theKmer = m_grandchild.complementVertex(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
+
+ if(!m_annotationFetcher.fetchDirections(&theKmer)){
+
+ // work a bit here
+ }else{
+ for(int i=0;i< (int) m_annotationFetcher.getDirections()->size(); i++){
+ Direction direction;
+ direction.constructor(m_annotationFetcher.getDirections()->at(i).getWave(),
+ m_annotationFetcher.getDirections()->at(i).getProgression(),
+ true);
+
+ m_rightDirections.push_back(direction);
+ }
+
+ m_fetchedGrandchildReverseDirections = true;
+ }
+ }else{
+
+ if(m_isPerfectBubble){
+
+#ifdef DEBUG_CODE_PATH
+ cout<<"BUBBLE_HIT first=";
+#endif
+
+ Kmer startingPoint;
+ int index = 0;
+ m_seed->at(index, &startingPoint);
+ cout << startingPoint.idToWord(m_parameters->getWordSize(), m_parameters->getColorSpaceMode());
+
+#if 0
+ cout << " grandparent= ";
+ cout << m_grandparent.idToWord(m_parameters->getWordSize(), m_parameters->getColorSpaceMode());
+#endif
+
+#ifdef DEBUG_CODE_PATH
+ cout<<" LeftPaths: " << m_leftDirections.size();
+ cout<<" RightPaths: " << m_rightDirections.size() << endl;
+#endif
+ }
+
+ map<PathHandle,int> counts;
+
+ for(int i=0; i < (int) m_leftDirections.size() ; i ++ )
+ counts[m_leftDirections[i].getWave()]++;
+
+ for(int i=0; i < (int) m_rightDirections.size() ; i ++ )
+ counts[m_rightDirections[i].getWave()]++;
+
+ for(map<PathHandle,int>::iterator i = counts.begin() ; i != counts.end() ; i ++){
+
+ // another, longer, seed covers this case.
+ if(i->second == 2){
+
+ m_valid = false;
+ }
+ }
+
+ // this is over.
+ return true;
+ }
+
+ return false;
+}
+
+bool AnnihilationWorker::getBestParent(Kmer*kmer){
+ return true;
+}
+
+bool AnnihilationWorker::getBestChild(Kmer*kmer){
+
+ return true;
+}
+
+bool AnnihilationWorker::getOtherBestChild(Kmer*kmer){
+ return true;
+
+}
+
+bool AnnihilationWorker::getOtherBestParent(Kmer*kmer){
+
+ return true;
+}
+
+/**
+ * RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE
+ * RAY_MPI_TAG_ASK_VERTEX_PATH
+ */
+void AnnihilationWorker::initialize(uint64_t identifier,GraphPath*seed, Parameters * parameters,
+ VirtualCommunicator * virtualCommunicator, RingAllocator*outboxAllocator,
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE, MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH
+ ){
+
+ m_identifier = identifier;
+ m_done = false;
+
+ m_seed = seed;
+
+ m_virtualCommunicator = virtualCommunicator;
+ m_parameters = parameters;
+
+ int stepValue = 0;
+
+ STEP_CHECK_LENGTH = stepValue++;
+ STEP_CHECK_DEAD_END_ON_THE_LEFT = stepValue ++;
+ STEP_CHECK_DEAD_END_ON_THE_RIGHT = stepValue ++;
+ STEP_CHECK_BUBBLE_PATTERNS= stepValue ++;
+ STEP_FETCH_FIRST_PARENT = stepValue ++;
+ STEP_FETCH_SECOND_PARENT = stepValue ++;
+ STEP_DOWNLOAD_ORIGINAL_ANNOTATIONS = stepValue ++;
+ STEP_GET_SEED_SEQUENCE_NOW = stepValue ++;
+
+ m_step = STEP_CHECK_DEAD_END_ON_THE_LEFT;
+
+ this->RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT = RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+
+ this->RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE = RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ this->RAY_MPI_TAG_ASK_VERTEX_PATH = RAY_MPI_TAG_ASK_VERTEX_PATH;
+
+ m_rank = m_parameters->getRank();
+ m_outboxAllocator = outboxAllocator;
+
+ m_startedToCheckDeadEndOnTheLeft = false;
+ m_startedToCheckDeadEndOnTheRight = false;
+
+ m_valid = true;
+
+ DIRECTION_PARENTS = 0;
+ DIRECTION_CHILDREN = 1;
+
+ m_fetchedFirstParent = false;
+ m_fetchedSecondParent = false;
+
+ m_attributeFetcher.initialize(parameters, virtualCommunicator,
+ identifier, outboxAllocator,
+ RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT);
+
+ m_annotationFetcher.initialize(parameters, virtualCommunicator,
+ identifier, outboxAllocator,
+ RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ RAY_MPI_TAG_ASK_VERTEX_PATH);
+
+ int nucleotides = m_seed->size() + m_parameters->getWordSize() -1;
+ int bubbleSize = 2 * m_parameters->getWordSize() - 3;
+
+ m_isPerfectBubble = false;
+
+ if(nucleotides == bubbleSize)
+ m_isPerfectBubble = true;
+
+#ifdef DEBUG_ISSUE_136
+ if(m_isPerfectBubble)
+ cout<<"BUBBLE_ITEM"<<endl;
+#endif
+
+/*
+ * the maximum depth for dead ends.
+ */
+ m_maximumDepthForExploration = 256;
+
+/*
+ * Maximum search depth for dead ends.
+ */
+ m_maximumAllowedDepth = m_parameters->getWordSize();
+}
+
+bool AnnihilationWorker::isValid(){
+ return m_valid;
+}
diff --git a/code/SpuriousSeedAnnihilator/AnnihilationWorker.h b/code/SpuriousSeedAnnihilator/AnnihilationWorker.h
new file mode 100644
index 0000000..962d87b
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/AnnihilationWorker.h
@@ -0,0 +1,141 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef _AnnihilationWorker_h
+#define _AnnihilationWorker_h
+
+#include "AttributeFetcher.h"
+#include "AnnotationFetcher.h"
+
+#include <code/SeedingData/GraphPath.h>
+#include <code/Mock/Parameters.h>
+#include <code/SeedExtender/Direction.h>
+
+#include <RayPlatform/scheduling/Worker.h>
+
+#include <stack>
+#include <vector>
+#include <set>
+using namespace std;
+
+#include <stdint.h>
+
+/**
+ * This is a worker that analyze a seed.
+ *
+ * \author Sébastien Boisvert
+ */
+class AnnihilationWorker: public Worker{
+
+ AttributeFetcher m_attributeFetcher;
+ AnnotationFetcher m_annotationFetcher;
+
+ uint64_t m_identifier; // TODO this should be in Worker because it's always there anyway
+ bool m_done; // TODO this should be in Worker because it's always there anyway
+ GraphPath * m_seed;
+
+ VirtualCommunicator * m_virtualCommunicator; // TODO this should be in Worker because it's always there anyway
+ Parameters * m_parameters;
+
+ Kmer m_parent;
+ Kmer m_grandparent;
+ Kmer m_child;
+ Kmer m_grandchild;
+
+ int m_step;
+
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH;
+
+ int STEP_CHECK_LENGTH;
+ int STEP_CHECK_DEAD_END_ON_THE_LEFT;
+ int STEP_CHECK_DEAD_END_ON_THE_RIGHT;
+ int STEP_CHECK_BUBBLE_PATTERNS;
+ int STEP_FETCH_FIRST_PARENT;
+ int STEP_FETCH_SECOND_PARENT;
+ int STEP_DOWNLOAD_ORIGINAL_ANNOTATIONS;
+ int STEP_GET_SEED_SEQUENCE_NOW;
+
+ bool m_queryWasSent;
+ Rank m_rank;
+ RingAllocator*m_outboxAllocator;
+
+ bool m_startedToCheckDeadEndOnTheLeft;
+ bool m_startedToCheckDeadEndOnTheRight;
+
+
+ stack<int> m_depths;
+ stack<Kmer> m_vertices;
+ int m_maximumAllowedDepth;
+ int m_actualMaximumDepth;
+ int m_maximumDepthForExploration;
+
+ bool m_valid;
+ set<Kmer> m_visited;
+
+ bool m_searchIsStarted;
+
+ int DIRECTION_PARENTS;
+ int DIRECTION_CHILDREN;
+
+ bool m_fetchedFirstParent;
+ bool m_fetchedSecondParent;
+ bool m_fetchedFirstChild;
+ bool m_fetchedSecondChild;
+
+ vector<Direction> m_leftDirections;
+ vector<Direction> m_rightDirections;
+ bool m_fetchedGrandparentDirections;
+ bool m_fetchedGrandchildDirections;
+ bool m_fetchedGrandparentReverseDirections;
+ bool m_fetchedGrandchildReverseDirections;
+ bool m_isPerfectBubble;
+
+// private methods
+
+ bool checkDeadEndOnTheLeft();
+ bool checkDeadEndOnTheRight();
+ bool searchGraphForNiceThings(int direction);
+ bool checkBubblePatterns();
+
+ bool getOtherBestParent(Kmer*kmer);
+ bool getOtherBestChild(Kmer*kmer);
+ bool getBestChild(Kmer*kmer);
+ bool getBestParent(Kmer*kmer);
+
+public:
+ void work();
+
+ bool isDone();
+
+ WorkerHandle getWorkerIdentifier();
+
+ void initialize(uint64_t identifier, GraphPath*seed, Parameters * parameters,
+ VirtualCommunicator * virtualCommunicator,
+ RingAllocator * outboxAllocator,
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE, MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH
+ );
+
+ bool isValid();
+};
+
+#endif
diff --git a/code/SpuriousSeedAnnihilator/AnnotationFetcher.cpp b/code/SpuriousSeedAnnihilator/AnnotationFetcher.cpp
new file mode 100644
index 0000000..2e33696
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/AnnotationFetcher.cpp
@@ -0,0 +1,154 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "AnnotationFetcher.h"
+
+void AnnotationFetcher::initialize(Parameters*parameters, VirtualCommunicator*virtualCommunicator,
+ WorkerHandle identifier, RingAllocator * outboxAllocator,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH
+ ){
+
+ m_virtualCommunicator = virtualCommunicator;
+ m_parameters = parameters;
+ m_identifier = identifier;
+ m_outboxAllocator = outboxAllocator;
+ m_rank = m_parameters->getRank();
+
+ this->RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE = RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ this->RAY_MPI_TAG_ASK_VERTEX_PATH = RAY_MPI_TAG_ASK_VERTEX_PATH;
+
+ reset();
+}
+
+void AnnotationFetcher::reset(){
+ m_initializedDirectionFetcher = false;
+}
+
+bool AnnotationFetcher::fetchDirections(Kmer*kmer){
+
+ if(!m_initializedDirectionFetcher){
+
+ m_initializedDirectionFetcher = true;
+
+ m_fetchedCount = false;
+ m_queryWasSent = false;
+
+ m_directions.clear();
+
+ }else if(!m_fetchedCount){
+
+ if(!m_queryWasSent){
+ MessageTag tag = RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ int elementsPerQuery=m_virtualCommunicator->getElementsPerQuery(tag);
+
+#ifdef DEBUG_LEFT_EXPLORATION
+ cout << "Sending message for count" << endl;
+#endif
+
+ Rank destination = m_parameters->vertexRank(kmer);
+ MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(elementsPerQuery*sizeof(MessageUnit));
+ int bufferPosition=0;
+ kmer->pack(message,&bufferPosition);
+ Message aMessage(message, elementsPerQuery,
+ destination, tag, m_rank);
+
+ m_virtualCommunicator->pushMessage(m_identifier, &aMessage);
+
+ m_queryWasSent = true;
+
+ }else if(m_virtualCommunicator->isMessageProcessed(m_identifier)){
+ vector<MessageUnit> elements;
+ m_virtualCommunicator->getMessageResponseElements(m_identifier, &elements);
+
+ m_numberOfPaths = elements[0];
+
+ m_fetchedCount = true;
+
+#ifdef DEBUG_LEFT_EXPLORATION
+ cout<<"Paths: "<<m_numberOfPaths << endl;
+#endif
+ m_pathIndex = 0;
+
+ m_queryWasSent = false;
+ m_reverseStrand = false;
+ }
+ }else if(m_pathIndex < m_numberOfPaths){
+
+ if(!m_queryWasSent){
+
+ Kmer theKmer=*kmer;
+
+ if(m_reverseStrand)
+ theKmer = theKmer.complementVertex(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
+
+ Rank destination = m_parameters->vertexRank(&theKmer);
+ int elementsPerQuery=m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_ASK_VERTEX_PATH);
+ MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(elementsPerQuery);
+ int outputPosition=0;
+ theKmer.pack(message,&outputPosition);
+ message[outputPosition++]=m_pathIndex;
+
+ Message aMessage(message, elementsPerQuery, destination,
+ RAY_MPI_TAG_ASK_VERTEX_PATH, m_parameters->getRank());
+
+ m_virtualCommunicator->pushMessage(m_identifier, &aMessage);
+
+ m_queryWasSent = true;
+
+ }else if(m_virtualCommunicator->isMessageProcessed(m_identifier)){
+
+ vector<MessageUnit> response;
+ m_virtualCommunicator->getMessageResponseElements(m_identifier, &response);
+
+ int bufferPosition=0;
+ /* skip the k-mer because we don't need it */
+ bufferPosition += kmer->getNumberOfU64();
+ PathHandle otherPathIdentifier=response[bufferPosition++];
+ int progression=response[bufferPosition++];
+
+ Direction direction;
+
+ direction.constructor(otherPathIdentifier, progression, m_reverseStrand);
+
+ m_directions.push_back(direction);
+
+ m_queryWasSent = false;
+ m_pathIndex ++;
+ }
+/*
+ }else if(!m_reverseStrand){
+
+ // fetch also reverse-complement entries.
+ m_reverseStrand = true;
+ m_pathIndex = 0;
+ m_queryWasSent = false;
+*/
+ }else{
+ return true;
+ }
+
+ return false;
+}
+
+vector<Direction>* AnnotationFetcher::getDirections(){
+
+ return &m_directions;
+}
diff --git a/code/SpuriousSeedAnnihilator/AnnotationFetcher.h b/code/SpuriousSeedAnnihilator/AnnotationFetcher.h
new file mode 100644
index 0000000..e11366e
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/AnnotationFetcher.h
@@ -0,0 +1,70 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef _AnnotationFetcher_h
+#define _AnnotationFetcher_h
+
+#include <code/SeedExtender/Direction.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/scheduling/Worker.h>
+
+#include <vector>
+using namespace std;
+
+/**
+ * This is a building block to fetch annotations.
+ * This is compatible with seeds and contigs.
+ *
+ * \author Sébastien Boisvert
+ */
+class AnnotationFetcher{
+
+ Rank m_rank;
+ Parameters*m_parameters;
+ VirtualCommunicator*m_virtualCommunicator;
+ WorkerHandle m_identifier;
+ RingAllocator * m_outboxAllocator;
+ bool m_queryWasSent;
+ bool m_initializedDirectionFetcher;
+ bool m_fetchedCount;
+ int m_pathIndex;
+ int m_numberOfPaths;
+ vector<Direction> m_directions;
+ bool m_reverseStrand;
+
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH;
+public:
+ void initialize(Parameters*parameters, VirtualCommunicator*virtualCommunicator,
+ WorkerHandle identifier, RingAllocator * outboxAllocator,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH);
+
+ bool fetchDirections(Kmer*kmer);
+
+ void reset();
+
+ vector<Direction>* getDirections();
+};
+
+#endif
diff --git a/code/SpuriousSeedAnnihilator/AttributeFetcher.cpp b/code/SpuriousSeedAnnihilator/AttributeFetcher.cpp
new file mode 100644
index 0000000..376a723
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/AttributeFetcher.cpp
@@ -0,0 +1,98 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "AttributeFetcher.h"
+
+bool AttributeFetcher::fetchObjectMetaData(Kmer * object){
+
+ if(!m_initializedFetcher){
+
+ m_queryWasSent=false;
+ m_initializedFetcher=true;
+
+ }else if(!m_queryWasSent){
+
+ MessageTag tag = RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+
+ Rank destination = m_parameters->vertexRank(object);
+ MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(MessageUnit));
+ int bufferPosition=0;
+ object->pack(message,&bufferPosition);
+ Message aMessage(message,m_virtualCommunicator->getElementsPerQuery(tag),
+ destination, tag, m_rank);
+
+ m_virtualCommunicator->pushMessage(m_identifier, &aMessage);
+
+ m_queryWasSent = true;
+
+ }else if(m_virtualCommunicator->isMessageProcessed(m_identifier)){
+
+ vector<MessageUnit> elements;
+ m_virtualCommunicator->getMessageResponseElements(m_identifier, &elements);
+
+ int bufferPosition=0;
+
+ uint8_t edges = elements[bufferPosition++];
+
+ m_depth=elements[bufferPosition++];
+#ifdef ASSERT
+ assert(m_depth>= 1);
+#endif
+
+ m_parents = object->getIngoingEdges(edges, m_parameters->getWordSize());
+ m_children = object->getOutgoingEdges(edges, m_parameters->getWordSize());
+
+ m_queryWasSent = false;
+
+ return true;
+ }
+
+ return false;
+}
+
+void AttributeFetcher::initialize(Parameters*parameters, VirtualCommunicator*virtualCommunicator,
+ WorkerHandle identifier, RingAllocator * outboxAllocator,
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT){
+
+ m_virtualCommunicator = virtualCommunicator;
+ m_parameters = parameters;
+ m_identifier = identifier;
+ m_outboxAllocator = outboxAllocator;
+ m_rank = m_parameters->getRank();
+ this->RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT = RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+
+ reset();
+}
+
+void AttributeFetcher::reset(){
+ m_initializedFetcher = false;
+}
+
+CoverageDepth AttributeFetcher::getDepth(){
+ return m_depth;
+}
+
+vector<Kmer>* AttributeFetcher::getParents(){
+ return &m_parents;
+}
+
+vector<Kmer>* AttributeFetcher::getChildren(){
+ return &m_children;
+}
diff --git a/code/SpuriousSeedAnnihilator/AttributeFetcher.h b/code/SpuriousSeedAnnihilator/AttributeFetcher.h
new file mode 100644
index 0000000..9a9575a
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/AttributeFetcher.h
@@ -0,0 +1,107 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef _AttributeFetcher_h
+#define _AttributeFetcher_h
+
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/scheduling/Worker.h>
+
+#include <vector>
+using namespace std;
+
+/**
+ * This is a building block to fetch the parents, the children
+ * and the coverage of a k-mer.
+ *
+ * Compatible with the API of:
+ *
+ * * VirtualCommunicator
+ * * VirtualProcessor
+ * * Ray message pool definition
+ *
+ * \author Sébastien Boisvert
+ */
+class AttributeFetcher{
+
+ Rank m_rank;
+ Parameters*m_parameters;
+ VirtualCommunicator*m_virtualCommunicator;
+ vector<Kmer> m_parents;
+ vector<Kmer> m_children;
+ CoverageDepth m_depth;
+
+ WorkerHandle m_identifier;
+ RingAllocator * m_outboxAllocator;
+ bool m_initializedFetcher;
+ bool m_queryWasSent;
+
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+public:
+
+ /**
+ * Initializes the object.
+ *
+ * This must be called only once.
+ */
+ void initialize(Parameters*parameters, VirtualCommunicator*virtualCommunicator,
+ WorkerHandle identifier, RingAllocator * outboxAllocator,
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT);
+
+ /**
+ * Fetches the parents, the children and the coverage of a k-mer.
+ *
+ * @returns true ifresult is available, false if additional calls are required
+ * for the request to complete.
+ */
+ bool fetchObjectMetaData(Kmer * object);
+
+ /**
+ * Resets the the object so that fetchObjectMetaData can be called on another object.
+ *
+ */
+ void reset();
+
+ /**
+ * Gets the coverage depth of the submitted object
+ *
+ * fetchObjectMetaData must have returned true before calling this.
+ */
+ CoverageDepth getDepth();
+
+ /**
+ * Gets the parents of a k-mer.
+ *
+ * fetchObjectMetaData must have returned true before calling this.
+ */
+ vector<Kmer>* getParents();
+
+ /**
+ * Gets the children of a k-mer.
+ *
+ * fetchObjectMetaData must have returned true before calling this.
+ */
+ vector<Kmer>* getChildren();
+};
+
+#endif
diff --git a/code/SpuriousSeedAnnihilator/GossipAssetManager.cpp b/code/SpuriousSeedAnnihilator/GossipAssetManager.cpp
new file mode 100644
index 0000000..05618ae
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/GossipAssetManager.cpp
@@ -0,0 +1,433 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+// this code is developped with defensive programming. (assertions)
+// -Sébastien Boisvert
+
+// #define CONFIG_DEBUG_GOSSIP_ASSET_MANAGER "Yes !!!"
+
+#include "GossipAssetManager.h"
+
+#include <code/Mock/common_functions.h>
+
+#include <iostream>
+using namespace std;
+
+void GossipAssetManager::addGossip(GraphSearchResult & gossip) {
+
+ //------------------------------------------------------------//
+ // step add the asset and get a handle for it (an index)
+ m_gossips.push_back(gossip);
+
+ string key = gossip.toString();
+
+#ifdef CONFIG_ASSERT
+ assert(m_gossipIndex.count(key) == 0);
+#endif
+
+ int index = m_gossips.size() - 1;
+ m_gossipIndex[key] = index;
+
+ classifyGossip(gossip);
+}
+
+void GossipAssetManager::classifyGossip(GraphSearchResult & gossip) {
+
+ string key = gossip.toString();
+
+ int index = m_gossipIndex[key];
+
+ bool mergedTwoClusters = false;
+
+ //------------------------------------------------------------//
+ // give a better name to the index for the rest of this method.
+ int & newGossipIndex = index;
+
+ PathHandle & path1 = gossip.getPathHandles()[0];
+ PathHandle & path2 = gossip.getPathHandles()[1];
+
+ //------------------------------------------------------------//
+ // Now, we add the gossip to an existing cluster or to a new one.
+ //
+ // step 3: try to merge the newCluster to existing clusters
+ // step 3.1 -- path1 and path2 in the gossip in newCluster have 0 match
+ if(m_pathToClusterTable.count(path1) == 0 && m_pathToClusterTable.count(path2) == 0) {
+ set<int> newCluster;
+ newCluster.insert(index);
+ m_gossipClusters.push_back(newCluster);
+
+ int clusterIndex = m_gossipClusters.size() - 1;
+
+ m_pathToClusterTable[path1] = clusterIndex;
+ m_pathToClusterTable[path2] = clusterIndex;
+
+ // step 3.2 -- path1 in the gossip newCluster has 1 match, but path2 in in the gossip in newCluster have 0 match
+ } else if(m_pathToClusterTable.count(path1) > 0 && m_pathToClusterTable.count(path2) == 0) {
+
+ // add the gossip in cluster 1 and also
+ // propagate the forward index for path2 -> oldCluster
+
+ int clusterIndex = m_pathToClusterTable[path1];
+
+ set<int> & clusterContent = m_gossipClusters[clusterIndex];
+
+ clusterContent.insert(newGossipIndex);
+ m_pathToClusterTable[path2] = clusterIndex;
+
+ // step 3.3 path1 has 0 matches but path2 has 1 match
+ } else if(m_pathToClusterTable.count(path1) == 0 && m_pathToClusterTable.count(path2) > 0) {
+
+ int clusterIndex = m_pathToClusterTable[path2];
+
+ set<int> & clusterContent = m_gossipClusters[clusterIndex];
+
+ clusterContent.insert(newGossipIndex);
+
+ m_pathToClusterTable[path1] = clusterIndex;
+
+ // case 3.4 both path have match in the same cluster
+ } else if(m_pathToClusterTable.count(path1) > 0
+ && m_pathToClusterTable.count(path2) > 0
+ && m_pathToClusterTable[path1] == m_pathToClusterTable[path2]) {
+
+ int clusterIndexForPath1 = m_pathToClusterTable[path1];
+
+ // it is the same cluster. (?)
+ // check if clusterIndexForPath1 and clusterIndexForPath2 are the
+ // same
+
+#ifdef CONFIG_ASSERT
+ int clusterIndexForPath2 = m_pathToClusterTable[path2];
+ assert(clusterIndexForPath1 == clusterIndexForPath2);
+#endif // CONFIG_ASSERT
+
+ int clusterIndex = clusterIndexForPath1;
+
+ set<int> & clusterContent = m_gossipClusters[clusterIndex];
+
+ clusterContent.insert(newGossipIndex);
+
+ // no update are necessary for the m_pathToClusterTable index
+
+ // step 3.5: path1 has 1 match and path2 has 1 match
+ // this bridges two existing clusters, how exciting !!!
+ } else if(m_pathToClusterTable.count(path1) > 0
+ && m_pathToClusterTable.count(path2) > 0) {
+
+ int clusterIndexForPath1 = m_pathToClusterTable[path1];
+ int clusterIndexForPath2 = m_pathToClusterTable[path2];
+
+#ifdef CONFIG_ASSERT
+ assert(clusterIndexForPath1 != clusterIndexForPath2);
+#endif // CONFIG_ASSERT
+
+ // TODO optimization: flip group1 and group2 if group2 is smaller than
+ // group1
+
+ mergedTwoClusters = true;
+
+ // add everything in the cluster of path1
+
+
+
+ set<int> & clusterContentForPath1 = m_gossipClusters[clusterIndexForPath1];
+ set<int> & clusterContentForPath2 = m_gossipClusters[clusterIndexForPath2];
+
+ for(set<int>::iterator i = clusterContentForPath2.begin() ;
+ i != clusterContentForPath2.end() ; ++i) {
+
+ int otherGossipIndex = *i;
+
+ GraphSearchResult & otherGossip = m_gossips[otherGossipIndex];
+
+#ifdef CONFIG_ASSERT
+ if(clusterContentForPath1.count(otherGossipIndex) > 0) {
+ cout << "Error: otherGossipIndex is in clusterContentForPath1 and clusterContentForPath2 ! ";
+ cout << " gossip: ";
+ cout << otherGossipIndex << " ";
+ otherGossip.print();
+ cout << " path1 " << path1;
+ cout << " path2 " << path2;
+ cout << " clusterIndexForPath1 " << clusterIndexForPath1;
+ cout << " clusterIndexForPath2 " << clusterIndexForPath2;
+ cout << " clusterContentForPath1 ";
+
+ for(set<int>::iterator j = clusterContentForPath1.begin();
+ j != clusterContentForPath1.end() ; ++j) {
+ cout << " " << *j;
+ }
+ cout << " clusterContentForPath2 ";
+
+ for(set<int>::iterator j = clusterContentForPath2.begin();
+ j != clusterContentForPath2.end() ; ++j) {
+ cout << " " << *j;
+ }
+ cout << endl;
+ }
+ assert(clusterContentForPath1.count(otherGossipIndex) == 0);
+#endif
+
+ PathHandle & otherPath1 = otherGossip.getPathHandles()[0];
+ PathHandle & otherPath2 = otherGossip.getPathHandles()[1];
+
+ // some assertions before changing things.
+#ifdef CONFIG_ASSERT
+ assert(m_pathToClusterTable.count(otherPath1) > 0);
+ assert(m_pathToClusterTable.count(otherPath2) > 0);
+
+ // These 2 assertions are overkill and invalid
+ // because otherPath1 (or otherPath2) can appear in more than
+ // 1 gossip !!! LOL XD XD
+ //assert(m_pathToClusterTable[otherPath1] == clusterIndexForPath1);
+ //assert(m_pathToClusterTable[otherPath2] == clusterIndexForPath2);
+#endif // CONFIG_ASSERT
+
+ clusterContentForPath1.insert(otherGossipIndex);
+
+ // update the index buckets
+ m_pathToClusterTable[otherPath1] = clusterIndexForPath1;
+ m_pathToClusterTable[otherPath2] = clusterIndexForPath1;
+
+ // some assertions *after* changing things.
+#ifdef CONFIG_ASSERT
+ assert(m_pathToClusterTable.count(otherPath1) > 0);
+ assert(m_pathToClusterTable.count(otherPath2) > 0);
+ assert(m_pathToClusterTable[otherPath1] == clusterIndexForPath1);
+ assert(m_pathToClusterTable[otherPath2] == clusterIndexForPath1);
+#endif // CONFIG_ASSERT
+
+
+ }
+
+ // at this point, the cluster for path1 contains everything that was in the old cluster
+ // for path1 and everything that was in the old cluster for path2
+
+ clusterContentForPath2.clear();
+
+ clusterContentForPath1.insert(newGossipIndex);
+
+ // we don't need to update m_pathToClusterTable because path1 and path2 are already
+ // indexed.
+ }
+
+
+ //------------------------------------------------------------//
+ // step 2. mark the gossip for future transportation
+
+ // get the cluster that contains gossip
+
+#ifdef CONFIG_ASSERT
+ assert(m_pathToClusterTable.count(path1) > 0);
+ assert(m_pathToClusterTable.count(path2) > 0);
+#endif
+
+ int finalCluster1 = m_pathToClusterTable[path1];
+
+#ifdef CONFIG_ASSERT
+ int finalCluster2 = m_pathToClusterTable[path2];
+ assert(finalCluster1 == finalCluster2);
+#endif
+
+ set<int> & finalClusterContent = m_gossipClusters[finalCluster1];
+
+#ifdef CONFIG_ASSERT
+ assert(finalClusterContent.count(newGossipIndex) > 0);
+#endif
+
+ // gather all destination for this cluster
+
+ set<Rank> destinations;
+
+ for(set<int>::iterator i = finalClusterContent.begin() ;
+ i != finalClusterContent.end() ; ++i) {
+
+ int theGossipIndex = *i;
+ GraphSearchResult & entry = m_gossips[theGossipIndex];
+
+ Rank rank1 = getRankFromPathUniqueId(entry.getPathHandles()[0]);
+ Rank rank2 = getRankFromPathUniqueId(entry.getPathHandles()[1]);
+
+ destinations.insert(rank1);
+ destinations.insert(rank2);
+ }
+
+ // here, we have a list of destinations to which we must send the gossip
+ // if mergedTwoClusters is false, we only need to send gossip to the destinations
+ //
+ // if mergedTwoClusters is true, however, we need to send every gossip in the cluster to
+ // every destination because a merge event occured.
+
+ set<int> dummySet;
+ dummySet.insert(newGossipIndex);
+
+ set<int> * gossipsToScheduleForTransportation = & dummySet;
+
+ if(mergedTwoClusters)
+ gossipsToScheduleForTransportation = & finalClusterContent;
+
+ int scheduledOperations = 0;
+
+ for(set<int>::iterator gossipIndexIterator = gossipsToScheduleForTransportation->begin();
+ gossipIndexIterator != gossipsToScheduleForTransportation->end();
+ ++gossipIndexIterator) {
+
+ int gossipIndexToSchedule = *gossipIndexIterator;
+
+ for(set<Rank>::iterator destinationIterator = destinations.begin() ;
+ destinationIterator != destinations.end();
+ ++destinationIterator) {
+
+ Rank destination = * destinationIterator;
+
+ if(scheduleTransportation(gossipIndexToSchedule, destination))
+ scheduledOperations ++;
+ }
+
+ }
+
+#ifdef CONFIG_DEBUG_GOSSIP_ASSET_MANAGER
+ cout << "DEBUG GossipAssetManager::classifyGossip classification of gossip ";
+ cout << newGossipIndex << " triggered " << scheduledOperations << " scheduling";
+ cout << " operations" << endl;
+#endif
+}
+
+bool GossipAssetManager::scheduleGossipTransportation(GraphSearchResult & gossip, Rank & destination) {
+
+ string key = gossip.toString();
+
+#ifdef CONFIG_ASSERT
+ assert(m_gossipIndex.count(key) > 0);
+#endif
+
+ int index = m_gossipIndex[key];
+
+ return scheduleTransportation(index, destination);
+}
+
+bool GossipAssetManager::scheduleTransportation(int gossipIndex, Rank destination) {
+
+ // the gossip is already on this destination
+ if(m_remoteGossipOwners.count(gossipIndex) > 0 && m_remoteGossipOwners[gossipIndex].count(destination) > 0)
+ return false;
+
+ m_futureRemoteGossipOwners[gossipIndex].insert(destination);
+
+#ifdef CONFIG_DEBUG_GOSSIP_ASSET_MANAGER
+ cout << "DEBUG /GossipAssetManager::scheduleTransportation scheduled gossip " << gossipIndex;
+ cout << " for immediate delivery to endpoint " << destination << endl;
+#endif
+
+ return true;
+}
+
+bool GossipAssetManager::hasGossipToShare() const {
+
+ return m_futureRemoteGossipOwners.size() > 0;
+}
+
+void GossipAssetManager::getGossipToShare(GraphSearchResult & gossip, Rank & destination) {
+
+#ifdef CONFIG_ASSERT
+ assert(hasGossipToShare());
+#endif
+
+ // return the first gossip to share
+ // and its first corresponding destination
+
+ map<int, set<Rank> >::iterator gossipIterator = m_futureRemoteGossipOwners.begin();
+
+ set<Rank> & destinations = gossipIterator->second;
+
+ const int & gossipIndex = gossipIterator->first;
+
+#ifdef CONFIG_ASSERT
+ assert(destinations.size() > 0);
+#endif
+
+ set<Rank>::iterator destinationIterator = destinations.begin();
+
+ const Rank & resultingDestination = *destinationIterator;
+
+ GraphSearchResult & resultingGossip = m_gossips[gossipIndex];
+
+ // give the objects to the caller
+
+ gossip = resultingGossip;
+ destination = resultingDestination;
+}
+
+vector<GraphSearchResult> & GossipAssetManager::getGossips() {
+ return m_gossips;
+}
+
+bool GossipAssetManager::hasGossip(const GraphSearchResult & gossip) const {
+
+ string key = gossip.toString();
+
+ return m_gossipIndex.count(key) > 0;
+}
+
+void GossipAssetManager::registerRemoteGossip(GraphSearchResult & gossip, Rank & destination) {
+
+ string key = gossip.toString();
+
+#ifdef CONFIG_ASSERT
+ assert(m_gossipIndex.count(key) > 0);
+#endif
+
+ int index = m_gossipIndex[key];
+
+#if 0
+ // check that the asset is registered to be sent to this remote
+ // Actually, we may want to register remote copies even when there is no scheduled
+ // operations.
+ assert(m_futureRemoteGossipOwners.count(index) > 0 && m_futureRemoteGossipOwners[index].count(destination) > 0);
+#endif
+
+ // remove any scheduled operation for this gossip and this destination
+ if(m_futureRemoteGossipOwners.count(index) > 0
+ && m_futureRemoteGossipOwners[index].count(destination) > 0) {
+
+ // at this point, everything looks OK.
+ m_futureRemoteGossipOwners[index].erase(destination);
+
+ if(m_futureRemoteGossipOwners[index].size() == 0) {
+ m_futureRemoteGossipOwners.erase(index);
+ }
+ }
+
+
+ /// this assertion is not valid
+#if 0
+#ifdef CONFIG_ASSERT
+ // the asset must not be already registered...
+ assert(m_remoteGossipOwners.count(index) == 0 || m_remoteGossipOwners[index].count(destination) == 0);
+#endif
+#endif
+
+
+ m_remoteGossipOwners[index].insert(destination);
+
+#if CONFIG_DEBUG_GOSSIP_ASSET_MANAGER
+ cout << "DEBUG GossipAssetManager::registerRemoteGossip" << index << " was successfully";
+ cout << " delivered to endpoint " << destination << endl;
+#endif
+}
diff --git a/code/SpuriousSeedAnnihilator/GossipAssetManager.h b/code/SpuriousSeedAnnihilator/GossipAssetManager.h
new file mode 100644
index 0000000..b9476d3
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/GossipAssetManager.h
@@ -0,0 +1,73 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef GossipAssetManagerHeader
+#define GossipAssetManagerHeader
+
+#include "GraphSearchResult.h"
+
+#include <vector>
+#include <map>
+#include <set>
+using namespace std;
+
+/**
+ * This class manages the assets (which are gossips).
+ *
+ * It also knows which gossip needs to be sent somewhere.
+ *
+ * This class dynamically computes the preference list for a gossip
+ * (the preference list of a gossip contains its primary actors).
+ *
+ *
+ * \author Sébastien Boisvert
+ */
+class GossipAssetManager {
+ vector<GraphSearchResult> m_gossips;
+
+ map<string, int> m_gossipIndex;
+
+ map<int, set<Rank> > m_remoteGossipOwners;
+ map<int, set<Rank> > m_futureRemoteGossipOwners;
+
+ vector<set<int> > m_gossipClusters;
+
+ map<PathHandle, int> m_pathToClusterTable;
+
+
+ void classifyGossip(GraphSearchResult & gossip);
+ bool scheduleTransportation(int gossipIndex, Rank destination);
+
+public:
+ void addGossip(GraphSearchResult & gossip);
+
+ bool hasGossipToShare() const;
+
+ void getGossipToShare(GraphSearchResult & gossip, Rank & destination);
+
+ vector<GraphSearchResult> & getGossips();
+
+ bool hasGossip(const GraphSearchResult & gossip) const;
+
+ void registerRemoteGossip(GraphSearchResult & gossip, Rank & destination);
+ bool scheduleGossipTransportation(GraphSearchResult & gossip, Rank & destination);
+};
+
+#endif
diff --git a/code/SpuriousSeedAnnihilator/GraphExplorer.cpp b/code/SpuriousSeedAnnihilator/GraphExplorer.cpp
new file mode 100644
index 0000000..5b7d6b3
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/GraphExplorer.cpp
@@ -0,0 +1,491 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "GraphExplorer.h"
+
+// #define DEBUG_EXPLORATION_123
+
+#define DEBUG_EXPLORATION_SHOW_SUMMARY
+
+void GraphExplorer::start(WorkerHandle key, Kmer * start, GraphPath * seed, int direction, Parameters * parameters,
+ VirtualCommunicator * virtualCommunicator,
+ RingAllocator * outboxAllocator,
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE, MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH,
+ PathHandle seedName
+) {
+
+ m_seed = seed;
+ m_start = *start;
+
+ //cout << "[DEBUG] starting graph search with explorer technology" << endl;
+
+ m_key = key;
+ m_direction = direction;
+
+ m_parameters = parameters;
+ m_virtualCommunicator = virtualCommunicator;
+ m_outboxAllocator = outboxAllocator;
+
+ this->RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT = RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+ this->RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE = RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ this->RAY_MPI_TAG_ASK_VERTEX_PATH = RAY_MPI_TAG_ASK_VERTEX_PATH;
+
+ m_done = false;
+ m_maximumDepth = 128;
+ m_maximumVisitedVertices = 1024;
+
+ while(!m_verticesToVisit.empty())
+ m_verticesToVisit.pop();
+
+ while(!m_depths.empty())
+ m_depths.pop();
+
+ m_verticesToVisit.push(*start);
+
+ int depth = 0;
+ m_depths.push(depth);
+
+ m_haveAttributes = false;
+ m_haveAnnotations = false;
+ m_haveAnnotationsReverse = false;
+
+ WorkerHandle identifier = m_key;
+
+ m_attributeFetcher.initialize(parameters, virtualCommunicator,
+ identifier, outboxAllocator,
+ RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT);
+
+ m_annotationFetcher.initialize(parameters, virtualCommunicator,
+ identifier, outboxAllocator,
+ RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ RAY_MPI_TAG_ASK_VERTEX_PATH);
+
+ m_annotationFetcherReverse.initialize(parameters, virtualCommunicator,
+ identifier, outboxAllocator,
+ RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ RAY_MPI_TAG_ASK_VERTEX_PATH);
+
+
+ m_stopAtFirstHit = true;
+
+ m_seedName = seedName;
+ m_visitedVertices = 0;
+ m_maximumVisitedDepth = 0;
+ m_searchDepthForFirstResult = -1;
+
+ m_vertexDepths.clear();
+ m_parents.clear();
+ m_searchResults.clear();
+
+ //cout << "[DEBUG] explorer is ready" << endl;
+
+ m_debug = false;
+
+#ifdef DEBUG_EXPLORATION_123
+ bool tryToDebug = true;
+
+ /**
+ * DEBUG URI http://genome.ulaval.ca:10241/client/?map=3§ion=0®ion=254&location=1734&depth=10
+ */
+
+ //const char * key1 = "GGAATCATGAGAAGTCAGCCG";
+ //const char * key1 = "AAATCCCTCTTTTTACAATTG";
+ //const char * key1 = "TTTCGTGAAAAAAGTTAACAA";
+ const char * key1 = "ATAATAAGAGTTATCATCTCC"; // \see http://genome.ulaval.ca:10241/client/?map=3§ion=0®ion=480&location=0&depth=10&zoom=0.8088851319448179
+
+ if(tryToDebug && start->idToWord(m_parameters->getWordSize(), m_parameters->getColorSpaceMode()) == key1) {
+ m_debug = true;
+ cout << "[DEBUG] 8d97f6e851 BEGIN" << endl;
+ }
+#endif
+
+
+}
+
+// TODO: use the coverage value instead of depth
+bool GraphExplorer::getBestParent(Kmer * parent, Kmer kmer) {
+
+ if(m_parents.count(kmer) == 0)
+ return false;
+
+ int bestDepth = 99999;
+ Kmer bestKmer;
+
+ for(vector<Kmer>::iterator i = m_parents[kmer].begin();
+ i != m_parents[kmer].end();
+ i++) {
+
+ Kmer theParent = *i;
+
+ int parentDepth = m_vertexDepths[theParent];
+
+ if(parentDepth < bestDepth) {
+ bestKmer = theParent;
+ bestDepth = parentDepth;
+ }
+ }
+
+ *parent = bestKmer;
+
+ return true;
+}
+
+/**
+ * Here, the word "parent" is context-specific. It means the parent in the
+ * course. If we use EXPLORER_LEFT, then parents are truly parents.
+ * But with EXPLORER_RIGHT, parents are in fact children.
+ */
+bool GraphExplorer::backtrackPath(vector<Kmer> * path, Kmer * vertex) {
+ Kmer item = *vertex;
+
+ set<Kmer> visited;
+
+ vector<Kmer> aPath;
+
+ while(1) {
+
+ if(visited.count(item) > 0)
+ break;
+
+ aPath.push_back(item);
+ visited.insert(item);
+
+ if(item == m_start)
+ break;
+
+ Kmer parent;
+
+ if(!getBestParent(&parent, item))
+ break;
+
+ item = parent;
+ }
+
+ // now the path include the source and the sink and also
+ // is in reverse order...
+
+#ifdef CONFIG_ASSERT
+ if(!(aPath.size() >= 3)) {
+ cout << "DEBUG Warning, backtrackPath yielded (expected >= 3)";
+ cout << aPath.size() << " " << aPath.size() << endl;
+
+ return false;
+ }
+ //assert(aPath.size() >= 3); // source + sink + at least one stranger.
+#endif
+
+ // remove first and last
+ int position = 1;
+ while(position <= (int)aPath.size() -2) {
+ path->push_back(aPath[position++]);
+ }
+
+ aPath.clear();
+
+ // reverse to enforce the de Bruijn property
+ if(m_direction == EXPLORER_RIGHT) {
+ int firstPosition = 0;
+ int lastPosition = path->size()-1;
+
+ // while(firstPosition < lastPosition) { error: stray ‘\302’ in program
+ while(firstPosition < lastPosition) {
+ Kmer holder = (*path)[firstPosition];
+ (*path)[firstPosition] = (*path)[lastPosition];
+ (*path)[lastPosition] = holder;
+ firstPosition ++;
+ lastPosition --;
+ }
+ }
+
+ return true;
+}
+
+bool GraphExplorer::processAnnotations(AnnotationFetcher & annotationFetcher, int currentDepth, Kmer & object) {
+
+ bool foundSomething = false;
+ for(int i=0;i< (int) annotationFetcher.getDirections()->size(); i++){
+
+ Direction & direction = annotationFetcher.getDirections()->at(i);
+
+ PathHandle pathName = direction.getPathHandle();
+ int position = direction.getPosition();
+ bool pathStrand = false;
+
+ if(m_direction == EXPLORER_RIGHT && position != 0)
+ pathStrand = true;
+ else if(m_direction == EXPLORER_LEFT && position == 0)
+ pathStrand = true;
+
+ // the self path will always be found at depth 0
+ // Streptococcus pneumoniae has a lot of these loops where a seeds touch itself via
+ // a short 2X-coverage region.
+ // This was seen in Ray Cloud Browser.
+
+ if(currentDepth != 0) {
+
+ // skip self loops
+ if(pathName == m_seedName) {
+ continue;
+ }
+
+#ifdef INTERNET_EXPLORER_DEBUG_PATHS
+ cout << "[DEBUG] GraphExplorer found path " << pathName << " during graph search";
+ cout << ", visited " << m_visitedVertices << ", started from " << m_seedName;
+
+ cout << " direction ";
+
+ if(m_direction == EXPLORER_LEFT)
+ cout << "EXPLORER_LEFT";
+ else if(m_direction == EXPLORER_RIGHT)
+ cout << "EXPLORER_RIGHT";
+
+ cout << " depth " << currentDepth;
+#endif
+
+
+
+ // here we can not use GraphPath directly because the de Bruijn property
+ // is hardly enforced in both directions
+ vector<Kmer> pathToOrigin;
+
+ if(!backtrackPath(&pathToOrigin, &object)) {
+ cout << "DEBUG Warning backtrackPath failed ";
+ cout << " m_seedName " << m_seedName << " ";
+ cout << " pathName " << pathName << endl;
+
+ continue;
+ }
+
+ // we found something !
+ foundSomething = true;
+
+#ifdef INTERNET_EXPLORER_DEBUG_PATHS
+ cout << " path has length " << pathToOrigin.size() << endl;
+#endif
+
+ GraphPath aPath;
+ aPath.setKmerLength(m_parameters->getWordSize());
+ for(int i = 0 ; i < (int)pathToOrigin.size() ; i++) {
+ Kmer kmer = pathToOrigin[i];
+ aPath.push_back(&kmer);
+ }
+
+ GraphSearchResult result;
+
+ if(m_direction == EXPLORER_RIGHT) {
+ result.addPathHandle(m_seedName, false);
+ result.addPath(aPath);
+ result.addPathHandle(pathName, pathStrand);
+ } else if(m_direction == EXPLORER_LEFT) {
+ result.addPathHandle(pathName, pathStrand);
+ result.addPath(aPath);
+ result.addPathHandle(m_seedName, false);
+ }
+
+ m_searchResults.push_back(result);
+ }
+ }
+
+ return foundSomething;
+}
+
+// TODO: query also the other DNA strand for annotations
+bool GraphExplorer::work() {
+
+ if(m_verticesToVisit.empty())
+ m_done = true;
+
+ if(m_done) {
+
+#ifdef DEBUG_EXPLORATION_SHOW_SUMMARY
+ m_debug = true;
+ if(m_debug) {
+ cout << "[DEBUG] 8d97f6e851 completed, m_visitedVertices " << m_visitedVertices;
+ cout << " path " << m_seedName;
+ cout << " m_searchDepthForFirstResult " << m_searchDepthForFirstResult;
+ cout << " m_maximumVisitedDepth " << m_maximumVisitedDepth;
+ cout << " lengthInKmers " << m_seed->size();
+ cout << " direction ";
+
+
+ if(m_direction == EXPLORER_LEFT)
+ cout << "EXPLORER_LEFT";
+ else
+ cout << "EXPLORER_RIGHT";
+
+ cout << " search results: " << m_searchResults.size();
+ cout << endl;
+ }
+ m_debug = false;
+#endif
+
+ return m_done;
+ }
+
+#ifdef ASSERT
+ assert(!m_verticesToVisit.empty());
+ assert(!m_depths.empty());
+#endif
+
+ Kmer object = m_verticesToVisit.top();
+ Kmer reverseObject = object.complementVertex(m_parameters->getWordSize(), m_parameters->getColorSpaceMode());
+
+ if(!m_haveAttributes && m_attributeFetcher.fetchObjectMetaData(&object)) {
+
+ m_haveAttributes = true;
+ m_haveAnnotations = false;
+ //cout << "[DEBUG] have attributes" << endl;
+
+ } else if(m_haveAttributes && !m_haveAnnotations && m_annotationFetcher.fetchDirections(&object)) {
+
+ m_haveAnnotations = true;
+ m_haveAnnotationsReverse = false;
+
+ } else if(m_haveAttributes && m_haveAnnotations && !m_haveAnnotationsReverse
+ && m_annotationFetcherReverse.fetchDirections(&reverseObject)) {
+
+ m_haveAnnotationsReverse = true;
+ //cout << "[DEBUG] have annotations" << endl;
+
+ } else if(m_haveAttributes && m_haveAnnotations && m_haveAnnotationsReverse) {
+
+#ifdef DEBUG_EXPLORATION_123
+ if(m_debug) {
+ cout << "[DEBUG] 8d97f6e851 vertex " << object.idToWord(m_parameters->getWordSize(), m_parameters->getColorSpaceMode());
+ cout << " depth " << m_attributeFetcher.getDepth() << " children [ ";
+
+ for(int i = 0 ; i < (int) m_attributeFetcher.getChildren()->size() ; ++i) {
+
+ cout << " " << m_attributeFetcher.getChildren()->at(i).idToWord(m_parameters->getWordSize(), m_parameters->getColorSpaceMode());
+ }
+ cout << " ]";
+
+ cout << " forward annotations: " << m_annotationFetcher.getDirections()->size();
+ cout << " reverse annotations: " << m_annotationFetcherReverse.getDirections()->size();
+ cout << endl;
+ }
+#endif
+
+#ifdef DEBUG_EXPLORATION_SHOW_SUMMARY
+
+ if(m_visitedVertices == 0) {
+ CoverageDepth coverageDepthForFirstVertex = 0;
+ coverageDepthForFirstVertex = m_attributeFetcher.getDepth();
+
+ cout << "DEBUG -> ";
+ cout << "BiologicalObject: ";
+ cout << object.idToWord(m_parameters->getWordSize(), m_parameters->getColorSpaceMode());
+ cout << " SequencingDepth: ";
+ cout << coverageDepthForFirstVertex;
+ cout << endl;
+ }
+#endif
+
+ int currentDepth = m_depths.top();
+
+ m_vertexDepths[object] = currentDepth;
+
+ bool foundSomething = false;
+
+ if(currentDepth > m_maximumVisitedDepth)
+ m_maximumVisitedDepth = currentDepth;
+
+ //cout << "[DEBUG] processing object now depth=" << currentDepth << " visited= " << m_visitedVertices << endl;
+
+ if(processAnnotations(m_annotationFetcher, currentDepth, object))
+ foundSomething = true;
+ if(processAnnotations(m_annotationFetcherReverse, currentDepth, object))
+ foundSomething = true;
+
+ if(foundSomething && m_searchDepthForFirstResult < 0)
+ m_searchDepthForFirstResult = currentDepth;
+
+#ifdef ASSERT
+ assert(!m_depths.empty());
+#endif
+
+ int newDepth = currentDepth + 1;
+
+ m_depths.pop();
+ m_verticesToVisit.pop();
+
+ vector<Kmer> * links = NULL;
+
+ if(m_direction == EXPLORER_LEFT)
+ links = m_attributeFetcher.getParents();
+ else if(m_direction == EXPLORER_RIGHT)
+ links = m_attributeFetcher.getChildren();
+
+#ifdef ASSERT
+ assert(links != NULL);
+#endif
+
+ if(newDepth <= m_maximumDepth
+ && m_visitedVertices + (int)links->size() <= m_maximumVisitedVertices) {
+
+ for(int i = 0 ; i < (int)links->size() ; i ++) {
+
+ if(!foundSomething) {
+ Kmer nextKmer = links->at(i);
+
+ // implemented already: check if there is not already another parent.
+ // if it is the case, select the path with the coverage
+ // that is the nearest to the one of both paths
+ m_parents[nextKmer].push_back(object);
+
+ m_verticesToVisit.push(nextKmer);
+ m_depths.push(newDepth);
+ }
+ }
+ }
+
+ m_annotationFetcher.reset();
+ m_annotationFetcherReverse.reset();
+ m_attributeFetcher.reset();
+
+ m_visitedVertices ++;
+
+ m_haveAttributes = false;
+ }
+
+ return m_done;
+}
+
+vector<GraphSearchResult> & GraphExplorer::getSearchResults() {
+ return m_searchResults;
+}
+
+/**
+ * \see http://stackoverflow.com/questions/13639535/what-are-the-naming-conventions-of-functions-that-return-boolean
+ */
+bool GraphExplorer::isValid() const {
+
+ if(m_searchResults.size() != 1)
+ return false;
+
+ if(m_visitedVertices >= m_maximumVisitedVertices)
+ return false;
+
+ if(m_maximumVisitedDepth >= m_maximumDepth)
+ return false;
+
+ return true;
+}
diff --git a/code/SpuriousSeedAnnihilator/GraphExplorer.h b/code/SpuriousSeedAnnihilator/GraphExplorer.h
new file mode 100644
index 0000000..155c846
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/GraphExplorer.h
@@ -0,0 +1,112 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef GraphExplorer_header
+#define GraphExplorer_header
+
+#include "AttributeFetcher.h"
+#include "AnnotationFetcher.h"
+#include "GraphSearchResult.h"
+
+#include <code/Mock/Parameters.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+
+#include <stack>
+using namespace std;
+
+#define EXPLORER_LEFT 0x89
+#define EXPLORER_RIGHT 0x452
+
+/**
+ * This class is an explorer to find paths leading to new paths.
+ *
+ * \author Sébastien Boisvert
+ */
+class GraphExplorer {
+
+ int m_searchDepthForFirstResult;
+ bool m_debug;
+ vector<GraphSearchResult> m_searchResults;
+
+ stack<Kmer> m_verticesToVisit;
+ stack<int> m_depths;
+
+ int m_maximumVisitedDepth;
+
+ AnnotationFetcher m_annotationFetcher;
+ AnnotationFetcher m_annotationFetcherReverse;
+ AttributeFetcher m_attributeFetcher;
+
+ WorkerHandle m_key;
+ int m_direction;
+ PathHandle m_seedName;
+
+ map<Kmer, vector<Kmer> > m_parents;
+ map<Kmer, int> m_vertexDepths;
+
+ CoverageDepth m_coverage1;
+
+ Kmer m_start;
+
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH;
+
+ bool m_haveAttributes;
+ bool m_haveAnnotations;
+ bool m_haveAnnotationsReverse;
+
+ bool m_stopAtFirstHit;
+ bool m_done;
+ int m_maximumDepth;
+
+ VirtualCommunicator * m_virtualCommunicator;
+ Parameters * m_parameters;
+ int m_maximumVisitedVertices;
+ int m_visitedVertices;
+ RingAllocator*m_outboxAllocator;
+ GraphPath * m_seed;
+
+ bool backtrackPath(vector<Kmer> * path, Kmer * vertex);
+ bool getBestParent(Kmer * parent, Kmer kmer);
+ bool processAnnotations(AnnotationFetcher & annotationFetcher, int currentDepth, Kmer & object);
+
+public:
+ void start(WorkerHandle worker, Kmer * start, GraphPath * seed, int direction, Parameters * parameters,
+ VirtualCommunicator * virtualCommunicator,
+ RingAllocator * outboxAllocator,
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE, MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH,
+ PathHandle seedName
+ );
+
+ bool work();
+
+ vector<GraphSearchResult> & getSearchResults();
+
+ bool isValid() const ;
+};
+
+#endif /* GraphExplorer_header */
+
+
diff --git a/code/SpuriousSeedAnnihilator/GraphSearchResult.cpp b/code/SpuriousSeedAnnihilator/GraphSearchResult.cpp
new file mode 100644
index 0000000..6345a55
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/GraphSearchResult.cpp
@@ -0,0 +1,267 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "GraphSearchResult.h"
+
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <algorithm>
+using namespace std;
+
+#include <string.h>
+
+bool GraphSearchResult::addPathHandle(PathHandle handle, bool orientation) {
+
+ // we need a path after each handle
+ if(m_computedPaths.size() != m_pathHandles.size())
+ return false;
+
+ m_pathHandles.push_back(handle);
+ m_pathOrientations.push_back(orientation);
+
+ return true;
+}
+
+bool GraphSearchResult::addPath(GraphPath & path) {
+
+ if(m_pathHandles.size() - 1 != m_computedPaths.size())
+ return false;
+
+ m_computedPaths.push_back(path);
+
+ return true;
+
+}
+
+void GraphSearchResult::print() {
+ int i = 0;
+
+ while(i < (int)m_pathHandles.size()) {
+ cout << " [" << m_pathHandles[i] << ":" << m_pathOrientations[i] << "]";
+
+ if(i != (int)m_pathHandles.size() -1) {
+
+ cout << " " << m_computedPaths[i].size();
+ }
+ i++;
+ }
+}
+
+int GraphSearchResult::load(const char * buffer) {
+
+ int position = 0;
+
+ uint32_t paths = 0;
+
+ int size = sizeof(uint32_t);
+
+ memcpy(&paths, buffer + position, size);
+ position += size;
+
+ //cout << "[DEBUG] GraphSearchResult::load paths " << paths << endl;
+
+ for(int i = 0 ; i < (int)paths; i++) {
+ PathHandle entry;
+ position += entry.load(buffer + position);
+ m_pathHandles.push_back(entry);
+ }
+
+ for(int i = 0 ; i < (int)paths; i++) {
+ size = sizeof(bool);
+ bool strand;
+ memcpy(&strand, buffer + position, size);
+ position += size;
+
+ m_pathOrientations.push_back(strand);
+ }
+
+ int computedPaths = paths - 1;
+
+ for(int i = 0 ; i < computedPaths ; i ++) {
+ GraphPath aParticularPath;
+ size = aParticularPath.load(buffer + position);
+ m_computedPaths.push_back(aParticularPath);
+ position += size;
+ }
+
+ return position;
+}
+
+int GraphSearchResult::dump(char * buffer) const {
+
+ int position = 0;
+
+ uint32_t paths = m_pathHandles.size();
+ int size = sizeof(uint32_t);
+
+ //cout << "[DEBUG] GraphSearchResult::dump paths " << paths << endl;
+
+ memcpy(buffer + position, &paths, size);
+ position += size;
+
+ for(int i = 0 ; i < (int)paths ; i++) {
+ position += m_pathHandles[i].dump(buffer + position);
+ }
+
+ for(int i = 0 ; i < (int)paths ; i ++) {
+ size = sizeof(bool);
+ bool value = m_pathOrientations[i];
+ memcpy(buffer + position, &value, size);
+ position += size;
+ }
+
+ int computedPaths = paths - 1;
+
+ for(int i = 0 ; i < computedPaths ; i ++) {
+#ifdef CONFIG_ASSERT
+ if(i >= (int)m_computedPaths.size()) {
+ cout << "Error i " << i << " m_computedPaths.size() ";
+ cout << m_computedPaths.size() << " computedPaths " << computedPaths << endl;
+ }
+
+ assert(i >= 0);
+ assert(i < (int)m_computedPaths.size());
+#endif /* CONFIG_ASSERT */
+
+ position += m_computedPaths[i].dump(buffer + position);
+ }
+
+ return position;
+}
+
+vector<PathHandle> & GraphSearchResult::getPathHandles() {
+ return m_pathHandles;
+}
+
+string GraphSearchResult::toString() const {
+
+ ostringstream value;
+
+ if(m_pathHandles.size() != 2)
+ return "Error-not-implemented";
+
+ PathHandle path1 = m_pathHandles[0];
+ PathHandle path2 = m_pathHandles[1];
+
+ if(path1 < path2)
+ value << path1 << "-" << path2;
+ else
+ value << path2 << "-" << path1;
+
+ return value.str();
+}
+
+bool GraphSearchResult::hasData() const {
+ return m_pathHandles.size() > 0;
+}
+
+vector<bool> & GraphSearchResult::getPathOrientations() {
+ return m_pathOrientations;
+}
+
+/**
+ * TODO implement this with something that is not a vector that allows push_front.
+ * std::list has that.
+ *
+ * it seems that vector has insert(), but it is linear in complexity...
+ */
+bool GraphSearchResult::addPathOnLeftSide(PathHandle & handle, bool strand, GraphPath & path) {
+
+ // don't add an already added bit
+ if(hasPath(handle))
+ return false;
+
+#ifdef CONFIG_ASSERT
+ assert(!hasPath(handle));
+#endif
+ m_pathHandles.insert(m_pathHandles.begin(), handle);
+ m_pathOrientations.insert(m_pathOrientations.begin(), strand);
+ m_computedPaths.insert(m_computedPaths.begin(), path);
+
+ return true;
+}
+
+bool GraphSearchResult::addPathOnRightSide(PathHandle & handle, bool strand, GraphPath & path) {
+
+ // don't add an already added bit
+ if(hasPath(handle))
+ return false;
+
+#ifdef CONFIG_ASSERT
+ assert(!hasPath(handle));
+#endif
+
+ m_pathHandles.push_back(handle);
+ m_pathOrientations.push_back(strand);
+ m_computedPaths.push_back(path);
+
+ return true;
+}
+
+bool GraphSearchResult::hasPath(PathHandle & handle) {
+ for(vector<PathHandle>::iterator i = m_pathHandles.begin() ;
+ i != m_pathHandles.end() ; ++i) {
+
+ PathHandle & otherHandle = *i;
+
+ if(otherHandle == handle)
+ return true;
+ }
+
+ return false;
+}
+
+vector<GraphPath> & GraphSearchResult::getComputedPaths() {
+ return m_computedPaths;
+}
+
+void GraphSearchResult::reverseContent() {
+
+ // reverse the order
+ reverse(m_pathHandles.begin(), m_pathHandles.end());
+ reverse(m_pathOrientations.begin(), m_pathOrientations.end());
+ reverse(m_computedPaths.begin(), m_computedPaths.end());
+
+ // reverse each computed path
+ for(int i = 0 ; i < (int) m_computedPaths.size() ; ++i) {
+ GraphPath newPath;
+ GraphPath & oldPath = m_computedPaths[i];
+ newPath.setKmerLength(oldPath.getKmerLength());
+
+ oldPath.reverseContent(newPath);
+
+ // don't forget to set the new path as well.
+ m_computedPaths[i] = newPath;
+ }
+
+ // reverse strands
+
+ for(int i = 0 ; i < (int) m_pathOrientations.size() ; ++i) {
+ m_pathOrientations[i] = !m_pathOrientations[i];
+ }
+}
+
+/**
+ * TODO not implemented...
+ */
+int GraphSearchResult::getRequiredNumberOfBytes() const {
+ return 0;
+}
diff --git a/code/SpuriousSeedAnnihilator/GraphSearchResult.h b/code/SpuriousSeedAnnihilator/GraphSearchResult.h
new file mode 100644
index 0000000..fd97245
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/GraphSearchResult.h
@@ -0,0 +1,89 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef GraphSearchResult_Header
+#define GraphSearchResult_Header
+
+#include <code/SeedingData/PathHandle.h>
+#include <code/SeedingData/GraphPath.h>
+
+#include <RayPlatform/store/CarriageableItem.h>
+
+#include <vector>
+using namespace std;
+
+/**
+ * This stores a graph search result.
+ *
+ * Path0 ----------> Path1 ------> Path2 -------> Path3
+ *
+ * start start start start
+ *
+ * end end end end
+ *
+ *
+ * There are:
+ *
+ * * N m_pathNames
+ * * N-1 m_computedPaths
+ * * N m_pathStarts
+ * * N m_pathEnds
+ *
+ * if start > end, it means that we must use the other DNA strand.
+ *
+ * \author Sébastien Boisvert
+ */
+class GraphSearchResult: public CarriageableItem {
+
+ vector<PathHandle> m_pathHandles;
+ vector<bool> m_pathOrientations; // false is normal, true is reverse
+ vector<GraphPath> m_computedPaths;
+
+ bool hasPath(PathHandle & handle);
+
+public:
+
+ bool addPathHandle(PathHandle handle, bool orientation);
+ bool addPath(GraphPath & path);
+
+ void print();
+
+ int load(const char * buffer);
+ int dump(char * buffer) const;
+ int getRequiredNumberOfBytes() const;
+
+ vector<PathHandle> & getPathHandles();
+ vector<bool> & getPathOrientations();
+ vector<GraphPath> & getComputedPaths();
+
+ bool addPathOnLeftSide(PathHandle & handle, bool strand, GraphPath & path);
+ bool addPathOnRightSide(PathHandle & handle, bool strand, GraphPath & path);
+
+ string toString() const;
+
+ /**
+ * Rotate the object to obtain a reverse-complement.
+ */
+ void reverseContent();
+
+ bool hasData() const;
+};
+
+#endif /* GraphSearchResult_Header */
diff --git a/code/SpuriousSeedAnnihilator/Makefile b/code/SpuriousSeedAnnihilator/Makefile
new file mode 100644
index 0000000..5a43f49
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/Makefile
@@ -0,0 +1,13 @@
+SpuriousSeedAnnihilator-y += code/SpuriousSeedAnnihilator/SpuriousSeedAnnihilator.o
+SpuriousSeedAnnihilator-y += code/SpuriousSeedAnnihilator/AnnihilationWorker.o
+SpuriousSeedAnnihilator-y += code/SpuriousSeedAnnihilator/SeedFilteringWorkflow.o
+SpuriousSeedAnnihilator-y += code/SpuriousSeedAnnihilator/AttributeFetcher.o
+SpuriousSeedAnnihilator-y += code/SpuriousSeedAnnihilator/AnnotationFetcher.o
+SpuriousSeedAnnihilator-y += code/SpuriousSeedAnnihilator/NanoMerger.o
+SpuriousSeedAnnihilator-y += code/SpuriousSeedAnnihilator/SeedMergingWorkflow.o
+SpuriousSeedAnnihilator-y += code/SpuriousSeedAnnihilator/GraphExplorer.o
+SpuriousSeedAnnihilator-y += code/SpuriousSeedAnnihilator/GraphSearchResult.o
+SpuriousSeedAnnihilator-y += code/SpuriousSeedAnnihilator/SeedGossipSolver.o
+SpuriousSeedAnnihilator-y += code/SpuriousSeedAnnihilator/GossipAssetManager.o
+
+obj-y += $(SpuriousSeedAnnihilator-y)
diff --git a/code/SpuriousSeedAnnihilator/NanoMerger.cpp b/code/SpuriousSeedAnnihilator/NanoMerger.cpp
new file mode 100644
index 0000000..114805d
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/NanoMerger.cpp
@@ -0,0 +1,167 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "NanoMerger.h"
+
+#include <stack>
+using namespace std;
+
+/**
+ * Here, we want to do a depth first search
+ * on the left and on the right to find
+ * nearby paths.
+ *
+ * Local arbitration will deal with ownership.
+ *
+ * \author Sébastien Boisvert
+ */
+void NanoMerger::work(){
+
+ //m_done = true;
+
+ if(!m_startedFirst) {
+
+ m_explorer.start(m_identifier, &m_first, m_seed, EXPLORER_LEFT, m_parameters,
+ m_virtualCommunicator,
+ m_outboxAllocator,
+ RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE, RAY_MPI_TAG_ASK_VERTEX_PATH,
+ m_seedName
+ );
+
+ m_startedFirst = true;
+
+ } else if(m_startedFirst && !m_startedLast && m_explorer.work()) {
+
+#ifdef DEBUG_EXPLORER_PATHS
+ cout << "[DEBUG] NanoMerger processed first, seed length is " << m_seed->size() << endl;
+#endif
+
+#ifdef DEBUG_EXPLORER_PATHS
+ cout << "[DEBUG] path " << m_seedName << " EXPLORER_LEFT " << m_explorer.getSearchResults().size() << " paths found" << endl;
+#endif
+
+ if(m_explorer.isValid() && m_explorer.getSearchResults().size() == 1) {
+
+ m_results.push_back(m_explorer.getSearchResults()[0]);
+#ifdef DEBUG_EXPLORER_PATHS
+ cout << "[DEBUG]";
+
+ m_explorer.getSearchResults()[0].print();
+
+ cout << endl;
+#endif
+ }
+
+ // now do the last one.
+ m_explorer.start(m_identifier, &m_last, m_seed, EXPLORER_RIGHT, m_parameters,
+ m_virtualCommunicator,
+ m_outboxAllocator,
+ RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE, RAY_MPI_TAG_ASK_VERTEX_PATH,
+ m_seedName
+ );
+
+ m_startedLast = true;
+
+ } else if(m_startedLast && m_explorer.work()) {
+
+#ifdef DEBUG_EXPLORER_PATHS
+ cout << "[DEBUG] path " << m_seedName << " EXPLORER_RIGHT " << m_explorer.getSearchResults().size() << " paths found" << endl;
+#endif
+
+ if(m_explorer.isValid() && m_explorer.getSearchResults().size() == 1) {
+
+ m_results.push_back(m_explorer.getSearchResults()[0]);
+
+#ifdef DEBUG_EXPLORER_PATHS
+ cout << "[DEBUG]";
+ m_explorer.getSearchResults()[0].print();
+ cout << endl;
+#endif
+ }
+
+
+#ifdef DEBUG_EXPLORER_PATHS
+ cout << "[DEBUG] NanoMerger processed last, seed length is " << m_seed->size() << endl;
+#endif
+ m_done = true;
+ }
+}
+
+bool NanoMerger::isDone(){
+
+ return m_done;
+}
+
+WorkerHandle NanoMerger::getWorkerIdentifier(){
+
+ return m_identifier;
+}
+
+/**
+ * RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE
+ * RAY_MPI_TAG_ASK_VERTEX_PATH
+ */
+void NanoMerger::initialize(WorkerHandle identifier,GraphPath*seed, Parameters * parameters,
+ VirtualCommunicator * virtualCommunicator, RingAllocator*outboxAllocator,
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE, MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH
+ ){
+
+ //cout << "[DEBUG] configuring nano merger now." << endl;
+
+ m_identifier = identifier;
+ m_done = false;
+
+ m_seed = seed;
+
+ m_virtualCommunicator = virtualCommunicator;
+ m_parameters = parameters;
+
+ this->RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT = RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+
+ this->RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE = RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ this->RAY_MPI_TAG_ASK_VERTEX_PATH = RAY_MPI_TAG_ASK_VERTEX_PATH;
+
+ m_rank = m_parameters->getRank();
+ m_outboxAllocator = outboxAllocator;
+
+ int index1 = 0;
+ m_seed->at(index1, &m_first);
+
+ int index2 = m_seed->size() - 1;
+ m_seed->at(index2, &m_last);
+
+#ifdef CONFIG_ASSERT
+ assert(index1 == 0);
+ assert(index2 >= 0);
+#endif
+ //cout << "[DEBUG] achieved with RayPlatform." << endl;
+
+ m_startedFirst = false;
+ m_startedLast = false;
+
+ m_seedName = getPathUniqueId(m_parameters->getRank(), identifier);
+}
+
+vector<GraphSearchResult> & NanoMerger::getResults() {
+ return m_results;
+}
diff --git a/code/SpuriousSeedAnnihilator/NanoMerger.h b/code/SpuriousSeedAnnihilator/NanoMerger.h
new file mode 100644
index 0000000..d4c3e4a
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/NanoMerger.h
@@ -0,0 +1,136 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef _NanoMerger_h
+#define _NanoMerger_h
+
+#include "GraphExplorer.h"
+#include "GraphSearchResult.h"
+
+#include <code/SeedingData/GraphPath.h>
+#include <code/Mock/Parameters.h>
+#include <code/SeedExtender/Direction.h>
+
+#include <RayPlatform/scheduling/Worker.h>
+
+#include <stack>
+#include <vector>
+#include <set>
+using namespace std;
+
+#include <stdint.h>
+
+/**
+ * This is a worker that analyze a seed.
+ *
+ * This class can not use ./SeedExtender/DepthFirstSearchData.h because
+ * ./SeedExtender/DepthFirstSearchData.h does not fetch annotations
+ * and is not 100% compatible with small workers.
+ *
+ * \author Sébastien Boisvert
+ */
+class NanoMerger: public Worker{
+
+ vector<GraphSearchResult> m_results;
+ bool m_startedFirst;
+ bool m_startedLast;
+ PathHandle m_seedName;
+
+ GraphExplorer m_explorer;
+
+ Rank m_rank;
+ RingAllocator*m_outboxAllocator;
+
+ uint64_t m_identifier; // TODO this should be in Worker because it's always there anyway
+ bool m_done; // TODO this should be in Worker because it's always there anyway
+ GraphPath * m_seed;
+
+ VirtualCommunicator * m_virtualCommunicator; // TODO this should be in Worker because it's always there anyway
+ Parameters * m_parameters;
+
+ int m_step;
+
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH;
+
+/*
+ stack<int> m_depths;
+ stack<Kmer> m_vertices;
+ int m_maximumAllowedDepth;
+ int m_actualMaximumDepth;
+ int m_maximumDepthForExploration;
+
+ bool m_valid;
+ set<Kmer> m_visited;
+
+ bool m_searchIsStarted;
+
+ int DIRECTION_PARENTS;
+ int DIRECTION_CHILDREN;
+
+ bool m_fetchedFirstParent;
+ bool m_fetchedSecondParent;
+ bool m_fetchedFirstChild;
+ bool m_fetchedSecondChild;
+
+ vector<Direction> m_leftDirections;
+ vector<Direction> m_rightDirections;
+ bool m_fetchedGrandparentDirections;
+ bool m_fetchedGrandchildDirections;
+ bool m_fetchedGrandparentReverseDirections;
+ bool m_fetchedGrandchildReverseDirections;
+ bool m_isPerfectBubble;
+*/
+// private methods
+
+ /*
+ bool checkDeadEndOnTheLeft();
+ bool checkDeadEndOnTheRight();
+ bool searchGraphForNiceThings(int direction);
+ bool checkBubblePatterns();
+
+ bool getOtherBestParent(Kmer*kmer);
+ bool getOtherBestChild(Kmer*kmer);
+ bool getBestChild(Kmer*kmer);
+ bool getBestParent(Kmer*kmer);
+*/
+ Kmer m_first;
+ Kmer m_last;
+public:
+ void work();
+
+ bool isDone();
+
+ WorkerHandle getWorkerIdentifier();
+
+ void initialize(uint64_t identifier, GraphPath*seed, Parameters * parameters,
+ VirtualCommunicator * virtualCommunicator,
+ RingAllocator * outboxAllocator,
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE, MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH
+ );
+
+ bool isValid();
+
+ vector<GraphSearchResult> & getResults();
+};
+
+#endif
diff --git a/code/SpuriousSeedAnnihilator/SeedFilteringWorkflow.cpp b/code/SpuriousSeedAnnihilator/SeedFilteringWorkflow.cpp
new file mode 100644
index 0000000..79ac028
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/SeedFilteringWorkflow.cpp
@@ -0,0 +1,138 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "SeedFilteringWorkflow.h"
+#include "AnnihilationWorker.h"
+
+/*
+ * Methods to implement for the TaskCreator interface.
+ *
+ * The TaskCreator stack is used in the handler
+ * RAY_SLAVE_MODE_FILTER_SEEDS.
+ */
+
+/** initialize the whole thing */
+void SeedFilteringWorkflow::initializeMethod(){
+
+ m_seedIndex=0;
+
+ m_states.resize(m_seeds->size());
+
+ for(int i=0;i<(int)m_states.size(); i++)
+ m_states[i] = true;
+
+#ifdef ASSERT
+ assert(m_states.size() == m_seeds->size());
+#endif
+
+ m_finished = 0 ;
+}
+
+/** finalize the whole thing */
+void SeedFilteringWorkflow::finalizeMethod(){
+
+ vector<GraphPath> newPaths;
+
+ for(int i=0;i< (int) m_seeds->size() ; i++){
+
+ if(!m_states[i])
+ continue;
+
+ newPaths.push_back(m_seeds->at(i));
+ }
+
+ (*m_seeds) = newPaths;
+
+ m_core->getSwitchMan()->closeSlaveModeLocally(m_core->getOutbox(),m_core->getRank());
+}
+
+/** has an unassigned task left to compute */
+bool SeedFilteringWorkflow::hasUnassignedTask(){
+
+ return m_seedIndex < (int) m_seeds->size () ;
+}
+
+/** assign the next task to a worker and return this worker */
+Worker*SeedFilteringWorkflow::assignNextTask(){
+
+#ifdef ASSERT
+ assert(m_seedIndex < (int)m_seeds->size());
+#endif
+
+ if(m_seedIndex % m_period == 0)
+ cout<<"Rank "<<m_rank<< " assignNextTask "<<m_seedIndex<< "/" << m_seeds->size()<<endl;
+
+ AnnihilationWorker*worker=new AnnihilationWorker();
+
+ worker->initialize(m_seedIndex, &((*m_seeds)[m_seedIndex]), m_parameters,
+ m_virtualCommunicator,
+ m_core->getOutboxAllocator(),
+ RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ RAY_MPI_TAG_ASK_VERTEX_PATH
+ );
+
+ m_seedIndex++;
+
+ return worker;
+}
+
+/** get the result of a worker */
+void SeedFilteringWorkflow::processWorkerResult(Worker*worker){
+
+ bool result = ((AnnihilationWorker*)worker)->isValid();
+
+ int seedIndex = worker->getWorkerIdentifier();
+
+ m_states[seedIndex] = result;
+
+ if(m_finished % m_period == 0){
+ cout<<"Rank " << m_rank << " processWorkerResult "<<m_finished<<"/" <<m_seeds->size()<<endl;
+ }
+
+ m_finished++;
+}
+
+/** destroy a worker */
+void SeedFilteringWorkflow::destroyWorker(Worker*worker){
+
+ delete worker;
+ worker = NULL;
+}
+
+void SeedFilteringWorkflow::initialize(vector<GraphPath>*seeds, VirtualCommunicator*virtualCommunicator,
+ VirtualProcessor * virtualProcessor,ComputeCore * core, Parameters * parameters,
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH){
+
+ m_rank = core->getRank();
+ m_seeds = seeds;
+ m_virtualCommunicator = virtualCommunicator;
+ m_virtualProcessor = virtualProcessor;
+ m_core = core;
+ m_parameters = parameters;
+
+ this->RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT = RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+ this->RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE = RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ this->RAY_MPI_TAG_ASK_VERTEX_PATH = RAY_MPI_TAG_ASK_VERTEX_PATH;
+
+ m_period = 100;
+}
diff --git a/code/SpuriousSeedAnnihilator/SeedFilteringWorkflow.h b/code/SpuriousSeedAnnihilator/SeedFilteringWorkflow.h
new file mode 100644
index 0000000..a5c57cb
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/SeedFilteringWorkflow.h
@@ -0,0 +1,94 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef _SeedFilteringWorkflow_h
+#define _SeedFilteringWorkflow_h
+
+#include <code/SeedingData/GraphPath.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/scheduling/TaskCreator.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+
+#include <vector>
+using namespace std;
+
+/**
+ * This is the workflow that will be deposited on the
+ * virtual processor.
+ *
+ * \author Sébastien Boisvert
+ */
+class SeedFilteringWorkflow: public TaskCreator {
+
+ ComputeCore*m_core;
+ int m_seedIndex;
+ Parameters * m_parameters;
+ vector<GraphPath>*m_seeds;
+ vector<bool> m_states;
+
+ int m_finished;
+ int m_rank;
+ int m_period;
+
+/* TODO: maybe this should be in the TaskCreator */
+ VirtualCommunicator * m_virtualCommunicator;
+
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH;
+
+public:
+
+/*
+ * Methods to implement for the TaskCreator interface.
+ *
+ * The TaskCreator stack is used in the handler
+ * RAY_SLAVE_MODE_FILTER_SEEDS.
+ */
+
+ /** initialize the whole thing */
+ void initializeMethod();
+
+ /** finalize the whole thing */
+ void finalizeMethod();
+
+ /** has an unassigned task left to compute */
+ bool hasUnassignedTask();
+
+ /** assign the next task to a worker and return this worker */
+ Worker*assignNextTask();
+
+ /** get the result of a worker */
+ void processWorkerResult(Worker*worker);
+
+ /** destroy a worker */
+ void destroyWorker(Worker*worker);
+
+ void initialize(vector<GraphPath>*seeds, VirtualCommunicator*virtualCommunicator,
+ VirtualProcessor * virtualProcessor, ComputeCore * core,
+ Parameters * parameters, MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH
+ );
+};
+
+#endif
diff --git a/code/SpuriousSeedAnnihilator/SeedGossipSolver.cpp b/code/SpuriousSeedAnnihilator/SeedGossipSolver.cpp
new file mode 100644
index 0000000..4c618db
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/SeedGossipSolver.cpp
@@ -0,0 +1,371 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "SeedGossipSolver.h"
+
+#include <iostream>
+using namespace std;
+
+//#define DEBUG_MODE
+
+SeedGossipSolver::SeedGossipSolver() {
+
+}
+
+void SeedGossipSolver::setInput(vector<GraphSearchResult> * gossips) {
+ this->m_gossips = gossips;
+
+ for(int i = 0 ; i < (int)m_gossips->size() ; ++i) {
+ GraphSearchResult & element = m_gossips->at(i);
+ PathHandle & path1 = element.getPathHandles()[0];
+ PathHandle & path2 = element.getPathHandles()[1];
+
+ m_index[path1].push_back(i);
+
+ if(path1 != path2)
+ m_index[path2].push_back(i);
+
+#ifdef CONFIG_ASSERT
+ assert(path1 != path2);
+#endif ///////////// CONFIG_ASSERT
+
+ }
+}
+
+/**
+ * scoping is everything
+ */
+void SeedGossipSolver::compute() {
+/*
+ cout << "[DEBUG] computing solution..." << endl;
+ cout << "[DEBUG] index size " << m_index.size() << endl;
+ */
+
+
+ while(hasFreeGossip()) {
+
+ GraphSearchResult & entry = getFreeGossip();
+
+ GraphSearchResult workingCopy = entry;
+
+ while(expand(workingCopy)) {
+ }
+
+ m_solution.push_back(workingCopy);
+ }
+
+ printSolution();
+}
+
+void SeedGossipSolver::printSolution() {
+
+ //cout << "DEBUG [printSolution]" << endl;
+
+#if 0
+ cout << "Solution has: " << m_solution.size() << " entries" << endl;
+ cout << "Initial gossips: " << m_gossips->size() << endl;
+ cout << "Objects: " << m_index.size() << endl;
+
+ cout << "edges: " << endl;
+#endif
+
+
+#if 0
+ int iterator = 0;
+ for(vector<GraphSearchResult>::iterator i = m_gossips->begin();
+ i != m_gossips->end() ; ++i) {
+ cout << "[DEBUG] gossip:" << iterator++ << " ";
+ (*i).print();
+
+ cout << endl;
+ }
+#endif
+
+ int objectsInSolution = 0;
+
+ map<PathHandle, vector<int> > summary;
+
+ //cout << "solution" << endl;
+
+ for(int i = 0 ; i < (int) m_solution.size() ; ++i) {
+ //cout << "[DEBUG] @" << i << " ";
+
+ int total = m_solution[i].getPathHandles().size();
+
+#if 0
+ cout << total << " ... ";
+ m_solution[i].print();
+ cout << endl;
+#endif
+
+ objectsInSolution += total;
+
+ vector<PathHandle> & handles = m_solution[i].getPathHandles();
+
+ for(vector<PathHandle>::iterator j = handles.begin() ;
+ j!= handles.end() ; ++j) {
+
+ PathHandle & handle = *j;
+ summary[handle].push_back(i);
+ }
+ }
+
+ //cout << "[DEBUG] list of GraphSearchResult entries" << endl;
+
+#if 0
+ for(map<PathHandle, vector<int> >::iterator i = summary.begin();
+ i!= summary.end() ; ++i) {
+
+ vector<int> & matches = i->second;
+
+ if(matches.size() != 1) {
+ cout << "[DEBUG] " << i->first << " is in ";
+
+ for(vector<int>::iterator j = matches.begin();
+ j != matches.end() ; ++j){
+ cout << " " << *j;
+ }
+ cout << endl;
+ cout << "[DEBUG] first gossip that contains the path: ";
+ m_solution.at(matches[0]).print();
+ cout << endl;
+ }
+ }
+#endif
+
+#if 0
+ //cout << "[DEBUG] objects found in solution ...... " << endl;
+ cout << objectsInSolution << endl;
+#endif
+}
+
+/**
+ * TODO implement this method
+ */
+bool SeedGossipSolver::expand(GraphSearchResult & partialSolution) {
+
+ // take the two paths, and look for them in the index
+
+ vector<PathHandle> & handles = partialSolution.getPathHandles();
+ vector<bool> & orientations = partialSolution.getPathOrientations();
+
+#ifdef CONFIG_ASSERT
+ assert(handles.size() >= 2);
+#endif
+
+ int leftIndex = 0;
+ PathHandle & leftPath = handles[leftIndex];
+ bool leftStrand = orientations[leftIndex];
+
+ /**
+ *
+ * add on the left.
+ *
+ * ??? *********--*********--**********
+ *
+ */
+ if(m_index.count(leftPath) > 0) {
+ vector<int> & positions = m_index[leftPath];
+
+ for(int i = 0 ; i < (int) positions.size() ; ++i) {
+ int index = positions[i];
+ if(m_processedEntries.count(index) > 0)
+ continue;
+
+ //cout << "[DEBUG] found a edge on the left" << endl;
+
+ GraphSearchResult & edge = m_gossips->at(index);
+
+ PathHandle & leftGossipPath = edge.getPathHandles()[0];
+ PathHandle & rightGossipPath = edge.getPathHandles()[1];
+
+ bool leftGossipStrand = edge.getPathOrientations()[0];
+ bool rightGossipStrand = edge.getPathOrientations()[1];
+
+ // Case 1.1
+ // easy match
+ //
+ // edge
+ //
+ // ******---*******
+ // pathx
+ // strandx
+ //
+ // partialSolution
+ // ********---********---**********
+ // pathx
+ // strandx
+
+ if(rightGossipStrand == leftStrand
+ && rightGossipPath == leftPath) {
+
+ //cout << "[DEBUG] Case 1.1 direct match on the left" << endl;
+
+ GraphPath & path = edge.getComputedPaths()[0];
+
+ partialSolution.addPathOnLeftSide(leftGossipPath, leftGossipStrand, path);
+ m_processedEntries.insert(index);
+
+ return true;
+
+ // Case 1.2
+ // Same as Case 1.1, but we need to rotate the edge.
+ } else if(leftGossipStrand == !leftStrand
+ && leftGossipPath == leftPath) {
+
+ GraphSearchResult rotatedEdge = edge;
+ rotatedEdge.reverseContent();
+
+#ifdef CONFIG_ASSERT
+ assert(rotatedEdge.getPathOrientations()[1] == leftStrand);
+#endif
+
+ //cout << "[DEBUG] Case 1.2 reverse match on the left" << endl;
+
+ partialSolution.addPathOnLeftSide(rotatedEdge.getPathHandles()[0],
+ rotatedEdge.getPathOrientations()[0],
+ rotatedEdge.getComputedPaths()[0]);
+ m_processedEntries.insert(index);
+
+ return true;
+ }
+ }
+
+ }
+
+ int rightIndex = handles.size() - 1;
+ PathHandle & rightPath = handles[rightIndex];
+ bool rightStrand = orientations[rightIndex];
+
+ /**
+ *
+ * *********--*********--**********--- ???????
+ *
+ */
+ if(m_index.count(rightPath) > 0) {
+ vector<int> & positions = m_index[rightPath];
+
+ for(int i = 0 ; i < (int) positions.size() ; ++i) {
+ int index = positions[i];
+ if(m_processedEntries.count(index) > 0)
+ continue;
+
+ //cout << "[DEBUG] found a edge on the right" << endl;
+
+ GraphSearchResult & edge = m_gossips->at(index);
+
+ PathHandle & leftGossipPath = edge.getPathHandles()[0];
+ PathHandle & rightGossipPath = edge.getPathHandles()[1];
+
+ bool leftGossipStrand = edge.getPathOrientations()[0];
+ bool rightGossipStrand = edge.getPathOrientations()[1];
+
+ // Case 2.1
+ // easy match
+ //
+ // edge
+ //
+ // ******---*******
+ // pathx
+ // strandx
+ //
+ // partialSolution
+ // ********---********---**********
+ // pathx
+ // strandx
+ if(leftGossipStrand == rightStrand
+ && leftGossipPath == rightPath) {
+
+ //cout << "[DEBUG] Case 2.1 direct match on the right" << endl;
+
+ GraphPath & path = edge.getComputedPaths()[0];
+ partialSolution.addPathOnRightSide(rightGossipPath, rightGossipStrand, path);
+ m_processedEntries.insert(index);
+
+ return true;
+
+ // Case 2.2
+ // Same as Case 2.1, but we need to rotate the edge.
+ } else if(rightGossipStrand == !rightStrand
+ && rightGossipPath == rightPath) {
+
+ GraphSearchResult rotatedEdge = edge;
+ rotatedEdge.reverseContent();
+
+#ifdef CONFIG_ASSERT
+ assert(rotatedEdge.getPathOrientations()[0] == rightStrand);
+ assert(rotatedEdge.getPathHandles()[0] == rightPath);
+#endif
+
+ //cout << "[DEBUG] Case 2.2 reverse match on the right" << endl;
+
+ partialSolution.addPathOnRightSide(rotatedEdge.getPathHandles()[1],
+ rotatedEdge.getPathOrientations()[1],
+ rotatedEdge.getComputedPaths()[0]);
+ m_processedEntries.insert(index);
+
+ return true;
+ }
+ }
+
+ }
+
+ return false;
+}
+
+bool SeedGossipSolver::hasFreeGossip() const {
+ return m_processedEntries.size() != m_gossips->size();
+}
+
+GraphSearchResult & SeedGossipSolver::getFreeGossip() {
+ for(int i = 0 ; i < (int) m_gossips->size() ; ++i) {
+ bool used = m_processedEntries.count(i) > 0;
+
+ if(used)
+ continue;
+
+ GraphSearchResult & entry = m_gossips->at(i);
+
+#ifdef CONFIG_ASSERT
+ assert(m_processedEntries.count(i) == 0);
+#endif ////////// CONFIG_ASSERT
+
+ m_processedEntries.insert(i);
+
+ return entry;
+ }
+
+ return m_dummy;
+}
+
+vector<GraphSearchResult> & SeedGossipSolver::getSolution() {
+ return m_solution;
+}
+
+void SeedGossipSolver::destroy() {
+ m_solution.clear();
+ m_index.clear();
+ m_gossips = NULL;
+ m_processedEntries.clear();
+}
+
+bool SeedGossipSolver::hasPathHandle(PathHandle & pathHandle) {
+ return m_index.count(pathHandle) > 0;
+}
diff --git a/code/SpuriousSeedAnnihilator/SeedGossipSolver.h b/code/SpuriousSeedAnnihilator/SeedGossipSolver.h
new file mode 100644
index 0000000..efd1392
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/SeedGossipSolver.h
@@ -0,0 +1,64 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef SeedGossipSolverHeader
+#define SeedGossipSolverHeader
+
+#include "GraphSearchResult.h"
+
+#include <code/SeedingData/PathHandle.h>
+
+#include <map>
+#include <set>
+#include <vector>
+using namespace std;
+
+/**
+ *
+ * This is a solver for seed gossips.
+ *
+ * \author Sébastien Boisvert
+ */
+class SeedGossipSolver {
+
+ map<PathHandle, vector<int> > m_index;
+ vector<GraphSearchResult> * m_gossips;
+ vector<GraphSearchResult> m_solution;
+ set<int> m_processedEntries;
+ GraphSearchResult m_dummy;
+
+ bool expand(GraphSearchResult & partialSolution);
+
+ bool hasFreeGossip() const;
+ GraphSearchResult & getFreeGossip();
+
+ void printSolution();
+
+public:
+
+ SeedGossipSolver();
+ void setInput(vector<GraphSearchResult> * gossips);
+ void compute();
+ vector<GraphSearchResult> & getSolution();
+ void destroy();
+ bool hasPathHandle(PathHandle & pathHandle);
+};
+
+#endif ////////////// SeedGossipSolverHeader
diff --git a/code/SpuriousSeedAnnihilator/SeedMergingWorkflow.cpp b/code/SpuriousSeedAnnihilator/SeedMergingWorkflow.cpp
new file mode 100644
index 0000000..af1328b
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/SeedMergingWorkflow.cpp
@@ -0,0 +1,169 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#include "SeedMergingWorkflow.h"
+#include "NanoMerger.h"
+
+/*
+ * Methods to implement for the TaskCreator interface.
+ *
+ * The TaskCreator stack is used in the handler
+ * RAY_SLAVE_MODE_FILTER_SEEDS.
+ */
+
+/** initialize the whole thing */
+void SeedMergingWorkflow::initializeMethod(){
+
+ m_seedIndex=0;
+
+ m_finished = 0 ;
+}
+
+/** finalize the whole thing */
+void SeedMergingWorkflow::finalizeMethod(){
+
+ //cout << "[DEBUG] number of relations: " << m_searchResults.size() << endl;
+
+ m_core->getSwitchMan()->closeSlaveModeLocally(m_core->getOutbox(),m_core->getRank());
+}
+
+/** has an unassigned task left to compute */
+bool SeedMergingWorkflow::hasUnassignedTask(){
+
+ /*
+ * This code is still buggy. I will do the 2.3.0 release
+ * without it.
+ */
+ bool enableSeedMerging = m_parameters->hasOption("-merge-seeds");
+
+ if(!enableSeedMerging)
+ return false;
+
+ return m_seedIndex < (int) m_seeds->size () ;
+}
+
+/** assign the next task to a worker and return this worker */
+Worker*SeedMergingWorkflow::assignNextTask(){
+
+#ifdef ASSERT
+ assert(m_seedIndex < (int)m_seeds->size());
+#endif
+
+ if(m_seedIndex % m_period == 0)
+ cout<<"Rank "<<m_rank<< " assignNextTask "<<m_seedIndex<< "/" << m_seeds->size()<<endl;
+
+ NanoMerger*worker = new NanoMerger();
+
+ worker->initialize(m_seedIndex, &((*m_seeds)[m_seedIndex]), m_parameters,
+ m_virtualCommunicator,
+ m_core->getOutboxAllocator(),
+ RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ RAY_MPI_TAG_ASK_VERTEX_PATH
+ );
+
+ m_seedIndex++;
+
+ return worker;
+}
+
+/** get the result of a worker */
+void SeedMergingWorkflow::processWorkerResult(Worker*worker){
+
+ NanoMerger * theWorker = (NanoMerger*)worker;
+
+ vector<GraphSearchResult> & results = theWorker->getResults();
+
+ bool runTransaction = true;
+
+ if(results.size() == 2) {
+
+ // make sure this is not something like
+ //
+ // A001 --------B
+ // | /
+ // A100 --------
+ //
+ // with repeats, both ends of A could link to B...
+
+ set<PathHandle> observations;
+
+ for(int i = 0 ; i < (int) results.size() ; i++) {
+ GraphSearchResult & result = results[i];
+
+ observations.insert(result.getPathHandles()[0]);
+ observations.insert(result.getPathHandles()[1]);
+ }
+
+ if(observations.size() != 3)
+ runTransaction = false;
+ }
+
+ // we must send the results now !!!A
+ // we will send at most 2 messages...
+ // here we don't send messages right away because these units are
+ // not regular.
+
+ for(int i = 0 ; i < (int) results.size() ; i ++) {
+ if(!runTransaction)
+ break;
+ m_searchResults.push_back(results[i]);
+ }
+
+ if(m_finished % m_period == 0){
+ cout<<"Rank " << m_rank << " processWorkerResult "<<m_finished<<"/" <<m_seeds->size()<<endl;
+ }
+
+ m_finished++;
+}
+
+/** destroy a worker */
+void SeedMergingWorkflow::destroyWorker(Worker*worker){
+
+ NanoMerger * concreteWorker = (NanoMerger*)worker;
+ delete concreteWorker;
+ worker = NULL; // this does nothing useful
+ concreteWorker = NULL;
+}
+
+void SeedMergingWorkflow::initialize(vector<GraphPath>*seeds, VirtualCommunicator*virtualCommunicator,
+ VirtualProcessor * virtualProcessor,ComputeCore * core, Parameters * parameters,
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH){
+
+ m_rank = core->getRank();
+ m_seeds = seeds;
+ m_virtualCommunicator = virtualCommunicator;
+ m_virtualProcessor = virtualProcessor;
+ m_core = core;
+ m_parameters = parameters;
+
+ this->RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT = RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+ this->RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE = RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ this->RAY_MPI_TAG_ASK_VERTEX_PATH = RAY_MPI_TAG_ASK_VERTEX_PATH;
+
+ m_period = 100;
+
+}
+
+vector<GraphSearchResult> & SeedMergingWorkflow::getResults() {
+ return m_searchResults;
+}
diff --git a/code/SpuriousSeedAnnihilator/SeedMergingWorkflow.h b/code/SpuriousSeedAnnihilator/SeedMergingWorkflow.h
new file mode 100644
index 0000000..4bbe15b
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/SeedMergingWorkflow.h
@@ -0,0 +1,103 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef _SeedMergingWorkflow_h
+#define _SeedMergingWorkflow_h
+
+#include "GraphSearchResult.h"
+
+#include <code/SeedingData/GraphPath.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/scheduling/TaskCreator.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+
+#include <vector>
+using namespace std;
+
+/**
+ *
+ * This is used to merge seeds.
+ *
+ * This class is derived directly from the SeedFilteringWorkflow.
+ *
+ * \author Sébastien Boisvert
+ */
+class SeedMergingWorkflow: public TaskCreator {
+
+ vector<GraphSearchResult> m_searchResults;
+ vector<GraphSearchResult> m_remoteResults;
+
+ ComputeCore*m_core;
+ int m_seedIndex;
+ Parameters * m_parameters;
+ vector<GraphPath>*m_seeds;
+ vector<bool> m_states;
+
+ int m_finished;
+ int m_rank;
+ int m_period;
+
+/* TODO: maybe this should be in the TaskCreator */
+ VirtualCommunicator * m_virtualCommunicator;
+
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH;
+
+public:
+
+/*
+ * Methods to implement for the TaskCreator interface.
+ *
+ * The TaskCreator stack is used in the handler
+ * RAY_SLAVE_MODE_FILTER_SEEDS.
+ */
+
+ /** initialize the whole thing */
+ void initializeMethod();
+
+ /** finalize the whole thing */
+ void finalizeMethod();
+
+ /** has an unassigned task left to compute */
+ bool hasUnassignedTask();
+
+ /** assign the next task to a worker and return this worker */
+ Worker*assignNextTask();
+
+ /** get the result of a worker */
+ void processWorkerResult(Worker*worker);
+
+ /** destroy a worker */
+ void destroyWorker(Worker*worker);
+
+ void initialize(vector<GraphPath>*seeds, VirtualCommunicator*virtualCommunicator,
+ VirtualProcessor * virtualProcessor, ComputeCore * core,
+ Parameters * parameters, MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH
+ );
+
+ vector<GraphSearchResult> & getResults();
+};
+
+#endif
diff --git a/code/SpuriousSeedAnnihilator/SpuriousSeedAnnihilator.cpp b/code/SpuriousSeedAnnihilator/SpuriousSeedAnnihilator.cpp
new file mode 100644
index 0000000..917e7db
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/SpuriousSeedAnnihilator.cpp
@@ -0,0 +1,2042 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+
+#include "SpuriousSeedAnnihilator.h"
+
+#include <code/VerticesExtractor/GridTableIterator.h>
+
+#include <RayPlatform/cryptography/crypto.h>
+
+#ifdef __unix__
+// gcc -dM -E - < /dev/null | less
+#include <unistd.h>
+
+#endif
+
+
+#include <sstream>
+#include <algorithm>
+using namespace std;
+
+__CreatePlugin(SpuriousSeedAnnihilator);
+
+__CreateMasterModeAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_REGISTER_SEEDS);
+__CreateMasterModeAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_FILTER_SEEDS);
+__CreateMasterModeAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_CLEAN_SEEDS);
+__CreateMasterModeAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_PROCESS_MERGING_ASSETS);
+__CreateMasterModeAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_PUSH_SEED_LENGTHS);
+__CreateMasterModeAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_MERGE_SEEDS);
+
+__CreateSlaveModeAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_REGISTER_SEEDS);
+__CreateSlaveModeAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_FILTER_SEEDS);
+__CreateSlaveModeAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_CLEAN_SEEDS);
+__CreateSlaveModeAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_PROCESS_MERGING_ASSETS);
+__CreateSlaveModeAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_PUSH_SEED_LENGTHS);
+__CreateSlaveModeAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_MERGE_SEEDS);
+
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_REGISTER_SEEDS);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_FILTER_SEEDS);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_CLEAN_SEEDS);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_PROCESS_MERGING_ASSETS);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_PUSH_SEED_LENGTHS);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SEND_SEED_LENGTHS);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_MERGE_SEEDS);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY_REPLY);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SAY_HELLO_TO_ARBITER);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_REQUEST_VERTEX_COVERAGE_WITH_POSITION);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SEED_GOSSIP);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SEED_GOSSIP_REPLY);
+__CreateMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_ARBITER_SIGNAL);
+
+SpuriousSeedAnnihilator::SpuriousSeedAnnihilator(){
+}
+
+/**
+ * Even if checkpoints exist, we don't skip this code path.
+ */
+void SpuriousSeedAnnihilator::call_RAY_SLAVE_MODE_PUSH_SEED_LENGTHS(){
+ if(!m_initialized){
+
+ vector<GraphPath> newSeeds;
+
+ for(int i=0;i<(int)(*m_seeds).size();i++){
+
+ if((*m_seeds)[i].isDeleted())
+ continue;
+
+ newSeeds.push_back((*m_seeds)[i]);
+ }
+
+ (*m_seeds) = newSeeds;
+
+ for(int i=0;i<(int)(*m_seeds).size();i++){
+
+ int length=getNumberOfNucleotides((*m_seeds)[i].size(),
+ m_parameters->getWordSize());
+ m_slaveSeedLengths[length]++;
+ }
+ m_iterator=m_slaveSeedLengths.begin();
+ m_initialized=true;
+ m_communicatorWasTriggered=false;
+
+
+ m_virtualCommunicator->resetCounters();
+ }
+
+ if(m_inbox->size()==1&&(*m_inbox)[0]->getTag()==RAY_MESSAGE_TAG_SEND_SEED_LENGTHS_REPLY)
+ m_communicatorWasTriggered=false;
+
+ if(m_communicatorWasTriggered)
+ return;
+
+ if(m_iterator==m_slaveSeedLengths.end()){
+
+ writeCheckpointForSeeds();
+
+ writeSingleSeedFile();
+
+ m_core->closeSlaveModeLocally();
+
+ return;
+ }
+
+ MessageUnit*messageBuffer=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+ int maximumPairs=MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit)/2;
+
+ int i=0;
+
+ //cout << "[DEBUG] call_RAY_SLAVE_MODE_PUSH_SEED_LENGTHS preparing payload." << endl;
+
+ while(i<maximumPairs && m_iterator!=m_slaveSeedLengths.end()){
+ int length=m_iterator->first;
+ int count=m_iterator->second;
+ messageBuffer[2*i]=length;
+ messageBuffer[2*i+1]=count;
+ i++;
+ m_iterator++;
+ }
+
+ Message aMessage(messageBuffer,2*i,MASTER_RANK,
+ RAY_MESSAGE_TAG_SEND_SEED_LENGTHS, m_rank);
+ m_outbox->push_back(&aMessage);
+}
+
+ComputeCore * SpuriousSeedAnnihilator::getCore() {
+ return m_core;
+}
+
+SpuriousSeedAnnihilator * SpuriousSeedAnnihilator::getThis() {
+ return this;
+}
+
+SpuriousSeedAnnihilator * SpuriousSeedAnnihilator::getThat() {
+
+ SpuriousSeedAnnihilator * that = this;
+
+ return that;
+}
+
+/*
+ * handlers.
+ */
+void SpuriousSeedAnnihilator::call_RAY_MASTER_MODE_REGISTER_SEEDS(){
+
+ if(!m_distributionIsStarted){
+ m_core->getSwitchMan()->openMasterMode(m_core->getOutbox(),m_core->getRank());
+
+ m_distributionIsStarted=true;
+
+ }else if(m_core->getSwitchMan()->allRanksAreReady()){
+
+ //m_core->getSwitchMan()->closeMasterMode();
+
+ if(!m_filteredSeeds) {
+
+ this->m_core->getSwitchMan()->setMasterMode(RAY_MASTER_MODE_FILTER_SEEDS);
+
+ m_filteredSeeds = true;
+ } else if(!m_mergedSeeds) {
+
+ this->m_core->getSwitchMan()->setMasterMode(RAY_MASTER_MODE_MERGE_SEEDS);
+
+ m_mergedSeeds = true;
+ }
+
+ m_distributionIsStarted = false;
+ }
+}
+
+void SpuriousSeedAnnihilator::writeSingleSeedFile(){
+
+ /** write seeds for debugging purposes */
+ if(m_parameters->hasOption("-write-seeds")){
+ ostringstream fileName;
+ fileName<<m_parameters->getPrefix()<<"Rank"<<m_parameters->getRank()<<".RaySeeds.fasta";
+ ofstream f(fileName.str().c_str());
+
+ for(int i=0;i<(int)(*m_seeds).size();i++){
+ PathHandle id=getPathUniqueId(m_parameters->getRank(),i);
+
+ f<<">RaySeed-"<<id<<endl;
+
+ f<<addLineBreaks(convertToString(&((*m_seeds)[i]),
+ m_parameters->getWordSize(),m_parameters->getColorSpaceMode()),
+ m_parameters->getColumns());
+ }
+ f.close();
+ }
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MASTER_MODE_PUSH_SEED_LENGTHS(){
+
+ if(!m_gatheringHasStarted){
+ m_core->getSwitchMan()->openMasterMode(m_core->getOutbox(),m_core->getRank());
+
+ m_gatheringHasStarted=true;
+
+ }else if(m_core->getSwitchMan()->allRanksAreReady()){
+
+ writeSeedStatistics();
+
+ m_core->getSwitchMan()->closeMasterMode();
+ }
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MASTER_MODE_FILTER_SEEDS(){
+
+ if(!m_filteringIsStarted){
+ m_core->getSwitchMan()->openMasterMode(m_core->getOutbox(),m_core->getRank());
+
+ m_filteringIsStarted=true;
+
+ }else if(m_core->getSwitchMan()->allRanksAreReady()){
+ m_core->getSwitchMan()->closeMasterMode();
+ }
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_PROCESS_MERGING_ASSETS(Message * message) {
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MASTER_MODE_PROCESS_MERGING_ASSETS() {
+
+ if(!m_processingIsStarted){
+ m_core->getSwitchMan()->openMasterMode(m_core->getOutbox(), m_core->getRank());
+
+ m_processingIsStarted=true;
+
+ }else if(m_core->getSwitchMan()->allRanksAreReady()){
+
+ m_core->getSwitchMan()->closeMasterMode();
+ }
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY(Message * message) {
+
+ //cout << "[DEBUG] received RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY" << endl;
+
+ char * buffer = (char *) message->getBuffer();
+
+ GraphSearchResult entry;
+ entry.load(buffer);
+
+ m_mergingTechnology.getResults().push_back(entry);
+
+ m_core->getSwitchMan()->sendEmptyMessage(m_outbox, m_rank, message->getSource(), RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY_REPLY);
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY_REPLY(Message * message) {
+ m_messageWasReceived = true;
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_SAY_HELLO_TO_ARBITER(Message * message) {
+
+ initializeMergingProcess();
+
+ m_synced ++;
+
+#if 0
+ cout << "[DEBUG] recv RAY_MESSAGE_TAG_SAY_HELLO_TO_ARBITER m_synced " << m_synced << endl;
+#endif
+
+ if(m_synced == m_core->getSize()) {
+ //cout << "[DEBUG] arbiter will advise" << endl;
+
+ m_mustAdviseRanks = true;
+ m_rankToAdvise = 0;
+ }
+}
+
+void SpuriousSeedAnnihilator::initializeMergingProcess() {
+ if(!m_initializedProcessing) {
+
+#if 0
+ cout << "[DEBUG] initialize RAY_SLAVE_MODE_PROCESS_MERGING_ASSETS" << endl;
+#endif
+
+ m_entryIndex = 0;
+
+ int value = 0;
+ MODE_SPREAD_DATA = value++;
+ MODE_CHECK_RESULTS = value++;
+ MODE_STOP_THIS_SITUATION = value++;
+ MODE_GATHER_COVERAGE_VALUES = value++;
+ MODE_SHARE_WITH_LINKED_ACTORS = value ++;
+ MODE_WAIT_FOR_ARBITER = value++;
+ MODE_EVALUATE_GOSSIPS = value++;
+ MODE_REBUILD_SEED_ASSETS = value++;
+ MODE_SHARE_PUSH_DATA_IN_KEY_VALUE_STORE = value++;
+ MODE_GENERATE_NEW_SEEDS = value++;
+ MODE_CLEAN_KEY_VALUE_STORE = value++;
+
+ m_mode = MODE_SPREAD_DATA;
+
+ m_toDistribute = m_mergingTechnology.getResults().size();
+
+ m_messageWasSent = false;
+
+ m_initializedProcessing = true;
+ m_synced = 0;
+ m_mustAdviseRanks = false;
+
+ // we can not return here because we could lose a message
+ //return;
+ }
+}
+
+void SpuriousSeedAnnihilator::spreadAcquiredData() {
+
+ if(m_entryIndex < (int) m_toDistribute) {
+
+ if(!m_messageWasSent) {
+
+ //cout << "[DEBUG] MODE_SPREAD_DATA send " << m_entryIndex << endl;
+
+ char *messageBuffer=(char *)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+ GraphSearchResult & entry = m_mergingTechnology.getResults()[m_entryIndex];
+ int bytes = entry.dump(messageBuffer);
+
+ int elements = bytes / sizeof(MessageUnit);
+
+ if(bytes % sizeof(MessageUnit))
+ elements ++;
+
+ // at least one rank is the current rank
+ Rank rank1 = getRankFromPathUniqueId(entry.getPathHandles()[0]);
+ Rank rank2 = getRankFromPathUniqueId(entry.getPathHandles()[1]);
+
+ Rank destination = rank1;
+
+ if(destination == m_rank)
+ destination = rank2;
+
+ //cout << "[DEBUG] MODE_SPREAD_DATA destination " << destination << endl;
+
+ if(destination != m_rank) {
+ Message aMessage((MessageUnit*)messageBuffer, elements , destination, RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY, m_rank);
+ m_outbox->push_back(&aMessage);
+
+ m_messageWasReceived = false;
+
+ } else {
+ // nothing to send,
+ // the local rank has both
+
+ m_messageWasReceived = true;
+ }
+
+ m_messageWasSent = true;
+
+ } else if(m_messageWasReceived) {
+
+ //cout << "[DEBUG] MODE_SPREAD_DATA receive " << m_entryIndex << endl;
+
+ m_entryIndex ++;
+
+ m_messageWasSent = false;
+ }
+ } else {
+
+ sendMessageToArbiter();
+ m_nextMode = MODE_CHECK_RESULTS;
+ }
+
+}
+
+void SpuriousSeedAnnihilator::sendMessageToArbiter() {
+
+#ifdef DEBUG_SEED_MERGING
+ cout << "[DEBUG] Rank " << m_core->getRank() << " sendMessageToArbiter ";
+ cout << " m_mode = " << m_mode << endl;
+#endif
+
+ Rank arbiter = getArbiter();
+ this->m_core->getSwitchMan()->sendEmptyMessage(m_outbox, m_rank, arbiter, RAY_MESSAGE_TAG_SAY_HELLO_TO_ARBITER);
+
+ //cout << "[DEBUG] saying hello to arbiter " << arbiter << " with RAY_MESSAGE_TAG_SAY_HELLO_TO_ARBITER" << endl;
+
+ m_mode = MODE_WAIT_FOR_ARBITER;
+}
+
+void SpuriousSeedAnnihilator::checkResults() {
+ //cout << "[DEBUG] MODE_CHECK_RESULTS m_toDistribute " << m_toDistribute << " now -> " << m_mergingTechnology.getResults().size() << endl;
+ //cout << "[DEBUG] MODE_CHECK_RESULTS scanning for duplicates" << endl;
+
+ map<PathHandle, map<PathHandle, vector<int> > > counts;
+
+ for(int i = 0 ; i < (int)m_mergingTechnology.getResults().size() ; i++) {
+
+ // there were no local hits...
+ // so let it fail
+ if(m_toDistribute == 0)
+ break;
+
+ GraphSearchResult & result = m_mergingTechnology.getResults()[i];
+
+ PathHandle handle1 = result.getPathHandles()[0];
+ PathHandle handle2 = result.getPathHandles()[1];
+
+ if(handle2 < handle1) {
+ PathHandle store = handle1;
+ handle1 = handle2;
+ handle2 = store;
+ }
+
+ counts[handle1][handle2].push_back(i);
+ }
+
+ for(map<PathHandle, map<PathHandle, vector<int> > >::iterator i = counts.begin();
+ i != counts.end(); ++i) {
+
+ for(map<PathHandle, vector<int> >::iterator j = i->second.begin();
+ j != i->second.end() ; j++) {
+
+ if(j->second.size() == 2) {
+ //cout << "[DEBUG] MODE_CHECK_RESULTS got a symmetric relation between " << i->first;
+ //cout << " and " << j->first << endl;
+
+ //m_indexesToShareWithArbiter.push_back(j->second[0]);
+ //m_indexesToShareWithArbiter.push_back(j->second[1]);
+
+ int index = j->second[0];
+ GraphSearchResult & gossip = m_mergingTechnology.getResults()[index];
+
+ // it is not clear how the gossip could already be there...
+ if(m_gossipAssetManager.hasGossip(gossip))
+ continue;
+
+ m_gossipAssetManager.addGossip(gossip);
+
+ // tell the manager that the current rank has its own copy already !
+
+ m_gossipAssetManager.registerRemoteGossip(gossip, m_rank);
+
+#if 0
+ // at least one rank is the current rank
+ Rank rank1 = getRankFromPathUniqueId(gossip.getPathHandles()[0]);
+ Rank rank2 = getRankFromPathUniqueId(gossip.getPathHandles()[1]);
+#endif
+
+ }
+ }
+
+ }
+
+ m_mergingTechnology.getResults().clear();
+
+ m_hasNewGossips = true;
+ m_lastGossipingEventTime = time(NULL);
+
+ //cout << "[DEBUG] MODE_CHECK_RESULTS gossiping begins..." << endl;
+
+ m_messagesSentForGossiping = 0;
+
+ m_initialGossips = m_gossipAssetManager.getGossips().size();
+
+ m_messageWasSentToArbiter = false;
+
+ // this is a barrier
+
+ bool enableBug = false;
+ int mode = MODE_SHARE_WITH_LINKED_ACTORS;
+
+ if(!enableBug) {
+ sendMessageToArbiter();
+ m_nextMode = mode;
+
+ } else {
+ if(m_core->getRank() == m_core->getSize() - 1) {
+
+#ifdef __unix__
+ int seconds = 30;
+ cout << "DEBUG sleeping ";
+ cout << seconds << " seconds to trigger bug" << endl;
+ sleep(seconds);
+#endif
+ }
+
+ m_mode = mode;
+
+ }
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_SEED_GOSSIP(Message * message) {
+
+ //Message * message = m_inbox->at(0);
+
+ if(m_mode != MODE_SHARE_WITH_LINKED_ACTORS) {
+
+ cout << "Error: received RAY_MESSAGE_TAG_SEED_GOSSIP outside ";
+ cout << "MODE_SHARE_WITH_LINKED_ACTORS m_mode = ";
+ cout << m_mode << endl;
+ }
+
+ // send the response now because the return
+ // statements below are evil.
+ // anyway, RayPlatform will only send the message in
+ // its sendMessages() call in its main loop so
+ // it does not change anything whatsoever.
+
+ this->m_core->getSwitchMan()->sendEmptyMessage(m_outbox, m_rank, message->getSource(),
+ RAY_MESSAGE_TAG_SEED_GOSSIP_REPLY);
+
+ int availableUnits = message->getCount();
+
+ // if the message is empty, we have nothing to do.
+ // this is likely because the object was too large
+ // to be transported.
+ if(availableUnits == 0) {
+ return;
+ }
+
+ GraphSearchResult gossip;
+ int position = 0;
+ char * buffer = (char *) message->getBuffer();
+
+ int bytes = gossip.load(buffer + position);
+ position += bytes;
+
+ // first check if the gossip is already known.
+ // this implementation is good enough, but it could
+ // use an index.
+
+ //string key = gossip.toString();
+
+ bool found = m_gossipAssetManager.hasGossip(gossip);
+
+ if(found) {
+ return;
+ }
+
+
+ // yay we got new gossip to share !!!
+ m_gossipAssetManager.addGossip(gossip);
+
+ m_lastGossipingEventTime = time(NULL);
+
+ // we could also update the m_gossipStatus
+ // because we know that message->getSource() has this gossip
+ // already.
+ // But anyway, it does not change much to the algorithm...
+ // update: source has gossip already.
+ //
+ // also important, gossipIndex values are local and not global to
+ // all ranks
+ //
+
+
+ Rank actor = message->getSource();
+
+#ifdef CONFIG_ASSERT
+ vector<GraphSearchResult> & gossips = m_gossipAssetManager.getGossips();
+ int gossipIndex = gossips.size() - 1;
+ assert(gossipIndex < (int)gossips.size());
+ assert(actor >= 0);
+ assert(actor < m_core->getSize());
+#endif
+
+ // register the remote copy to avoid sending it back ot this source
+ // since the information came from there in the first place, at least
+ // for the current rank
+ m_gossipAssetManager.registerRemoteGossip(gossip, actor);
+
+ // also, advise the asset manager that the current rank has its own
+ // copy too !
+ //
+
+ m_gossipAssetManager.registerRemoteGossip(gossip, m_rank);
+
+#ifdef CONFIG_ASSERT
+ assert(m_core->getRank() == m_rank);
+ assert(m_core->getRank() == message->getDestination());
+#endif // CONFIG_ASSERT
+
+ //m_gossipStatus[gossipIndex].insert(actor);
+
+ // we have new gossip, so this is important to store.
+ m_hasNewGossips = true;
+
+ //cout << "[DEBUG] MODE_SHARE_WITH_LINKED_ACTORS Rank rank:" << m_rank << " received gossip gossip:" << key << " from rank rank:" << actor << endl;
+
+ //return;
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_SEED_GOSSIP_REPLY(Message * message) {
+
+ m_activeQueries --;
+
+#ifdef CONFIG_ASSERT
+ assert(m_activeQueries >= 0);
+#endif // CONFIG_ASSERT
+
+ m_lastGossipingEventTime = time(NULL);
+}
+
+
+void SpuriousSeedAnnihilator::shareWithLinkedActors() {
+
+ // find a information that was not shared with a given actor
+ // this is a good-enough implementation
+ // this is like a gossip algorithm, but here the pairs are not randomly
+ // selected, they are obtained from the relationships computed
+ // in the de Bruijn graph.
+
+ /*
+ m_mode = MODE_STOP_THIS_SITUATION; // remove this
+ return; // remove this
+ */
+
+ /*
+ * Add a messaging regulator here.
+ */
+ int maximumActiveQueries =16;
+
+ if(m_core->getSize() / 2 < maximumActiveQueries)
+ maximumActiveQueries = m_core->getSize() / 2;
+
+ if(maximumActiveQueries < 2)
+ maximumActiveQueries = 2;
+
+ // for correctness, use 1
+ maximumActiveQueries = 1;
+
+ /**
+ *
+ * We wait a specific amount of time before stopping the
+ * gossip process.
+ *
+ * This code will fail if the process receives a SIGSTOP
+ *
+ * TODO or if the process is not scheduled during too many seconds.
+ */
+
+ if(!m_hasNewGossips && !m_messageWasSentToArbiter) {
+
+ // we don't care about the response because
+ // the rank does not have anything to send...
+ //if(m_activeQueries > 0)
+ //m_activeQueries = 0;
+
+ if(m_activeQueries > 0)
+ return;
+
+ // check delay and latency
+ time_t currentTime = time(NULL);
+
+ int minimumWaitTime = 10; // seconds
+ int distance = currentTime - m_lastGossipingEventTime;
+
+ if(distance < minimumWaitTime)
+ return;
+
+ //cout << "[DEBUG] MODE_SHARE_WITH_LINKED_ACTORS Rank " << m_rank << " gossips have spreaded." << endl;
+
+ //m_gossipStatus.clear();
+
+ // synchronize indexes in m_indexesToShareWithArbiter
+ //m_mode = MODE_EVALUATE_GOSSIPS;
+ //m_mode = MODE_SHARE_PUSH_DATA_IN_KEY_VALUE_STORE;
+
+ cout << "Rank " << m_core->getRank() << ": gossiping generated " << m_messagesSentForGossiping;
+ cout << " messages";
+ cout << " (gossips: " << m_initialGossips;
+ cout << " ---> " << m_gossipAssetManager.getGossips().size();
+ cout << ")" << endl;
+
+ int currentMode = m_mode;
+ sendMessageToArbiter();
+
+ // remain in the current mode to be able to send the messages correctly
+ // for incoming new gossips
+ m_mode = currentMode;
+ m_nextMode = MODE_SHARE_PUSH_DATA_IN_KEY_VALUE_STORE;
+
+ //cout << "DEBUG call sendMessageToArbiter for m_messageWasSentToArbiter" << endl;
+
+ m_messageWasSentToArbiter = true;
+
+ return;
+ }
+
+ // at this point, we have messages of type
+ // RAY_MESSAGE_TAG_SEED_GOSSIP to send.
+
+ // \\ messaging regulator
+ if(m_activeQueries >= maximumActiveQueries)
+ return;
+
+ // pick up a gossip and throw it at someone !
+ if(m_gossipAssetManager.hasGossipToShare()) {
+
+ GraphSearchResult dummyGossip;
+ Rank destination = 0;
+
+ m_gossipAssetManager.getGossipToShare(dummyGossip, destination);
+
+ // we found a gossip that needs to be shared with actor <actor>
+
+ m_lastGossipingEventTime = time(NULL);
+
+ char *messageBuffer = (char *)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
+
+#ifdef ASSERT_CONFIG
+ assert( messageBuffer != NULL );
+#endif /* ASSERT_CONFIG */
+
+ int position = 0;
+ GraphSearchResult & gossip = dummyGossip;
+
+ Rank & actor = destination;
+
+ int requiredBytes = gossip.getRequiredNumberOfBytes();
+
+ int availableBytes = MAXIMUM_MESSAGE_SIZE_IN_BYTES - position;
+
+ if(availableBytes < requiredBytes) {
+ // TODO: these objects should be transported with the RayPlatform key-value store.
+ // the RayPlatform key-value store does not have any size limitation.
+ cout << "Warning: object is too large to be transported, skipping." << endl;
+ } else {
+ position += gossip.dump(messageBuffer + position);
+ }
+
+ int units = position / sizeof(MessageUnit);
+
+ if(position % sizeof(MessageUnit))
+ units ++;
+
+
+ Message aMessage((MessageUnit*)messageBuffer, units, actor,
+ RAY_MESSAGE_TAG_SEED_GOSSIP, m_rank);
+ m_outbox->push_back(&aMessage);
+
+ m_activeQueries ++;
+
+ // tell the asset manager that there is not a remote copy for this asset...
+ // we need to this becasue we don't want to send it back later
+ // in the execution to this remote actor.
+ m_gossipAssetManager.registerRemoteGossip(gossip, actor);
+
+ // m_gossipStatus[gossipIndex].insert(actor);
+
+#if 0
+ string key = gossip.toString();
+ cout << "[DEBUG] MODE_SHARE_WITH_LINKED_ACTORS Rank rank:" << m_rank << " sent gossip gossip:" << key << " from rank rank:" << actor << endl;
+#endif
+
+ m_hasNewGossips = true;
+
+ m_messagesSentForGossiping ++;
+
+ return;
+ }
+
+ // at this point, we tried every gossip and they are all synchronized.
+
+ m_hasNewGossips = false;
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_ARBITER_SIGNAL(Message * message) {
+
+#if 0
+ cout << "[DEBUG] arbiter advises to continue with m_mode <- ";
+ cout << m_nextMode << endl;
+#endif
+
+ m_mode = m_nextMode;
+}
+
+void SpuriousSeedAnnihilator::call_RAY_SLAVE_MODE_PROCESS_MERGING_ASSETS() {
+
+ initializeMergingProcess();
+
+ if(m_mustAdviseRanks) {
+
+#ifdef CONFIG_ASSERT
+ assert(m_rankToAdvise < m_core->getSize());
+#endif
+
+ // reset the reset count
+ // here because we may receive a notification
+ // during the advise session
+ if(m_rankToAdvise == 0) {
+ m_synced = 0;
+ }
+
+ //cout << "Arbiter advises " << m_rankToAdvise << endl;
+ this->m_core->getSwitchMan()->sendEmptyMessage(m_outbox, m_rank, m_rankToAdvise, RAY_MESSAGE_TAG_ARBITER_SIGNAL);
+ m_rankToAdvise ++;
+
+ if(m_rankToAdvise == m_core->getSize()) {
+ m_mustAdviseRanks = false;
+
+ //cout << "[DEBUG] everybody received the arbiter advise" << endl;
+ }
+ }
+
+ if (m_mode == MODE_SPREAD_DATA) {
+
+ spreadAcquiredData();
+
+ } else if(m_mode == MODE_WAIT_FOR_ARBITER) {
+
+ } else if(m_mode == MODE_CHECK_RESULTS) {
+
+ checkResults();
+
+ } else if(m_mode == MODE_SHARE_WITH_LINKED_ACTORS) {
+
+ shareWithLinkedActors();
+
+ } else if(m_mode == MODE_SHARE_PUSH_DATA_IN_KEY_VALUE_STORE) {
+
+ pushDataInKeyValueStore();
+
+ } else if(m_mode == MODE_EVALUATE_GOSSIPS) {
+
+ //cout << "DEBUG Calling evaluateGossips" << endl;
+ evaluateGossips();
+
+ } else if(m_mode == MODE_REBUILD_SEED_ASSETS) {
+
+ rebuildSeedAssets();
+
+ } else if(m_mode == MODE_GENERATE_NEW_SEEDS) {
+
+ generateNewSeeds();
+
+ } else if(m_mode == MODE_CLEAN_KEY_VALUE_STORE) {
+
+ cleanKeyValueStore();
+
+ } else if(m_mode == MODE_GATHER_COVERAGE_VALUES) {
+
+ gatherCoverageValues();
+
+ } else if(m_mode == MODE_STOP_THIS_SITUATION) {
+
+ m_core->closeSlaveModeLocally();
+ }
+
+}
+
+void SpuriousSeedAnnihilator::getSeedKey(PathHandle & handle, string & keyObject) {
+
+ ostringstream key;
+
+ // something like
+ //
+ // /seeds/1000012
+
+ key << "/seeds/" << handle;
+
+ keyObject = key.str();
+
+}
+
+void SpuriousSeedAnnihilator::pushDataInKeyValueStore() {
+
+ for(int i = 0 ; i < (int)m_seeds->size() ; ++i) {
+
+ GraphPath & seed = m_seeds->at(i);
+
+ PathHandle handle = getPathUniqueId(m_core->getRank(), i);
+
+ string keyObject;
+ getSeedKey(handle, keyObject);
+
+ // obviously content is just a test...
+
+ int bytes = seed.getRequiredNumberOfBytes();
+ char * content = m_core->getKeyValueStore().allocateMemory(bytes);
+ seed.dump(content);
+
+ if(!(m_core->getKeyValueStore().insertLocalKey(keyObject, content, bytes))) {
+
+ // failed to insert key
+ // we should catch the error...
+ //
+ // For example, it will return false if the key is already there.
+ }
+ }
+
+ sendMessageToArbiter();
+ m_nextMode = MODE_EVALUATE_GOSSIPS;
+
+ //cout << "DEBUG m_nextMode <- MODE_EVALUATE_GOSSIPS " << MODE_EVALUATE_GOSSIPS << endl;
+
+#ifdef DEBUG_SEED_MERGING
+ cout << "[DEBUG] wait before evaluating gossips..." << endl;
+#endif
+}
+
+void SpuriousSeedAnnihilator::rebuildSeedAssets() {
+
+ //cout << "DEBUG rebuildSeedAssets" << endl;
+
+ //string testKey = "/seeds/115000022";
+ //Rank testRank = 22;
+
+ /*
+ string testKey = "/seeds/19";
+ Rank testRank = 19;
+ */
+
+ /*
+ string testKey = "/seeds/60000001";
+ Rank testRank = 1;
+ */
+
+ if(!m_initialized) {
+
+ //cout << "DEBUG rebuildSeedAssets initializing..." << endl;
+
+ m_seedIndex = 0;
+ m_pathIndex = 0;
+ m_location = 0;
+
+ /*
+ * This was for testing purposes.
+ *
+ * The API of RayPlatform works great.
+ */
+ //m_core->getKeyValueStore().pullRemoteKey(testKey, testRank, m_request);
+
+ m_initialized = true;
+
+ /*
+ } else if(!m_core->getKeyValueStore().test(m_request)) {
+ */
+
+ } else if(m_seedIndex < (int)m_newSeedBluePrints.size()) {
+
+#if 0
+ m_seedIndex ++;
+ return;
+#endif
+
+ vector<PathHandle> & pathHandles = m_newSeedBluePrints[m_seedIndex].getPathHandles();
+
+
+ if(m_pathIndex < (int)pathHandles.size()) {
+
+ string key = "";
+ PathHandle & handle = pathHandles[m_pathIndex];
+ getSeedKey(handle, key);
+ Rank rank = getRankFromPathUniqueId(handle);
+
+ if(!m_messageWasSent) {
+
+#if 0
+ cout << "DEBUG downloading keys for this blueprint: " << endl;
+ m_newSeedBluePrints[m_seedIndex].print();
+ cout << endl;
+
+ cout << "[DEBUG] downloading key " << key << " from rank " << rank;
+ cout << endl;
+#endif
+
+ m_core->getKeyValueStore().pullRemoteKey(key, rank, m_request);
+
+ m_messageWasSent = true;
+
+ } else if(m_core->getKeyValueStore().test(m_request)) {
+
+ /*
+ * The transfer is completed.
+ */
+
+
+ char * value = NULL;
+ int size = 0;
+ m_core->getKeyValueStore().getLocalKey(key, value, size);
+
+#if 0
+ cout << "[DEBUG] transfer is now completed for key " << key << " ";
+ cout << " " << size << " bytes" << endl;
+#endif
+
+ m_pathIndex ++;
+ m_messageWasSent = false;
+ }
+
+ } else {
+ m_seedIndex++;
+
+ m_pathIndex = 0;
+ }
+
+ } else {
+
+#if 0
+ /**
+ *
+ * This code was useful to test the transport of
+ * data.
+ */
+ char * value = NULL;
+ int valueLength = 0;
+ m_core->getKeyValueStore().getLocalKey(testKey, &value, &valueLength);
+
+
+ cout << "[DEBUG] Rank " << m_core->getRank() << " downloaded key ";
+ cout << testKey << " from " << testRank;
+ cout << ", value length is " << valueLength << " bytes ";
+
+ cout << " CRC32= 0-3999 .. ";
+ cout << computeCyclicRedundancyCode32((uint8_t*) value, (uint32_t)4000);
+
+ cout << " CRC32= 4000- .. ";
+ cout << computeCyclicRedundancyCode32((uint8_t*) value + 4000, (uint32_t)valueLength - 4000);
+
+ cout << " CRC32= ";
+ cout << computeCyclicRedundancyCode32((uint8_t*) value, (uint32_t)valueLength);
+
+ cout << endl;
+#endif
+
+ sendMessageToArbiter();
+ m_nextMode = MODE_GENERATE_NEW_SEEDS;
+
+ }
+}
+
+
+void SpuriousSeedAnnihilator::generateNewSeeds() {
+
+ m_initialized = false;
+ m_seedIndex = 0;
+ m_pathIndex = 0;
+ m_location = 0;
+
+ // we have everything we need in the key-value store now.
+
+ int oldCount = m_seeds->size();
+
+ m_seeds->clear();
+
+ for(int metaSeedIndex = 0 ; metaSeedIndex < (int) m_newSeedBluePrints.size() ;
+ ++ metaSeedIndex) {
+
+#if 0
+ cout << "[DEBUG] ***" << endl;
+ cout << "DEBUG generateNewSeeds metaSeedIndex= " << metaSeedIndex << endl;
+ cout << " blueprint ";
+ m_newSeedBluePrints[metaSeedIndex].print();
+ cout << endl;
+#endif
+
+ GraphPath emptyPath;
+ emptyPath.setKmerLength(m_parameters->getWordSize());
+
+ vector<PathHandle> & handles = m_newSeedBluePrints[metaSeedIndex].getPathHandles();
+ vector<bool> & orientations = m_newSeedBluePrints[metaSeedIndex].getPathOrientations();
+ vector<GraphPath> & gaps = m_newSeedBluePrints[metaSeedIndex].getComputedPaths();
+
+ for(int pathHandleIndex = 0 ;
+ pathHandleIndex < (int) handles.size(); ++ pathHandleIndex) {
+
+ PathHandle & handle = handles[pathHandleIndex];
+ bool orientation = orientations[pathHandleIndex];
+
+ char * value = NULL;
+ int valueSize = 0;
+
+ string key = "";
+ getSeedKey(handle, key);
+
+ m_core->getKeyValueStore().getLocalKey(key, value, valueSize);
+
+#if 0
+ cout << "DEBUG generateNewSeeds getLocalKey on key " << key << " -> " << valueSize << " bytes";
+#endif
+
+ GraphPath seedPath;
+ seedPath.load(value);
+
+ if(orientation) {
+ GraphPath newPath;
+ seedPath.reverseContent(newPath);
+ seedPath = newPath;
+ }
+
+#if 0
+ cout << "DEBUG append path " << pathHandleIndex << " " << handle;
+ cout << " container " << emptyPath.size() << " object " << seedPath.size() << endl;
+#endif
+
+
+ emptyPath.appendPath(seedPath);
+
+ // if it is not the last one, append the gap
+ // too
+ if(pathHandleIndex != (int) handles.size() -1) {
+
+ GraphPath & gapPath = gaps[pathHandleIndex];
+
+#if 0
+ cout << "DEBUG append gap path ";
+ cout << "container " << emptyPath.size() << " object " << gapPath.size() << endl;
+#endif
+ emptyPath.appendPath(gapPath);
+ }
+ }
+
+ // add the new longer seed...
+
+ emptyPath.reserveSpaceForCoverage();
+
+ m_seeds->push_back(emptyPath);
+ }
+
+ // rebuild seeds here for real
+
+ int newCount = m_seeds->size();
+
+ // sort the seeds by length
+ vector<GraphPath> & seeds = (*m_seeds);
+
+ std::sort(seeds.begin(),
+ seeds.end(), comparePaths);
+
+
+ cout << "Rank " << m_core->getRank() << " merged its seeds: " << oldCount << " seeds -> ";
+ cout << newCount << " seeds" << endl;
+
+ sendMessageToArbiter();
+ m_nextMode = MODE_CLEAN_KEY_VALUE_STORE;
+
+ //cout << "DEBUG m_nextMode = MODE_CLEAN_KEY_VALUE_STORE ";
+
+#if 0
+ cout << MODE_CLEAN_KEY_VALUE_STORE << endl;
+#endif
+}
+
+void SpuriousSeedAnnihilator::cleanKeyValueStore() {
+
+#if 0
+ cout << "DEBUG Cleaning RKV store" << endl;
+#endif
+
+ m_core->getKeyValueStore().clear();
+
+ m_seedIndex = 0;
+ m_seedPosition = 0;
+ m_mode = MODE_GATHER_COVERAGE_VALUES;
+}
+
+void SpuriousSeedAnnihilator::gatherCoverageValues() {
+
+ /*
+ * We need to gather coverage values here.
+ *
+ * The message tag is RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE.
+ *
+ * For each kmer we put in the message, we will receive a coverage value.
+ *
+ * For this, we will use a workflow.
+ */
+
+ if(m_inbox->hasMessage(RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_REPLY)) {
+
+ Message * message = m_inbox->at(0);
+
+ Rank source = message->getSource();
+
+ MessageUnit * buffer = message->getBuffer();
+
+ int count = message->getCount();
+
+ for(int i = 0 ; i < count ; i += KMER_U64_ARRAY_SIZE) {
+
+ int seedIndex = m_buffersForPaths->getAt(source, i);
+ int positionIndex = m_buffersForPositions->getAt(source, i);
+
+#ifdef CONFIG_ASSERT
+ assert(seedIndex < (int) m_seeds->size());
+ assert(positionIndex < (int) m_seeds->at(seedIndex).size());
+#endif
+
+ CoverageDepth coverage = buffer[i];
+
+#ifdef CONFIG_ASSERT
+ if(coverage <= 0) {
+ cout << "Error: coverage " << coverage << endl;
+
+ }
+ assert(coverage > 0);
+#endif
+
+ GraphPath & seed = m_seeds->at(seedIndex);
+
+ seed.setCoverageValueAt(positionIndex, coverage);
+
+#if 0
+ cout << "[DEBUG] gatherCoverageValues operation: SetCoverage(path: " << seedIndex;
+ cout << " position: " << positionIndex << " value: " << coverage << endl;
+#endif
+ }
+
+ // we don't need to clear the buffers for m_buffersForMessages because
+ // the flush (or flushAll) call did that for us
+ m_buffersForPaths->reset(source);
+ m_buffersForPositions->reset(source);
+ m_pendingMessages--;
+ }
+
+ if(m_pendingMessages)
+ return;
+
+ bool hasSeeds = ( m_seeds->size() > 0 );
+
+ if(hasSeeds && m_seedIndex < (int)m_seeds->size()) {
+
+ GraphPath & seed = m_seeds->at(m_seedIndex);
+
+ if(m_seedPosition < seed.size()) {
+
+ if(m_seedIndex == 0 && m_seedPosition == 0) {
+
+ m_buffersForMessages = new BufferedData();
+ m_buffersForPaths = new BufferedData();
+ m_buffersForPositions = new BufferedData();
+
+ m_buffersForMessages->constructor(m_core->getSize(),
+ MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit),
+ "/dev/memory-for-messages", m_parameters->showMemoryAllocations(), KMER_U64_ARRAY_SIZE);
+
+ m_buffersForPaths->constructor(m_core->getSize(),
+ MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit),
+ "/dev/memory-for-paths", m_parameters->showMemoryAllocations(), KMER_U64_ARRAY_SIZE);
+
+ m_buffersForPositions->constructor(m_core->getSize(),
+ MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit),
+ "/dev/memory-for-positions", m_parameters->showMemoryAllocations(), KMER_U64_ARRAY_SIZE);
+
+ }
+
+ Kmer kmer;
+ seed.at(m_seedPosition, &kmer);
+
+ Rank rankToFlush = kmer.vertexRank(m_parameters->getSize(),m_parameters->getWordSize(),
+ m_parameters->getColorSpaceMode());
+
+ for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
+ m_buffersForMessages->addAt(rankToFlush, kmer.getU64(i));
+ m_buffersForPaths->addAt(rankToFlush, m_seedIndex);
+ m_buffersForPositions->addAt(rankToFlush, m_seedPosition);
+ }
+
+ if(m_buffersForMessages->flush(rankToFlush, KMER_U64_ARRAY_SIZE, RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE,
+ m_outboxAllocator, m_outbox,
+ m_rank, false)){
+
+ m_pendingMessages++;
+ }
+
+ // do something with this kmer now...
+ // possibly use BufferedData
+
+ m_seedPosition ++;
+ } else {
+ m_seedIndex ++;
+ m_seedPosition = 0;
+ }
+
+ // flush buffers if we have some
+ } else if (hasSeeds && !m_buffersForMessages->isEmpty()) {
+
+ // flush the remaining bits
+ m_pendingMessages += m_buffersForMessages->flushAll(RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE,
+ m_outboxAllocator, m_outbox, m_rank);
+
+ } else {
+
+ // only clear the buffers if we have seeds
+ if(hasSeeds) {
+ m_buffersForMessages->clear();
+ m_buffersForPaths->clear();
+ m_buffersForPositions->clear();
+
+ delete m_buffersForMessages;
+ delete m_buffersForPaths;
+ delete m_buffersForPositions;
+
+ m_buffersForMessages = NULL;
+ m_buffersForPaths = NULL;
+ m_buffersForPositions = NULL;
+ }
+
+ for(int i = 0 ; i < (int)m_seeds->size() ; ++i) {
+
+#ifdef CONFIG_ASSERT
+ assert(hasSeeds == true);
+#endif
+
+ m_seeds->at(i).computePeakCoverage();
+ }
+
+ m_mode = MODE_STOP_THIS_SITUATION;
+ }
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_REQUEST_VERTEX_COVERAGE_WITH_POSITION(Message*message) {
+
+ void*buffer=message->getBuffer();
+ Rank source=message->getSource();
+ MessageUnit*incoming=(MessageUnit*)buffer;
+ int count=message->getCount();
+ MessageUnit*message2=(MessageUnit*)m_outboxAllocator->allocate(count*sizeof(MessageUnit));
+
+ for(int i=0;i<count;i+= (KMER_U64_ARRAY_SIZE + 1)){
+
+ int position = incoming[i];
+ Kmer vertex;
+ int bufferPosition=i;
+ vertex.unpack(incoming + 1, &bufferPosition);
+
+ Vertex*node=m_subgraph->find(&vertex);
+
+ // if it is not there, then it has a coverage of 0
+ CoverageDepth coverage=0;
+
+ if(node!=NULL){
+ coverage=node->getCoverage(&vertex);
+
+ #ifdef CONFIG_ASSERT
+ assert(coverage!=0);
+ #endif
+ }
+
+ message2[i] = position;
+ message2[i + 1] = coverage;
+ }
+
+ Message aMessage(message2, count, source,
+ RAY_MESSAGE_TAG_REQUEST_VERTEX_COVERAGE_WITH_POSITION_REPLY, m_rank);
+
+ m_outbox->push_back(&aMessage);
+}
+
+void SpuriousSeedAnnihilator::evaluateGossips() {
+
+#if 0
+ cout << "[DEBUG] evaluateGossips" << endl;
+#endif
+
+ /**
+ * Here, we do that in batch.
+ */
+
+ vector<GraphSearchResult> & gossips = m_gossipAssetManager.getGossips();
+
+ m_seedGossipSolver.setInput(&gossips);
+ m_seedGossipSolver.compute();
+
+ vector<GraphSearchResult> & solution = m_seedGossipSolver.getSolution();
+
+ //cout << "[DEBUG] MODE_EVALUATE_GOSSIPS Rank " << m_rank << " gossip count: " << gossips.size() << endl;
+ //cout << "[DEBUG] MODE_EVALUATE_GOSSIPS solution has " << solution.size() << " entries !" << endl;
+
+ /**
+ * Now, the actor needs to check how many of its seeds are in the solution.
+ * The actor effectively loses ownership for any of these seeds in the solution.
+ *
+ * The actor can keep the seeds not in the solution.
+ *
+ * After that, the seeds in the solution need to be assigned ownership.
+ * For the first implementation, the ownership will be based on actors associated
+ * with the seeds in the solution.
+ *
+ * For instance, for the result A-B-C where A is owned by 0, B by 1 and C by 2. Then this
+ * A-B-C solution will be owned by 0, 1 or 2.
+ */
+
+ // current seeds are stored in m_seeds
+ // new seeds are in solution;
+ // to make this a generalized process, let's add the seeds that are not in the solution
+ // in a new solution
+
+ set<PathHandle> localPathsInSolution;
+
+ for(vector<GraphSearchResult>::iterator i = solution.begin();
+ i != solution.end() ; ++i) {
+
+ GraphSearchResult & result = *i;
+
+#if 0
+ cout << "DEBUG inspecting solution entry" << endl;
+ result.print();
+ cout << endl;
+#endif
+
+ vector<PathHandle> & handles = result.getPathHandles();
+
+ for(vector<PathHandle>::iterator j = handles.begin() ; j != handles.end() ; ++j) {
+
+ PathHandle & thePath = *j;
+ localPathsInSolution.insert(thePath);
+ }
+ }
+
+ //cout << "[DEBUG] MODE_EVALUATE_GOSSIPS " << solution.size() << " entries need an owner." << endl;
+
+ for(vector<GraphSearchResult>::iterator i = solution.begin() ;
+ i!= solution.end() ; ++i) {
+
+ GraphSearchResult & entry = *i;
+
+#if 0
+ cout << "DEBUG inspecting again this solution entry" << endl;
+ entry.print();
+ cout << endl;
+#endif
+
+ PathHandle & firstHandle = entry.getPathHandles()[0];
+ PathHandle & lastHandle = entry.getPathHandles()[entry.getPathHandles().size()-1];
+
+ // this algorithm is stupid because it does not enforce
+ // load balancing.
+ //
+ // TODO: implement a true load balancing algorithme here...
+
+ PathHandle smallest = firstHandle;
+ if(lastHandle < firstHandle)
+ smallest = lastHandle;
+
+ Rank owner = getRankFromPathUniqueId(smallest);
+
+ if(owner == m_core->getRank()) {
+ m_newSeedBluePrints.push_back(entry);
+
+#if 0
+ // The bug is here already for the blueprint
+ cout << "DEBUG adding this blueprint:" << endl;
+ entry.print();
+ cout << endl;
+#endif
+ }
+ }
+
+ //cout << "[DEBUG] MODE_EVALUATE_GOSSIPS Rank " << m_core->getRank() << " claimed ownership for " << m_newSeedBluePrints.size();
+
+#ifdef DEBUG_SEED_MERGING
+ cout << " before merging its own assets in the pool." << endl;
+#endif
+
+ // this needs to run after trimming those short seeds with dead-ends
+
+ // add local seeds that are not in the local solution
+ // a local seeds can't be in a remote solution if it is in
+ // the local solution anyway
+ for(int i = 0 ; i < (int) m_seeds->size() ; ++i) {
+
+ PathHandle identifier = getPathUniqueId(m_parameters->getRank(), i);
+
+ if(localPathsInSolution.count(identifier) == 0) {
+
+ GraphSearchResult result;
+ result.addPathHandle(identifier, false);
+
+ m_newSeedBluePrints.push_back(result);
+ }
+ }
+
+ //cout << "[DEBUG] MODE_EVALUATE_GOSSIPS Rank " << m_core->getRank() << " will assume ownership for " << m_newSeedBluePrints.size();
+ //cout << " objects, had " << m_seeds->size() << " before merging" << endl;
+
+#if 0
+ int index = 0;
+ for(vector<GraphSearchResult>::iterator i = m_newSeedBluePrints.begin() ;
+ i != m_newSeedBluePrints.end() ; ++i) {
+ cout << "[DEBUG] OWNED OBJECT @" << index++ << " ";
+ (*i).print();
+ cout << endl;
+ }
+#endif
+
+ //cout << "DEBUG next = MODE_REBUILD_SEED_ASSETS" << endl;
+
+ m_mode = MODE_REBUILD_SEED_ASSETS;
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MASTER_MODE_CLEAN_SEEDS(){
+
+ if(!m_cleaningIsStarted){
+ m_core->getSwitchMan()->openMasterMode(m_core->getOutbox(),m_core->getRank());
+
+ m_cleaningIsStarted=true;
+
+ }else if(m_core->getSwitchMan()->allRanksAreReady()){
+
+ if(!this->m_mergedSeeds){
+
+ this->m_core->getSwitchMan()->setMasterMode(RAY_MASTER_MODE_REGISTER_SEEDS);
+ this->m_core->getSwitchMan()->reset();
+
+ } else {
+
+ // the registered handle with the API is
+ // RAY_MASTER_MODE_PUSH_SEED_LENGTHS
+ m_core->getSwitchMan()->closeMasterMode();
+ }
+
+ // anyway, the code in this score changed the fate of future cycles.
+ m_cleaningIsStarted = false;
+ }
+}
+
+void SpuriousSeedAnnihilator::call_RAY_SLAVE_MODE_REGISTER_SEEDS(){
+
+ if(!m_initializedSeedRegistration) {
+
+ //cout << "[DEBUG] call_RAY_SLAVE_MODE_REGISTER_SEEDS: initializing..." << endl;
+
+ m_seedIndex=0;
+ m_seedPosition=0;
+ m_initializedSeedRegistration = true;
+ m_registrationIterations++;
+
+ if(!m_debugCode && m_parameters->hasCheckpoint("Seeds")){
+ m_hasCheckpointFilesForSeeds = true;
+ }
+
+ return;
+ }
+
+ // iteration 1: for the seed elimination (skipped for small k)
+ // iteration 2: for the merging (never skipped)
+ if((!m_debugCode && m_hasCheckpointFilesForSeeds) || (m_skip && m_registrationIterations == 1)){
+
+ m_core->closeSlaveModeLocally();
+
+#ifdef DEBUG_SEED_REGISTRATION
+ cout << "[DEBUG] skipping call_RAY_SLAVE_MODE_REGISTER_SEEDS" << endl;
+#endif
+
+ m_initializedSeedRegistration = false;
+
+ return;
+ }
+
+ if(m_inbox->hasMessage(RAY_MESSAGE_TAG_PUSH_SEEDS_REPLY)){
+ m_activeQueries--;
+ return;
+ }
+
+ #ifdef ASSERT
+ assert(m_activeQueries>=0);
+ #endif
+
+ if(m_activeQueries > 0)
+ return;
+
+ #ifdef ASSERT
+ assert(m_activeQueries==0);
+ #endif
+
+ if(m_seedIndex < (int)m_seeds->size()){
+
+ if(m_seedPosition < (*m_seeds)[m_seedIndex].size()){
+
+ if(m_seedIndex==0 && m_seedPosition == 0)
+ cout<<"Rank "<<m_core->getRank()<<" has "<<m_seeds->size()<<" seeds to register."<<endl;
+
+ Kmer vertex;
+ (*m_seeds)[m_seedIndex].at(m_seedPosition,&vertex);
+
+ Rank destination=m_parameters->vertexRank(&vertex);
+
+ MessageTag tag=RAY_MESSAGE_TAG_PUSH_SEEDS;
+ int elements = m_virtualCommunicator->getElementsPerQuery(tag);
+
+ for(int i=0;i<vertex.getNumberOfU64();i++){
+ m_buffers.addAt(destination,vertex.getU64(i));
+ }
+
+ PathHandle identifier = getPathUniqueId(m_rank, m_seedIndex);
+
+ m_buffers.addAt(destination, identifier.getValue());
+ m_buffers.addAt(destination, m_seedPosition);
+
+ if(m_buffers.flush(destination, elements, tag, m_outboxAllocator, m_outbox, m_rank,false)){
+ m_activeQueries++;
+ }
+
+#if 0
+ if(m_seedPosition % 1000 == 0) {
+ cout << "Rank "<<m_rank << " registered " << m_seedIndex << "/" <<m_seeds->size();
+ cout<< " "<< m_seedPosition << "/" << (*m_seeds)[m_seedIndex].size() << endl;
+ }
+#endif
+
+ m_seedPosition++;
+ }else{
+
+ if(m_seedIndex % 1000 == 0)
+ cout << "Rank "<<m_rank << " registered " << m_seedIndex << "/" <<m_seeds->size() << endl;
+
+ m_seedIndex++;
+
+ m_seedPosition = 0;
+ }
+
+ }else if(!m_buffers.isEmpty()){
+
+/**
+ * Flush additional queries
+ */
+ MessageTag tag=RAY_MESSAGE_TAG_PUSH_SEEDS;
+
+ m_activeQueries += m_buffers.flushAll(tag, m_outboxAllocator, m_outbox, m_rank);
+
+ }else{
+ m_initializedSeedRegistration = false;
+
+ cout << "Rank "<<m_rank << " registered " << m_seedIndex - 1 << "/" <<m_seeds->size() << endl;
+ cout<<"Rank "<<m_rank << " registered its seeds" << endl;
+
+ m_core->closeSlaveModeLocally();
+ }
+}
+
+/**
+ *
+ * \see https://github.com/sebhtml/ray/issues/188
+ * \author Sébastien Boisvert
+ */
+void SpuriousSeedAnnihilator::call_RAY_MASTER_MODE_MERGE_SEEDS() {
+ // merge the seeds using arbitration
+
+ if(!m_mergingIsStarted){
+ m_core->getSwitchMan()->openMasterMode(m_core->getOutbox(),m_core->getRank());
+
+ m_mergingIsStarted = true;
+
+ }else if(m_core->getSwitchMan()->allRanksAreReady()){
+
+ //this->getThis()->getThat()->getCore()->getSwitchMan()->setMasterMode(RAY_MASTER_MODE_PUSH_SEED_LENGTHS);
+ this->getThis()->getThat()->getCore()->getSwitchMan()->setMasterMode(RAY_MASTER_MODE_CLEAN_SEEDS);
+ getCore()->getSwitchMan()->reset();
+ }
+
+ /*
+ * parallel design:
+ *
+ * while true
+ * 1. register seeds
+ * 2. merge seeds using arbitration
+ * 3. clean seeds
+ *
+ * then, exit via the backdoor called "RAY_MASTER_MODE_PUSH_SEED_LENGTHS"
+ *
+ * checkpoints are saved top storage devices after this current step.
+ */
+}
+
+void SpuriousSeedAnnihilator::call_RAY_SLAVE_MODE_MERGE_SEEDS() {
+ // merge the seeds using arbitration
+
+ if(this->m_debug) {
+ cout << m_core->getRank() << " merging seeds now." << endl;
+ }
+
+ m_mergingTechnology.mainLoop();
+
+ // It is the class SeedMergingWorkflow (interface is TaskCreator) that
+ // will actually close the slave mode.
+ //m_core->closeSlaveModeLocally();
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_MERGE_SEEDS(Message*message) {
+ // merge the seeds using arbitration
+}
+
+void SpuriousSeedAnnihilator::call_RAY_SLAVE_MODE_FILTER_SEEDS(){
+
+ if((!m_debugCode && m_hasCheckpointFilesForSeeds) || m_skip){
+
+ m_core->closeSlaveModeLocally();
+ return;
+ }
+
+ m_workflow.mainLoop();
+}
+
+void SpuriousSeedAnnihilator::call_RAY_SLAVE_MODE_CLEAN_SEEDS(){
+
+ m_cleaningIterations ++;
+
+ // this must be skipped for the first iteration if m_skip is true
+ if((!m_debugCode && m_hasCheckpointFilesForSeeds) || (m_skip && m_cleaningIterations == 1)){
+
+ m_core->closeSlaveModeLocally();
+ return;
+ }
+
+// Trace was here -> PASS
+
+ #ifdef ASSERT
+ assert(m_parameters != NULL);
+ assert(m_subgraph != NULL);
+ #endif
+
+ // clear graph
+ GridTableIterator iterator;
+ iterator.constructor(m_subgraph, m_parameters->getWordSize(), m_parameters);
+
+ #ifdef ASSERT
+ LargeCount cleared=0;
+ #endif
+
+// Trace was here -> PASS
+
+ while(iterator.hasNext()){
+ iterator.next();
+ Kmer key=*(iterator.getKey());
+ m_subgraph->clearDirections(&key);
+
+ #ifdef ASSERT
+ cleared++;
+
+ Vertex*node=m_subgraph->find(&key);
+ assert(node->getFirstDirection() == NULL);
+
+ #endif
+ }
+
+// Trace was here -> PASS
+
+ #ifdef ASSERT
+ assert(cleared == m_subgraph->size());
+ #endif
+
+// Trace was here -> PASS
+
+ int bytes=m_directionsAllocator->getChunkSize() * m_directionsAllocator->getNumberOfChunks();
+
+ cout<<"Rank "<<m_rank<<" freed "<< bytes <<" bytes from the path memory pool ";
+ cout << "(chunks: " << m_directionsAllocator->getNumberOfChunks() << ")" << endl;
+
+ m_directionsAllocator->clear();
+
+// Trace was here -> <s>FAIL</s> PASS
+
+/*
+ * Tell another rank that we are done with this.
+ */
+ m_core->closeSlaveModeLocally();
+
+#if 0
+ cout << "DEBUG closing slave mode locally" << endl;
+#endif
+}
+
+/**
+ * The 3 methods below are not used, but they were added to test this
+ * new RayPlatform API call:
+ *
+ * __ConfigureMessageTagHandler
+ *
+ */
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_REGISTER_SEEDS(Message*message){
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_PUSH_SEED_LENGTHS(Message*message){
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_FILTER_SEEDS(Message*message){
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_CLEAN_SEEDS(Message*message){
+}
+
+void SpuriousSeedAnnihilator::call_RAY_MESSAGE_TAG_SEND_SEED_LENGTHS(Message*message){
+ void*buffer=message->getBuffer();
+ MessageUnit*incoming=(MessageUnit*)buffer;
+ int count=message->getCount();
+ for(int i=0;i<count;i+=2){
+ int seedLength=incoming[i];
+ int number=incoming[i+1];
+ m_masterSeedLengths[seedLength]+=number;
+ }
+
+ Message aMessage(NULL,0,message->getSource(),RAY_MESSAGE_TAG_SEND_SEED_LENGTHS_REPLY,m_rank);
+ m_outbox->push_back(&aMessage);
+}
+
+void SpuriousSeedAnnihilator::writeCheckpointForSeeds(){
+
+ /* write the Seeds checkpoint */
+ if(m_parameters->writeCheckpoints() && !m_parameters->hasCheckpoint("Seeds")){
+
+ ofstream f(m_parameters->getCheckpointFile("Seeds").c_str());
+ ostringstream buffer;
+
+ cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint Seeds"<<endl;
+ int count=(*m_seeds).size();
+
+ buffer.write((char*)&count, sizeof(int));
+
+ // TODO: don't store coverage in seed checkpoints...
+
+ for(int i=0;i<(int)(*m_seeds).size();i++){
+ int length=(*m_seeds)[i].size();
+ buffer.write((char*)&length, sizeof(int));
+
+ for(int j=0;j<(int)(*m_seeds)[i].size();j++){
+ Kmer theKmer;
+ (*m_seeds)[i].at(j,&theKmer);
+ theKmer.write(&buffer);
+
+ CoverageDepth coverageValue=0;
+ coverageValue=(*m_seeds)[i].getCoverageAt(j);
+ buffer.write((char*)&coverageValue, sizeof(CoverageDepth));
+ flushFileOperationBuffer(false, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
+ }
+ }
+ flushFileOperationBuffer(true, &buffer, &f, CONFIG_FILE_IO_BUFFER_SIZE);
+ f.close();
+ }
+}
+
+bool SpuriousSeedAnnihilator::isPrimeNumber(int number) {
+
+ for(int i = 2 ; i < number ; ++i) {
+
+ if(number % i == 0)
+ return false;
+ }
+
+ return true;
+}
+
+Rank SpuriousSeedAnnihilator::getArbiter() {
+ int rank = m_parameters->getSize();
+
+ while(rank > 0 && !isPrimeNumber(rank))
+ rank--;
+
+ return rank;
+}
+
+void SpuriousSeedAnnihilator::writeSeedStatistics(){
+ ostringstream file;
+ file<<m_parameters->getPrefix();
+ file<<"SeedLengthDistribution.txt";
+ ofstream f(file.str().c_str());
+
+ f<<"# SeedLengthInNucleotides Frequency"<<endl;
+
+ for(map<int,int>::iterator i=m_masterSeedLengths.begin();i!=m_masterSeedLengths.end();i++){
+ int length=i->first;
+ int count=i->second;
+ f<<length<<"\t"<<count<<endl;
+ }
+ f.close();
+}
+
+/*
+ * Methods to implement for the CorePlugin interface.
+ *
+ * The entry point is RAY_MASTER_MODE_REGISTER_SEEDS.
+ *
+ * The exit point is RAY_MASTER_MODE_TRIGGER_DETECTION (not here).
+ */
+void SpuriousSeedAnnihilator::registerPlugin(ComputeCore*core){
+
+ m_core=core;
+ PluginHandle plugin=core->allocatePluginHandle();
+ m_plugin=plugin;
+
+ core->setPluginName(plugin,"SpuriousSeedAnnihilator");
+ core->setPluginDescription(plugin,"Pre-processing of seeds.");
+ core->setPluginAuthors(plugin,"Sébastien Boisvert");
+ core->setPluginLicense(plugin,"GNU General Public License version 3");
+
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SEED_GOSSIP);
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SEED_GOSSIP_REPLY);
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_ARBITER_SIGNAL);
+
+ __ConfigureMasterModeHandler(SpuriousSeedAnnihilator, RAY_MASTER_MODE_REGISTER_SEEDS);
+ __ConfigureMasterModeHandler(SpuriousSeedAnnihilator, RAY_MASTER_MODE_FILTER_SEEDS);
+ __ConfigureMasterModeHandler(SpuriousSeedAnnihilator, RAY_MASTER_MODE_CLEAN_SEEDS);
+ __ConfigureMasterModeHandler(SpuriousSeedAnnihilator, RAY_MASTER_MODE_PUSH_SEED_LENGTHS);
+ __ConfigureMasterModeHandler(SpuriousSeedAnnihilator, RAY_MASTER_MODE_MERGE_SEEDS);
+ __ConfigureMasterModeHandler(SpuriousSeedAnnihilator, RAY_MASTER_MODE_PROCESS_MERGING_ASSETS);
+
+ __ConfigureSlaveModeHandler(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_REGISTER_SEEDS);
+ __ConfigureSlaveModeHandler(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_FILTER_SEEDS);
+ __ConfigureSlaveModeHandler(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_CLEAN_SEEDS);
+ __ConfigureSlaveModeHandler(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_PUSH_SEED_LENGTHS);
+ __ConfigureSlaveModeHandler(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_MERGE_SEEDS);
+ __ConfigureSlaveModeHandler(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_PROCESS_MERGING_ASSETS);
+
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_REGISTER_SEEDS);
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_FILTER_SEEDS);
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_CLEAN_SEEDS);
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_PUSH_SEED_LENGTHS);
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SEND_SEED_LENGTHS);
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_MERGE_SEEDS);
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_PROCESS_MERGING_ASSETS);
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY);
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY_REPLY);
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SAY_HELLO_TO_ARBITER);
+ __ConfigureMessageTagHandler(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_REQUEST_VERTEX_COVERAGE_WITH_POSITION);
+
+ RAY_MESSAGE_TAG_SEND_SEED_LENGTHS_REPLY = m_core->allocateMessageTagHandle(m_plugin);
+ m_core->setMessageTagSymbol(m_plugin, RAY_MESSAGE_TAG_SEND_SEED_LENGTHS_REPLY, "RAY_MESSAGE_TAG_SEND_SEED_LENGTHS_REPLY");
+
+ RAY_MESSAGE_TAG_REQUEST_VERTEX_COVERAGE_WITH_POSITION_REPLY = m_core->allocateMessageTagHandle(m_plugin);
+ m_core->setMessageTagSymbol(m_plugin, RAY_MESSAGE_TAG_REQUEST_VERTEX_COVERAGE_WITH_POSITION_REPLY,
+ "RAY_MESSAGE_TAG_REQUEST_VERTEX_COVERAGE_WITH_POSITION_REPLY");
+
+
+ m_outboxAllocator = m_core->getOutboxAllocator();
+ m_outbox = m_core->getOutbox() ;
+ m_rank = m_core->getRank() ;
+ m_inbox = m_core->getInbox();
+
+ m_size = m_core->getSize();
+
+ m_distributionIsStarted=false;
+ m_filteringIsStarted=false;
+ m_cleaningIsStarted=false;
+
+ m_initializedSeedRegistration = false;
+
+ core->setMasterModeNextMasterMode(m_plugin, RAY_MASTER_MODE_REGISTER_SEEDS, RAY_MASTER_MODE_FILTER_SEEDS);
+ core->setMasterModeNextMasterMode(m_plugin, RAY_MASTER_MODE_FILTER_SEEDS, RAY_MASTER_MODE_CLEAN_SEEDS);
+
+ core->setMasterModeNextMasterMode(m_plugin, RAY_MASTER_MODE_CLEAN_SEEDS, RAY_MASTER_MODE_PROCESS_MERGING_ASSETS);
+ core->setMasterModeNextMasterMode(m_plugin, RAY_MASTER_MODE_PROCESS_MERGING_ASSETS, RAY_MASTER_MODE_PUSH_SEED_LENGTHS);
+
+ core->setMasterModeToMessageTagSwitch(m_plugin, RAY_MASTER_MODE_PROCESS_MERGING_ASSETS, RAY_MESSAGE_TAG_PROCESS_MERGING_ASSETS);
+ core->setMasterModeToMessageTagSwitch(m_plugin, RAY_MASTER_MODE_REGISTER_SEEDS, RAY_MESSAGE_TAG_REGISTER_SEEDS);
+ core->setMasterModeToMessageTagSwitch(m_plugin, RAY_MASTER_MODE_FILTER_SEEDS, RAY_MESSAGE_TAG_FILTER_SEEDS);
+ core->setMasterModeToMessageTagSwitch(m_plugin, RAY_MASTER_MODE_CLEAN_SEEDS, RAY_MESSAGE_TAG_CLEAN_SEEDS);
+ core->setMasterModeToMessageTagSwitch(m_plugin, RAY_MASTER_MODE_PUSH_SEED_LENGTHS, RAY_MESSAGE_TAG_PUSH_SEED_LENGTHS);
+ core->setMasterModeToMessageTagSwitch(m_plugin, RAY_MASTER_MODE_MERGE_SEEDS, RAY_MESSAGE_TAG_MERGE_SEEDS);
+
+ core->setMessageTagToSlaveModeSwitch(m_plugin, RAY_MESSAGE_TAG_REGISTER_SEEDS, RAY_SLAVE_MODE_REGISTER_SEEDS);
+ core->setMessageTagToSlaveModeSwitch(m_plugin, RAY_MESSAGE_TAG_PROCESS_MERGING_ASSETS, RAY_SLAVE_MODE_PROCESS_MERGING_ASSETS);
+ core->setMessageTagToSlaveModeSwitch(m_plugin, RAY_MESSAGE_TAG_FILTER_SEEDS, RAY_SLAVE_MODE_FILTER_SEEDS);
+ core->setMessageTagToSlaveModeSwitch(m_plugin, RAY_MESSAGE_TAG_CLEAN_SEEDS, RAY_SLAVE_MODE_CLEAN_SEEDS);
+ core->setMessageTagToSlaveModeSwitch(m_plugin, RAY_MESSAGE_TAG_PUSH_SEED_LENGTHS, RAY_SLAVE_MODE_PUSH_SEED_LENGTHS);
+ core->setMessageTagToSlaveModeSwitch(m_plugin, RAY_MESSAGE_TAG_MERGE_SEEDS, RAY_SLAVE_MODE_MERGE_SEEDS);
+
+ m_activeQueries=0;
+
+ m_virtualCommunicator = m_core->getVirtualCommunicator();
+ m_virtualProcessor = m_core->getVirtualProcessor();
+
+ m_initialized=false;
+
+ __BindPlugin(SpuriousSeedAnnihilator);
+
+ m_hasCheckpointFilesForSeeds = false;
+
+}
+
+void SpuriousSeedAnnihilator::resolveSymbols(ComputeCore*core){
+
+ // before arriving here, there is this edition to be performed elsewhere in the code
+ // replace RAY_MASTER_MODE_TRIGGER_DETECTION by RAY_MASTER_MODE_CLEAN_SEEDS
+
+ RAY_MASTER_MODE_TRIGGER_DETECTION=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_TRIGGER_DETECTION");
+ core->setMasterModeNextMasterMode(m_plugin, RAY_MASTER_MODE_PUSH_SEED_LENGTHS, RAY_MASTER_MODE_TRIGGER_DETECTION);
+
+ m_seeds=(vector<GraphPath>*) m_core->getObjectFromSymbol(m_plugin, "/RayAssembler/ObjectStore/Seeds.ray");
+ m_parameters=(Parameters*)m_core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/Parameters.ray");
+
+ RAY_MESSAGE_TAG_PUSH_SEEDS = m_core->getMessageTagFromSymbol(m_plugin, "RAY_MESSAGE_TAG_PUSH_SEEDS");
+ RAY_MESSAGE_TAG_PUSH_SEEDS_REPLY = m_core->getMessageTagFromSymbol(m_plugin, "RAY_MESSAGE_TAG_PUSH_SEEDS_REPLY");
+ RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS = m_core->getMessageTagFromSymbol(m_plugin, "RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS");
+ RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE = m_core->getMessageTagFromSymbol(m_plugin, "RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE");
+ RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_REPLY = m_core->getMessageTagFromSymbol(m_plugin, "RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_REPLY");
+
+ int elements = m_virtualCommunicator->getElementsPerQuery(RAY_MESSAGE_TAG_PUSH_SEEDS);
+
+ m_buffers.constructor(m_size,MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit),
+ "/memory/SpuriousSeedAnnihilator",m_parameters->showMemoryAllocations(),elements);
+
+ m_subgraph=(GridTable*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/deBruijnGraph_part.ray");
+ m_directionsAllocator = (MyAllocator*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/directionMemoryPool.ray");
+
+ RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT = m_core->getMessageTagFromSymbol(m_plugin, "RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT");
+ RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE = m_core->getMessageTagFromSymbol(m_plugin, "RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE");
+ RAY_MPI_TAG_ASK_VERTEX_PATH = m_core->getMessageTagFromSymbol(m_plugin, "RAY_MPI_TAG_ASK_VERTEX_PATH");
+
+ m_workflow.initialize(m_seeds, m_virtualCommunicator, m_virtualProcessor, m_core, m_parameters,
+ RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT, RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ RAY_MPI_TAG_ASK_VERTEX_PATH
+ );
+
+ m_mergingTechnology.initialize(m_seeds, m_virtualCommunicator, m_virtualProcessor, m_core, m_parameters,
+ RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT, RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,
+ RAY_MPI_TAG_ASK_VERTEX_PATH
+ );
+
+
+/**
+ * Turn this on to run this code even when checkpoints exist.
+ * This should be set to false in production.
+ */
+ m_debugCode = m_parameters->hasOption("-debug-seed-filter");
+
+ m_skip = 2 * m_parameters->getWordSize() < m_parameters->getMinimumContigLength();
+
+ m_cleaningIterations = 0;
+ m_registrationIterations = 0;
+
+ m_debug = false;
+ m_mergingIsStarted = false;
+
+ m_filteredSeeds = false;
+ m_mergedSeeds = false;
+
+ m_initializedProcessing = false;
+
+ // for communication
+ m_pendingMessages = 0;
+
+}
diff --git a/code/SpuriousSeedAnnihilator/SpuriousSeedAnnihilator.h b/code/SpuriousSeedAnnihilator/SpuriousSeedAnnihilator.h
new file mode 100644
index 0000000..76d07e0
--- /dev/null
+++ b/code/SpuriousSeedAnnihilator/SpuriousSeedAnnihilator.h
@@ -0,0 +1,363 @@
+/*
+ * Ray -- Parallel genome assemblies for parallel DNA sequencing
+ * Copyright (C) 2013 Sébastien Boisvert
+ *
+ * http://DeNovoAssembler.SourceForge.Net/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You have received a copy of the GNU General Public License
+ * along with this program (gpl-3.0.txt).
+ * see <http://www.gnu.org/licenses/>
+ */
+
+#define GOSSIP_ALGORITHM_FOR_SEEDS "THOR synchronization algorithm v15.2.1" // see also GossipAssetManager
+
+#include "SeedFilteringWorkflow.h"
+#include "SeedMergingWorkflow.h"
+#include "SeedGossipSolver.h"
+#include "GossipAssetManager.h"
+
+#include <code/SeedingData/GraphPath.h>
+#include <code/Mock/Parameters.h>
+#include <code/VerticesExtractor/GridTable.h>
+
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/communication/VirtualCommunicator.h>
+#include <RayPlatform/communication/BufferedData.h>
+
+#include <vector>
+#include <map>
+using namespace std;
+
+__DeclarePlugin(SpuriousSeedAnnihilator);
+
+__DeclareMasterModeAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_REGISTER_SEEDS);
+__DeclareMasterModeAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_FILTER_SEEDS);
+__DeclareMasterModeAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_CLEAN_SEEDS);
+__DeclareMasterModeAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_PUSH_SEED_LENGTHS);
+__DeclareMasterModeAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_MERGE_SEEDS);
+__DeclareMasterModeAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_PROCESS_MERGING_ASSETS);
+
+__DeclareSlaveModeAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_REGISTER_SEEDS);
+__DeclareSlaveModeAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_FILTER_SEEDS);
+__DeclareSlaveModeAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_CLEAN_SEEDS);
+__DeclareSlaveModeAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_PUSH_SEED_LENGTHS);
+__DeclareSlaveModeAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_MERGE_SEEDS);
+__DeclareSlaveModeAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_PROCESS_MERGING_ASSETS);
+
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_REGISTER_SEEDS);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_FILTER_SEEDS);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_CLEAN_SEEDS);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_PUSH_SEED_LENGTHS);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SEND_SEED_LENGTHS);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_MERGE_SEEDS);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_PROCESS_MERGING_ASSETS);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY_REPLY);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SAY_HELLO_TO_ARBITER);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_REQUEST_VERTEX_COVERAGE_WITH_POSITION);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SEED_GOSSIP);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SEED_GOSSIP_REPLY);
+__DeclareMessageTagAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_ARBITER_SIGNAL);
+
+#ifndef _SpuriousSeedAnnihilator_h
+#define _SpuriousSeedAnnihilator_h
+
+/**
+ * This class cleans undesired seeds.
+ *
+ * Undesired seeds are produced sometimes when using
+ * large kmers.
+ *
+ * Steps:
+ *
+ * 1. Distribute seeds in the graph;
+ * 2. Run the filter on seeds;
+ * 3. Clean the seed annotations.
+ *
+ * Design:
+ *
+ * SpuriousSeedAnnihilator
+ * | \---------------------------------
+ * | |
+ * | |
+ * SeedFilteringWorkflow SeedMergingWorkflow
+ * | |
+ * | |
+ * | |
+ * AnnihilationWorker NanoMerger
+ * | | | |
+ * | | | |
+ * | | | |
+ * AttributeFetcher | | AnnotationFetcher
+ * | |
+ * AnnotationFetcher |
+ * AttributeFetcher
+ *
+ * \author Sébastien Boisvert
+ *
+ */
+class SpuriousSeedAnnihilator: public CorePlugin {
+
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_REGISTER_SEEDS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_FILTER_SEEDS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_CLEAN_SEEDS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_PUSH_SEED_LENGTHS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_MERGE_SEEDS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MASTER_MODE_PROCESS_MERGING_ASSETS);
+
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_REGISTER_SEEDS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_FILTER_SEEDS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_CLEAN_SEEDS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_PUSH_SEED_LENGTHS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_MERGE_SEEDS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_SLAVE_MODE_PROCESS_MERGING_ASSETS);
+
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_REGISTER_SEEDS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_FILTER_SEEDS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_CLEAN_SEEDS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_PUSH_SEED_LENGTHS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SEND_SEED_LENGTHS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_MERGE_SEEDS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_PROCESS_MERGING_ASSETS);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY_REPLY);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SAY_HELLO_TO_ARBITER);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_REQUEST_VERTEX_COVERAGE_WITH_POSITION);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SEED_GOSSIP);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_SEED_GOSSIP_REPLY);
+ __AddAdapter(SpuriousSeedAnnihilator, RAY_MESSAGE_TAG_ARBITER_SIGNAL);
+
+ int m_toDistribute;
+ SeedGossipSolver m_seedGossipSolver;
+ vector<GraphSearchResult> m_newSeedBluePrints;
+
+ MasterMode RAY_MASTER_MODE_REGISTER_SEEDS;
+ MasterMode RAY_MASTER_MODE_FILTER_SEEDS;
+ MasterMode RAY_MASTER_MODE_CLEAN_SEEDS;
+ MasterMode RAY_MASTER_MODE_PUSH_SEED_LENGTHS;
+ MasterMode RAY_MASTER_MODE_TRIGGER_DETECTION;
+ MasterMode RAY_MASTER_MODE_MERGE_SEEDS;
+ MasterMode RAY_MASTER_MODE_PROCESS_MERGING_ASSETS;
+
+ SlaveMode RAY_SLAVE_MODE_REGISTER_SEEDS;
+ SlaveMode RAY_SLAVE_MODE_FILTER_SEEDS;
+ SlaveMode RAY_SLAVE_MODE_CLEAN_SEEDS;
+ SlaveMode RAY_SLAVE_MODE_PUSH_SEED_LENGTHS;
+ SlaveMode RAY_SLAVE_MODE_MERGE_SEEDS;
+ SlaveMode RAY_SLAVE_MODE_PROCESS_MERGING_ASSETS;
+
+ MessageTag RAY_MESSAGE_TAG_REQUEST_VERTEX_COVERAGE_WITH_POSITION;
+ MessageTag RAY_MESSAGE_TAG_REQUEST_VERTEX_COVERAGE_WITH_POSITION_REPLY;
+
+ MessageTag RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE;
+ MessageTag RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE_REPLY;
+
+ MessageTag RAY_MESSAGE_TAG_REGISTER_SEEDS;
+ MessageTag RAY_MESSAGE_TAG_FILTER_SEEDS;
+ MessageTag RAY_MESSAGE_TAG_CLEAN_SEEDS;
+ MessageTag RAY_MESSAGE_TAG_PUSH_SEED_LENGTHS;
+ MessageTag RAY_MESSAGE_TAG_MERGE_SEEDS;
+ MessageTag RAY_MESSAGE_TAG_PROCESS_MERGING_ASSETS;
+
+ MessageTag RAY_MESSAGE_TAG_PUSH_SEEDS;
+ MessageTag RAY_MESSAGE_TAG_PUSH_SEEDS_REPLY;
+ MessageTag RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE;
+ MessageTag RAY_MPI_TAG_ASK_VERTEX_PATH;
+
+ MessageTag RAY_MESSAGE_TAG_SEND_SEED_LENGTHS;
+ MessageTag RAY_MESSAGE_TAG_SEND_SEED_LENGTHS_REPLY;
+ MessageTag RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS;
+ MessageTag RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY;
+ MessageTag RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY_REPLY;
+ MessageTag RAY_MESSAGE_TAG_ARBITER_SIGNAL;
+ MessageTag RAY_MESSAGE_TAG_SEED_GOSSIP;
+ MessageTag RAY_MESSAGE_TAG_SEED_GOSSIP_REPLY;
+ MessageTag RAY_MESSAGE_TAG_SAY_HELLO_TO_ARBITER;
+
+ GossipAssetManager m_gossipAssetManager;
+ int m_initialGossips;
+ int m_messagesSentForGossiping;
+
+ int MODE_SPREAD_DATA;
+ int MODE_GATHER_COVERAGE_VALUES;
+ int MODE_STOP_THIS_SITUATION;
+ int MODE_REBUILD_SEED_ASSETS;
+ int MODE_SHARE_PUSH_DATA_IN_KEY_VALUE_STORE;
+ int MODE_CHECK_RESULTS;
+ int MODE_SHARE_WITH_LINKED_ACTORS;
+ int MODE_WAIT_FOR_ARBITER;
+ int MODE_EVALUATE_GOSSIPS;
+ int MODE_CLEAN_KEY_VALUE_STORE;
+ int MODE_GENERATE_NEW_SEEDS;
+ Rank m_rankToAdvise;
+
+ bool m_messageWasSentToArbiter;
+
+ bool m_mustAdviseRanks;
+ int m_synced;
+ int m_mode;
+ int m_nextMode;
+ bool m_initializedProcessing;
+ int m_entryIndex;
+
+ bool m_processingIsStarted;
+ bool m_debug;
+ bool m_mergingIsStarted;
+ bool m_skip;
+ bool m_distributionIsStarted;
+ bool m_filteringIsStarted;
+ bool m_cleaningIsStarted;
+ bool m_gatheringHasStarted;
+
+ bool m_filteredSeeds;
+ bool m_mergedSeeds;
+ bool m_initializedSeedRegistration;
+
+ int m_cleaningIterations;
+ int m_registrationIterations;
+ map<int,int> m_masterSeedLengths;
+ map<int,int> m_slaveSeedLengths;
+ map<int,int>::iterator m_iterator;
+
+ KeyValueStoreRequest m_request;
+
+/*
+ * The workflow implements the TaskCreator interface so that it's possible for a
+ * single CorePlugin to implement several workflows that use VirtualCommunicator and
+ * VirtualProcessor via TaskCreator.
+ */
+ SeedFilteringWorkflow m_workflow;
+ SeedMergingWorkflow m_mergingTechnology;
+
+ GridTable*m_subgraph;
+ MyAllocator*m_directionsAllocator;
+
+ vector<GraphPath>*m_seeds;
+ vector<GraphPath> m_newSeeds;
+
+ Parameters*m_parameters;
+
+ int m_seedIndex;
+ int m_seedPosition;
+
+ int m_activeQueries;
+ BufferedData m_buffers;
+
+ VirtualCommunicator * m_virtualCommunicator;
+ VirtualProcessor * m_virtualProcessor;
+
+ Rank m_rank;
+ int m_size;
+ RingAllocator*m_outboxAllocator;
+ StaticVector*m_inbox;
+ StaticVector*m_outbox;
+
+ bool m_hasCheckpointFilesForSeeds;
+ bool m_initialized;
+ bool m_communicatorWasTriggered;
+
+ void writeSeedStatistics();
+ void writeSingleSeedFile();
+ void writeCheckpointForSeeds();
+
+ bool m_debugCode;
+ bool m_messageWasSent;
+ bool m_messageWasReceived;
+
+#ifdef GOSSIP_ALGORITHM_FOR_SEEDS
+ bool m_hasNewGossips;
+ time_t m_lastGossipingEventTime;
+ map<int, set<Rank> > m_gossipStatus;
+ set<Rank> m_linkedActorsForGossip;
+#endif /* GOSSIP_ALGORITHM_FOR_SEEDS */
+
+ ComputeCore * getCore();
+
+ /* some useless methods. */
+ SpuriousSeedAnnihilator * getThis();
+ SpuriousSeedAnnihilator * getThat();
+
+ // stuff to gather coverage values
+
+ BufferedData * m_buffersForMessages;
+ BufferedData * m_buffersForPaths;
+ BufferedData * m_buffersForPositions;
+ int m_pendingMessages;
+
+ int m_pathIndex;
+ int m_location;
+
+ Rank getArbiter();
+ bool isPrimeNumber(int number);
+
+ void initializeMergingProcess();
+ void spreadAcquiredData();
+ void shareWithLinkedActors();
+ void checkResults();
+ void evaluateGossips();
+ void rebuildSeedAssets();
+ void pushDataInKeyValueStore();
+ void sendMessageToArbiter();
+
+ void getSeedKey(PathHandle & handle, string & keyObject);
+
+ void generateNewSeeds();
+ void cleanKeyValueStore();
+ void gatherCoverageValues();
+
+public:
+
+ SpuriousSeedAnnihilator();
+
+/*
+ * Methods to implement for the CorePlugin interface.
+ */
+ void registerPlugin(ComputeCore*core);
+ void resolveSymbols(ComputeCore*core);
+
+/*
+ * handlers.
+ */
+ void call_RAY_MASTER_MODE_REGISTER_SEEDS();
+ void call_RAY_MASTER_MODE_FILTER_SEEDS();
+ void call_RAY_MASTER_MODE_CLEAN_SEEDS();
+ void call_RAY_MASTER_MODE_PUSH_SEED_LENGTHS();
+ void call_RAY_MASTER_MODE_MERGE_SEEDS();
+ void call_RAY_MASTER_MODE_PROCESS_MERGING_ASSETS();
+
+ void call_RAY_SLAVE_MODE_REGISTER_SEEDS();
+ void call_RAY_SLAVE_MODE_FILTER_SEEDS();
+ void call_RAY_SLAVE_MODE_CLEAN_SEEDS();
+ void call_RAY_SLAVE_MODE_PUSH_SEED_LENGTHS();
+ void call_RAY_SLAVE_MODE_MERGE_SEEDS();
+ void call_RAY_SLAVE_MODE_PROCESS_MERGING_ASSETS();
+
+ void call_RAY_MESSAGE_TAG_REGISTER_SEEDS(Message*message);
+ void call_RAY_MESSAGE_TAG_FILTER_SEEDS(Message*message);
+ void call_RAY_MESSAGE_TAG_CLEAN_SEEDS(Message*message);
+ void call_RAY_MESSAGE_TAG_PUSH_SEED_LENGTHS(Message*message);
+ void call_RAY_MESSAGE_TAG_SEND_SEED_LENGTHS(Message*message);
+ void call_RAY_MESSAGE_TAG_MERGE_SEEDS(Message*message);
+ void call_RAY_MESSAGE_TAG_PROCESS_MERGING_ASSETS(Message * message);
+ void call_RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY(Message * message);
+ void call_RAY_MESSAGE_TAG_GATHER_PROXIMITY_ENTRY_REPLY(Message * message);
+ void call_RAY_MESSAGE_TAG_SAY_HELLO_TO_ARBITER(Message * message);
+
+ void call_RAY_MESSAGE_TAG_REQUEST_VERTEX_COVERAGE_WITH_POSITION(Message * message);
+
+ void call_RAY_MESSAGE_TAG_ARBITER_SIGNAL(Message * message);
+ void call_RAY_MESSAGE_TAG_SEED_GOSSIP(Message*message);
+ void call_RAY_MESSAGE_TAG_SEED_GOSSIP_REPLY(Message*message);
+};
+
+#endif /* _SpuriousSeedAnnihilator_h */
diff --git a/code/Surveyor/CoalescenceManager.cpp b/code/Surveyor/CoalescenceManager.cpp
new file mode 100644
index 0000000..7a069bf
--- /dev/null
+++ b/code/Surveyor/CoalescenceManager.cpp
@@ -0,0 +1,425 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of Ray Surveyor.
+
+ Ray Surveyor is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ Ray Surveyor is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Ray Surveyor. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "CoalescenceManager.h"
+#include "StoreKeeper.h"
+
+#include <code/Mock/constants.h>
+#include <code/Mock/common_functions.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/VerticesExtractor/Vertex.h>
+
+#include <iostream>
+using namespace std;
+
+CoalescenceManager::CoalescenceManager() {
+
+ m_kmerLength = 0;
+ m_colorSpaceMode = false;
+
+ m_buffers = NULL;
+ m_bufferSizes = NULL;
+}
+
+CoalescenceManager::~CoalescenceManager() {
+
+
+ free(m_buffers);
+ free(m_bufferSizes);
+
+ m_buffers = NULL;
+ m_bufferSizes = NULL;
+}
+
+void CoalescenceManager::runAssertions() {
+
+#ifdef CONFIG_ASSERT
+ for(int i = 0 ; i < m_storageActors; ++i) {
+ assert(getBufferUsedBytes(i) == 0);
+ }
+#endif
+
+}
+
+
+
+void CoalescenceManager::receive(Message & message) {
+
+ int tag = message.getTag();
+ int source = message.getSourceActor();
+
+ /*
+ printName();
+ cout << " CoalescenceManager DEBUG receive message !";
+ cout << endl;
+*/
+
+ if(tag == PAYLOAD) {
+
+ receivePayload(message);
+
+ } else if(tag == DIE) {
+
+#ifdef CONFIG_ASSERT
+ runAssertions();
+#endif
+
+
+
+ die();
+
+ } else if(tag == SET_KMER_LENGTH) {
+
+ int kmerLength = 0;
+ char * buffer = (char*)message.getBufferBytes();
+ memcpy(&kmerLength, buffer, sizeof(kmerLength));
+
+ if(m_kmerLength == 0)
+ m_kmerLength = kmerLength;
+
+ if(m_kmerLength != kmerLength) {
+
+ printName();
+ cout << " Error: the k-mer length is not the same in all input files !";
+ cout << endl;
+ }
+
+ // cout << "DEBUG m_kmerLength = " << m_kmerLength << endl;
+
+ // the color space mode is an artefact.
+ m_colorSpaceMode = false;
+
+ Message response;
+ response.setTag(SET_KMER_LENGTH_OK);
+
+ int source = message.getSourceActor();
+
+ /*
+ printName();
+ cout << "DEBUG Sending SET_KMER_LENGTH_OK to " << source << endl;
+ */
+
+ send(source, response);
+
+ /*
+ * It is not the job here to find the kmer length
+*/
+
+ } else if(tag == INTRODUCE_STORE) {
+
+ char * buffer = (char*) message.getBufferBytes();
+
+ int localStore = -1;
+
+ memcpy(&localStore, buffer, sizeof(localStore));
+
+#ifdef CONFIG_ASSERT
+ assert(localStore >= 0);
+#endif
+ // the the node on which we are
+ int rank = getRank();
+ int numberOfRanks = getSize();
+
+ /*
+ * We have N MPI ranks.
+ * The local rank is x.
+ * The local StoreKeeper actor is y.
+ * There are PLAN_STORE_KEEPER_ACTORS_PER_RANK StoreKeeper actors per rank
+ *
+ * So basically, we need to find the actor name on rank 0 (first StoreKeeper actor)
+ * then, we add PLAN_STORE_KEEPER_ACTORS_PER_RANK * getSize to that -1 (the last StoreKeeper
+ * actor)
+ *
+ * This obviously assumes that allocation is regular.
+ * This assumption is correct since everything is spawned by Mother actors (in Surveyor).
+ *
+ * y = x + i * N
+ *
+ * Find i
+ *
+ * y - x = i * N
+ * i = (y - x) / N
+ *
+ * or
+ *
+ */
+
+ int iterator = ( localStore - rank ) / numberOfRanks;
+
+ m_localStore = localStore;
+
+ int first = 0 + iterator * numberOfRanks;
+ int last = first + ( numberOfRanks * PLAN_STORE_KEEPER_ACTORS_PER_RANK ) -1;
+
+ m_storeFirstActor = first;
+ m_storeLastActor = last;
+
+ printName();
+ cout << " is now acquainted with StoreKeeper actors from ";
+ cout << m_storeFirstActor << " to " << m_storeLastActor << endl;
+
+ // allocate buffers too
+
+ if(m_buffers == NULL) {
+
+ int storageActors = m_storeLastActor - m_storeFirstActor + 1;
+
+ int bytesAvailable = MAXIMUM_MESSAGE_SIZE_IN_BYTES;
+
+ m_bufferTotalSize = bytesAvailable;
+ m_storageActors = storageActors;
+
+ m_buffers = (char*) malloc(storageActors * bytesAvailable);
+ m_bufferSizes = (int*) malloc(storageActors * sizeof(int));
+
+ for(int i = 0 ; i < storageActors ; ++i) {
+ m_bufferSizes[i] = 0;
+ }
+ }
+
+ } else if(tag == StoreKeeper::PUSH_SAMPLE_VERTEX_OK) {
+
+ int producer = -1;
+ char * buffer = (char*) message.getBufferBytes();
+
+ memcpy(&producer, buffer, sizeof(producer));
+
+ int source = producer;
+
+ // this is a message from self ! how exciting...
+ if(source == getName()) {
+
+ flushAnyBuffer();
+
+ } else {
+
+ // respond to the producer now
+ Message response;
+ response.setTag(PAYLOAD_RESPONSE);
+ send(source, response);
+ }
+
+ //cout << "Resume reader 2" << endl;
+
+ } else if(tag == FLUSH_BUFFERS) {
+
+ // DONE !!! -> flush buffers at the end.
+
+ m_mother = source;
+
+ flushAnyBuffer();
+
+
+ }
+}
+
+void CoalescenceManager::flushAnyBuffer() {
+
+ bool flushRemainingBits = false;
+
+ flushRemainingBits = true; // !!!
+
+ for(int i = 0; i < m_storageActors; ++i) {
+
+ if(!flushRemainingBits)
+ break;
+
+ // storage actors are consecutive
+ int destination = i + m_storeFirstActor;
+
+ int bytes = getBufferUsedBytes(i);
+
+ if(bytes > 0) {
+
+ int fakeSource = getName();
+ flushBuffer(fakeSource, destination);
+
+ //m_bufferSizes[i] = 0;
+
+ return;
+ }
+ }
+
+
+ int source = m_mother;
+
+ Message response;
+ response.setTag(FLUSH_BUFFERS_OK);
+ send(source, response);
+}
+
+void CoalescenceManager::receivePayload(Message & message) {
+
+ int source = message.getSourceActor();
+
+ char * buffer = (char*)message.getBufferBytes();
+ //int bytes = message.getNumberOfBytes();
+
+ int position = 0;
+ Vertex vertex;
+ position += vertex.load(buffer + position);
+
+ int sample = -1;
+ memcpy(&sample, buffer + position, sizeof(sample));
+ position += sizeof(sample);
+
+ int producer = source;
+
+ if(!classifyKmerInBuffer(producer, sample, vertex)) {
+
+ Message response;
+ response.setTag(PAYLOAD_RESPONSE);
+ send(source, response);
+ //cout << "Resume reader 1" << endl;
+ }
+}
+
+bool CoalescenceManager::classifyKmerInBuffer(int producer, int & sample, Vertex & vertex) {
+
+ Kmer kmer = vertex.getKey();
+ int storageDestination = getVertexDestination(kmer);
+
+#if 0
+ printName();
+ int source = -999;
+
+ cout << "DEBUG/CoalescenceManager received PAYLOAD from " << source;
+ cout << " ";
+ vertex.print(m_kmerLength, m_colorSpaceMode);
+ cout << endl;
+
+
+ cout << "Destination -> " << storageDestination << endl;
+#endif
+
+ return addKmerInBuffer(producer, storageDestination, sample, vertex);
+}
+
+bool CoalescenceManager::addKmerInBuffer(int producer, int & actor, int & sample, Vertex & vertex) {
+
+ int actorIndex = actor - m_storeFirstActor;
+
+#ifdef CONFIG_ASSERT
+ assert(actorIndex < m_storageActors);
+#endif
+
+ char * buffer = getBuffer(actorIndex);
+ int offset = m_bufferSizes[actorIndex];
+
+ int requiredBytes = 0;
+ requiredBytes += vertex.getRequiredNumberOfBytes();
+ requiredBytes += sizeof(sample);
+
+ offset += vertex.dump(buffer + offset);
+ memcpy(buffer + offset, &sample, sizeof(sample));
+ offset += sizeof(sample);
+
+ m_bufferSizes[actorIndex] = offset;
+
+ // flush the message if no more bytes are available
+ int availableBytes = m_bufferTotalSize - offset;
+
+ // also reserve space for the producer
+ availableBytes -= sizeof(int);
+
+ if(availableBytes < requiredBytes) {
+
+ // store producer
+
+ flushBuffer(producer, actor);
+
+ return true;
+ }
+
+#if 0
+ printName();
+ cout << "sends bits to StoreKeeper # " << storageDestination;
+ cout << endl;
+#endif
+
+ return false;
+}
+
+char * CoalescenceManager::getBuffer(int actorIndex) {
+
+ return m_buffers + actorIndex * m_bufferTotalSize * sizeof(char);
+
+}
+
+int CoalescenceManager::getBufferUsedBytes(int index) {
+
+ int actorIndex = index;
+
+ return m_bufferSizes[actorIndex];
+}
+
+void CoalescenceManager::flushBuffer(int sourceActor, int destinationActor) {
+
+
+ int producer = sourceActor;
+ int actor = destinationActor;
+
+ int actorIndex = actor - m_storeFirstActor;
+
+#if 0
+ cout << "DEBUG flushBuffer SRC " << sourceActor << " DST ";
+ cout << " IDX " << destinationActor << endl;
+#endif
+
+ char * buffer = getBuffer(actorIndex);
+ int offset = getBufferUsedBytes(actorIndex);
+
+ memcpy(buffer + offset, &producer, sizeof(producer));
+ offset += sizeof(producer);
+
+ int bytes = offset;
+
+ /*
+ printName();
+ cout << "flushing data, sending stuff to " << actor << endl;
+ */
+
+ Message routedMessage;
+ routedMessage.setTag(StoreKeeper::PUSH_SAMPLE_VERTEX);
+ routedMessage.setBuffer(buffer);
+ routedMessage.setNumberOfBytes(bytes);
+
+ // free the buffer.
+ m_bufferSizes[actorIndex] = 0;
+
+ int storageDestination = actor;
+
+ // DONE: do some aggregation or something !
+ send(storageDestination, routedMessage);
+
+}
+
+int CoalescenceManager::getVertexDestination(Kmer & kmer) {
+
+ uint64_t hash = kmer.getTwinHash1(m_kmerLength, m_colorSpaceMode);
+
+ //cout << "DEBUG getTwinHash1 " << hash << endl;
+
+ int storageActors = m_storeLastActor - m_storeFirstActor + 1;
+
+ int actor = (hash % storageActors) + m_storeFirstActor;
+
+ return actor;
+}
diff --git a/code/Surveyor/CoalescenceManager.h b/code/Surveyor/CoalescenceManager.h
new file mode 100644
index 0000000..be64af1
--- /dev/null
+++ b/code/Surveyor/CoalescenceManager.h
@@ -0,0 +1,82 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of Ray Surveyor.
+
+ Ray Surveyor is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ Ray Surveyor is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Ray Surveyor. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#ifndef CoalescenceManagerHeader
+#define CoalescenceManagerHeader
+
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/VerticesExtractor/Vertex.h>
+
+#include <RayPlatform/actors/Actor.h>
+
+class CoalescenceManager : public Actor {
+
+private:
+
+ int m_mother;
+ int * m_bufferSizes;
+ char * m_buffers;
+ int m_bufferTotalSize;
+
+ int m_localStore;
+
+ int m_kmerLength;
+ bool m_colorSpaceMode;
+
+ int m_storeFirstActor;
+ int m_storeLastActor;
+ int m_storageActors;
+
+ int getVertexDestination(Kmer & kmer);
+
+ bool classifyKmerInBuffer(int producer, int & sample, Vertex & vertex);
+ bool addKmerInBuffer(int producer, int & actor, int & sample, Vertex & vertex);
+
+ char * getBuffer(int actorIndex);
+ void flushBuffer(int producer, int consumer);
+ void flushAnyBuffer();
+ int getBufferUsedBytes(int i);
+
+ void runAssertions();
+
+public:
+
+ CoalescenceManager();
+ ~CoalescenceManager();
+
+ void receive(Message & message);
+ void receivePayload(Message & message);
+
+ enum {
+ FIRST_TAG = 10150,
+ PAYLOAD,
+ PAYLOAD_RESPONSE,
+ SET_KMER_LENGTH,
+ SET_KMER_LENGTH_OK,
+ INTRODUCE_STORE,
+ FLUSH_BUFFERS,
+ FLUSH_BUFFERS_OK,
+ DIE,
+ LAST_TAG
+ };
+};
+
+#endif
diff --git a/code/Surveyor/ExperimentVertex.cpp b/code/Surveyor/ExperimentVertex.cpp
new file mode 100644
index 0000000..cc92e66
--- /dev/null
+++ b/code/Surveyor/ExperimentVertex.cpp
@@ -0,0 +1,49 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of Ray Surveyor.
+
+ Ray Surveyor is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ Ray Surveyor is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Ray Surveyor. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "ExperimentVertex.h"
+
+ExperimentVertex::ExperimentVertex() {
+
+}
+
+ExperimentVertex::~ExperimentVertex() {
+
+}
+
+Kmer ExperimentVertex::getKey() const {
+
+ return m_kmer;
+}
+
+void ExperimentVertex::setKey(Kmer & kmer) {
+
+ m_kmer = kmer;
+}
+
+void ExperimentVertex::setVirtualColor(VirtualKmerColorHandle handle) {
+
+ m_color = handle;
+}
+
+VirtualKmerColorHandle ExperimentVertex::getVirtualColor() const {
+
+ return m_color;
+}
diff --git a/code/Surveyor/ExperimentVertex.h b/code/Surveyor/ExperimentVertex.h
new file mode 100644
index 0000000..1d7e43e
--- /dev/null
+++ b/code/Surveyor/ExperimentVertex.h
@@ -0,0 +1,58 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of Ray Surveyor.
+
+ Ray Surveyor is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ Ray Surveyor is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Ray Surveyor. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#ifndef ExperimentVertexHeader
+#define ExperimentVertexHeader
+
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/Searcher/VirtualKmerColor.h>
+#include <code/Searcher/ColorSet.h>
+
+/**
+ * This class stores a vertex for the multi-sample
+ * analysis problem.
+ *
+ * \author Sébastien Boisvert
+ */
+class ExperimentVertex {
+
+
+private:
+
+ VirtualKmerColorHandle m_color;
+
+ // TODO: don't store kmers here since it is the key elsewhere already !
+ // Otherwise, this consumes more memory and this is not good.
+ Kmer m_kmer;
+
+public:
+
+ ExperimentVertex();
+ ~ExperimentVertex();
+
+ Kmer getKey() const;
+ void setKey(Kmer & kmer);
+
+ void setVirtualColor(VirtualKmerColorHandle handle);
+ VirtualKmerColorHandle getVirtualColor() const;
+};
+
+#endif
diff --git a/code/Surveyor/GenomeGraphReader.cpp b/code/Surveyor/GenomeGraphReader.cpp
new file mode 100644
index 0000000..f9be0bd
--- /dev/null
+++ b/code/Surveyor/GenomeGraphReader.cpp
@@ -0,0 +1,251 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of Ray Surveyor.
+
+ Ray Surveyor is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ Ray Surveyor is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Ray Surveyor. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+// TODO: validate that the kmer length is the same for this file
+// and the provided -k argument
+
+#include "GenomeGraphReader.h"
+#include "CoalescenceManager.h"
+
+#include <code/Mock/constants.h>
+#include <code/Mock/common_functions.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/VerticesExtractor/Vertex.h>
+
+#include <iostream>
+#include <sstream>
+using namespace std;
+
+#include <string.h>
+
+GenomeGraphReader::GenomeGraphReader() {
+
+}
+
+GenomeGraphReader::~GenomeGraphReader() {
+
+}
+
+void GenomeGraphReader::receive(Message & message) {
+
+ int type = message.getTag();
+
+ /*
+ printName();
+ cout << "received tag " << type << endl;
+*/
+
+ if(type == START_PARTY) {
+
+ startParty(message);
+
+ } else if(type == CoalescenceManager::PAYLOAD_RESPONSE) {
+
+ /*
+ printName();
+ cout << " DEBUG readLine because PAYLOAD_RESPONSE" << endl;
+ */
+ // read the next line now !
+ readLine();
+ }
+}
+
+void GenomeGraphReader::startParty(Message & message) {
+
+ char * buffer = (char*) message.getBufferBytes();
+
+ memcpy(&m_aggregator, buffer, sizeof(int));
+ //m_aggregator = *(int*)(message.getBufferBytes());
+
+ m_reader.open(m_fileName.c_str());
+ m_loaded = 0;
+
+ m_parent = message.getSourceActor();
+
+ /*
+ printName();
+ cout << "DEBUG startParty" << endl;
+ cout << " bytes in message: " << message.getNumberOfBytes();
+ cout << " must send messages to aggregator " << m_aggregator;
+ cout << endl;
+*/
+
+ int source = message.getSourceActor();
+ Message response;
+ response.setTag(START_PARTY_OK);
+
+ send(source, response);
+
+ readLine();
+}
+
+// DONE 2013-10-16: add a BufferedLineReader class in RayPlatform
+// and use it here.
+void GenomeGraphReader::readLine() {
+
+ char buffer[1024];
+ buffer[0] = '\0';
+
+ while(!m_reader.eof()) {
+ m_reader.getline(buffer, 1024);
+
+ // skip comment
+ if(strlen(buffer) > 0 && buffer[0] == '#')
+ continue;
+
+ break;
+ }
+
+ if(m_reader.eof()) {
+
+ m_reader.close();
+
+ printName();
+ cout << " finished reading file " << m_fileName;
+ cout << " got " << m_loaded << " objects" << endl;
+
+ Message finishedMessage;
+ finishedMessage.setTag(DONE);
+
+ send(m_parent, finishedMessage);
+
+ die();
+ } else {
+
+ // AGCTGTGAAACTGGTGCAAGCTACCAGAATC;36;A;C
+ string sequence;
+ CoverageDepth coverage;
+ string parents;
+ string children;
+
+ for(int i = 0 ; i < (int) strlen(buffer) ; ++i) {
+ if(buffer[i] == ';')
+ buffer[i] = ' ';
+ }
+
+ istringstream stringBuffer(buffer);
+
+ stringBuffer >> sequence;
+ stringBuffer >> coverage;
+ stringBuffer >> parents;
+ stringBuffer >> children;
+
+#if 0
+ cout << "DEBUG " << sequence << " with " << coverage << endl;
+#endif
+
+ // if this is the first one, send the k-mer length too
+ if(m_loaded == 0) {
+
+ Message aMessage;
+ aMessage.setTag(CoalescenceManager::SET_KMER_LENGTH);
+
+ int length = sequence.length();
+ aMessage.setBuffer(&length);
+ aMessage.setNumberOfBytes(sizeof(length));
+
+ send(m_aggregator, aMessage);
+ }
+
+ Kmer kmer;
+ kmer.loadFromTextRepresentation(sequence.c_str());
+
+ Vertex vertex;
+ vertex.setKey(kmer);
+ vertex.setCoverageValue(coverage);
+
+ // add parents
+ for(int i = 0 ; i < (int)parents.length() ; ++i) {
+
+ string parent = sequence;
+ for(int j = 0 ; j < (int) parent.length()-1 ; ++j) {
+ parent[j + 1] = parent[j];
+ }
+ parent[0] = parents[i];
+
+ Kmer parentKmer;
+ parentKmer.loadFromTextRepresentation(parent.c_str());
+
+ vertex.addIngoingEdge(&kmer, &parentKmer, sequence.length());
+ }
+
+ // add children
+ for(int i = 0 ; i < (int)children.length() ; ++i) {
+
+ string child = sequence;
+ for(int j = 0 ; j < (int) child.length()-1 ; ++j) {
+ child[j] = child[j + 1];
+ }
+ child[child.length() - 1] = children[i];
+
+ Kmer childKmer;
+ childKmer.loadFromTextRepresentation(child.c_str());
+
+ vertex.addOutgoingEdge(&kmer, &childKmer, sequence.length());
+ }
+
+ char messageBuffer[100];
+ int position = 0;
+
+ position += vertex.dump(messageBuffer + position);
+ memcpy(messageBuffer + position, &m_sample, sizeof(m_sample));
+ position += sizeof(m_sample);
+
+// maybe: accumulate many objects before flushing it.
+// we can go up to MAXIMUM_MESSAGE_SIZE_IN_BYTES bytes.
+
+ /*
+ printName();
+ cout << " got data line " << buffer;
+ cout << " sending PAYLOAD to " << m_aggregator << endl;
+*/
+ Message message;
+ message.setTag(CoalescenceManager::PAYLOAD);
+ message.setBuffer(messageBuffer);
+ message.setNumberOfBytes(position);
+
+#if 0
+ printName();
+ cout << "DEBUG sending PAYLOAD to " << m_aggregator;
+ cout << " with " << position << " bytes ";
+ vertex.print(sequence.length(), false);
+ cout << endl;
+#endif
+
+ if(m_loaded % 1000000 == 0) {
+ printName();
+ cout << " loaded " << m_loaded << " sequences" << endl;
+
+ }
+ m_loaded ++;
+ send(m_aggregator, message);
+ }
+}
+
+void GenomeGraphReader::setFileName(string & fileName, int sample) {
+
+ m_sample = sample;
+ m_fileName = fileName;
+
+#if 0
+ printName();
+ cout << " DEBUG setFileName " << m_fileName << endl;
+#endif
+}
diff --git a/code/Surveyor/GenomeGraphReader.h b/code/Surveyor/GenomeGraphReader.h
new file mode 100644
index 0000000..93418c1
--- /dev/null
+++ b/code/Surveyor/GenomeGraphReader.h
@@ -0,0 +1,75 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of Ray Surveyor.
+
+ Ray Surveyor is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ Ray Surveyor is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Ray Surveyor. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#ifndef GenomeGraphReaderHeader
+#define GenomeGraphReaderHeader
+
+#include <RayPlatform/actors/Actor.h>
+#include <RayPlatform/files/FileReader.h>
+
+
+#include <string>
+#include <fstream>
+using namespace std;
+
+#define I_LIKE_FAST_IO
+
+class GenomeGraphReader: public Actor {
+
+private:
+ int m_sample;
+ int m_loaded;
+
+#ifdef I_LIKE_FAST_IO
+
+ // fast IO using a wrapper.
+ FileReader m_reader;
+#else
+
+ ifstream m_reader;
+#endif
+
+ string m_fileName;
+
+ int m_aggregator;
+ int m_parent;
+
+ void startParty(Message & message);
+
+public:
+
+ enum {
+ FIRST_TAG = 10200,
+ START_PARTY,
+ START_PARTY_OK,
+ DONE,
+ LAST_TAG
+ };
+
+
+ GenomeGraphReader();
+ ~GenomeGraphReader();
+ void receive(Message & message);
+ void readLine();
+ void setFileName(string & fileName, int sample);
+};
+
+#endif
diff --git a/code/Surveyor/Makefile b/code/Surveyor/Makefile
new file mode 100644
index 0000000..37125ed
--- /dev/null
+++ b/code/Surveyor/Makefile
@@ -0,0 +1,8 @@
+Surveyor-y += code/Surveyor/Mother.o
+Surveyor-y += code/Surveyor/StoreKeeper.o
+Surveyor-y += code/Surveyor/GenomeGraphReader.o
+Surveyor-y += code/Surveyor/CoalescenceManager.o
+Surveyor-y += code/Surveyor/ExperimentVertex.o
+Surveyor-y += code/Surveyor/MatrixOwner.o
+
+obj-y += $(Surveyor-y)
diff --git a/code/Surveyor/MatrixOwner.cpp b/code/Surveyor/MatrixOwner.cpp
new file mode 100644
index 0000000..83b384c
--- /dev/null
+++ b/code/Surveyor/MatrixOwner.cpp
@@ -0,0 +1,232 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of Ray Surveyor.
+
+ Ray Surveyor is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ Ray Surveyor is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Ray Surveyor. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "MatrixOwner.h"
+#include "CoalescenceManager.h" // for DIE
+
+#include <RayPlatform/core/OperatingSystem.h>
+
+#include <fstream>
+#include <string>
+#include <sstream>
+using namespace std;
+
+#include <math.h>
+
+MatrixOwner::MatrixOwner() {
+
+ m_completedStoreActors = 0;
+}
+
+MatrixOwner::~MatrixOwner() {
+
+}
+
+
+void MatrixOwner::receive(Message & message) {
+
+ int tag = message.getTag();
+ char * buffer = message.getBufferBytes();
+ int source = message.getSourceActor();
+
+ if( tag == CoalescenceManager::DIE) {
+
+ die();
+
+ } else if(tag == GREETINGS) {
+
+ int offset = 0;
+ memcpy(&m_parameters, buffer + offset, sizeof(m_parameters));
+ offset += sizeof(m_parameters);
+ memcpy(&m_sampleNames, buffer + offset, sizeof(m_sampleNames));
+ offset += sizeof(m_sampleNames);
+
+#ifdef CONFIG_ASSERT
+ assert(m_parameters != NULL);
+ assert(m_sampleNames != NULL);
+#endif
+
+ m_mother = source;
+
+ } else if(tag == PUSH_PAYLOAD) {
+
+ SampleIdentifier sample1 = -1;
+ SampleIdentifier sample2 = -1;
+ LargeCount count = 0;
+
+ int offset = 0;
+
+ memcpy(&sample1, buffer + offset, sizeof(sample1));
+ offset += sizeof(sample1);
+ memcpy(&sample2, buffer + offset, sizeof(sample2));
+ offset += sizeof(sample2);
+ memcpy(&count, buffer + offset, sizeof(count));
+ offset += sizeof(count);
+
+#ifdef CONFIG_ASSERT
+ assert(sample1 >= 0);
+ assert(sample2 >= 0);
+ assert(count >= 0);
+#endif
+
+ /*
+ printName();
+ cout << "DEBUG add " << sample1 << " " << sample2 << " " << count << endl;
+*/
+
+ m_localGramMatrix[sample1][sample2] += count;
+
+ Message response;
+ response.setTag(PUSH_PAYLOAD_OK);
+ send(source, response);
+
+ } else if(tag == PUSH_PAYLOAD_END) {
+
+ m_completedStoreActors++;
+
+ if(m_completedStoreActors == getSize()) {
+
+
+ // create directory for Surveyor
+ ostringstream matrixFile;
+ matrixFile << m_parameters->getPrefix() << "/Surveyor/";
+ string surveyorDirectory = matrixFile.str();
+ createDirectory(surveyorDirectory.c_str());
+
+ // create SimilarityMatrix
+ matrixFile << "SimilarityMatrix.tsv";
+
+ string similarityMatrix = matrixFile.str();
+ ofstream similarityFile;
+ similarityFile.open(similarityMatrix.c_str());
+ printLocalGramMatrix(similarityFile, m_localGramMatrix);
+ similarityFile.close();
+
+ printName();
+ cout << "MatrixOwner printed the Similarity Matrix: ";
+ cout << similarityMatrix << endl;
+
+ // create DistanceMatrix
+
+ computeDistanceMatrix();
+
+ ostringstream matrixFileForDistances;
+ matrixFileForDistances << m_parameters->getPrefix() << "/Surveyor/";
+ matrixFileForDistances << "DistanceMatrix.tsv";
+
+
+ string distanceMatrix = matrixFileForDistances.str();
+ ofstream distanceFile;
+ distanceFile.open(distanceMatrix.c_str());
+ printLocalGramMatrix(distanceFile, m_kernelDistanceMatrix);
+ distanceFile.close();
+
+ printName();
+ cout << "MatrixOwner printed the Distance Matrix: ";
+ cout << distanceMatrix << endl;
+
+
+ // tell Mother that the matrix is ready now.
+
+ Message coolMessage;
+ coolMessage.setTag(MATRIX_IS_READY);
+ send(m_mother, coolMessage);
+
+
+ // clear matrices
+
+ m_localGramMatrix.clear();
+ m_kernelDistanceMatrix.clear();
+ }
+ }
+}
+
+
+// TODO: save time by only computing the lower triangle.
+void MatrixOwner::computeDistanceMatrix() {
+
+ for(map<SampleIdentifier, map<SampleIdentifier, LargeCount> >::iterator row = m_localGramMatrix.begin();
+ row != m_localGramMatrix.end(); ++row) {
+
+ SampleIdentifier sample1 = row->first;
+
+ for(map<SampleIdentifier, LargeCount>::iterator cell = row->second.begin();
+ cell != row->second.end(); ++cell) {
+
+ SampleIdentifier sample2 = cell->first;
+
+ // d(x, x') = sqrt( k(x,x) + k(x', x') - 2 k (x, x'))
+ LargeCount distance = 0;
+ distance += m_localGramMatrix[sample1][sample1];
+ distance += m_localGramMatrix[sample2][sample2];
+ distance -= 2 * m_localGramMatrix[sample1][sample2];
+
+ distance = (LargeCount) sqrt((double)distance);
+
+ m_kernelDistanceMatrix[sample1][sample2] = distance;
+ m_kernelDistanceMatrix[sample2][sample1] = distance;
+
+ }
+
+ }
+}
+
+/**
+ * Write it in RaySurveyorResults/SurveyorMatrix.tsv
+ * Also write a distance matrix too !
+ */
+void MatrixOwner::printLocalGramMatrix(ostream & stream, map<SampleIdentifier, map<SampleIdentifier, LargeCount> > & matrix) {
+
+ /*
+ printName();
+ cout << "Local Gram Matrix: " << endl;
+ cout << endl;
+ */
+
+ for(map<SampleIdentifier, map<SampleIdentifier, LargeCount> >::iterator column = matrix.begin();
+ column != matrix.end(); ++column) {
+
+ SampleIdentifier sample = column->first;
+
+ stream << " " << m_sampleNames->at(sample);
+ }
+
+ stream << endl;
+
+ for(map<SampleIdentifier, map<SampleIdentifier, LargeCount> >::iterator row = matrix.begin();
+ row != matrix.end(); ++row) {
+
+ SampleIdentifier sample1 = row->first;
+
+ stream << m_sampleNames->at(sample1);
+
+ for(map<SampleIdentifier, LargeCount>::iterator cell = row->second.begin();
+ cell != row->second.end(); ++cell) {
+
+ //SampleIdentifier sample2 = cell->first;
+
+ LargeCount hits = cell->second;
+
+ stream << " " << hits;
+ }
+
+ stream << endl;
+ }
+}
diff --git a/code/Surveyor/MatrixOwner.h b/code/Surveyor/MatrixOwner.h
new file mode 100644
index 0000000..a6d691c
--- /dev/null
+++ b/code/Surveyor/MatrixOwner.h
@@ -0,0 +1,68 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of Ray Surveyor.
+
+ Ray Surveyor is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ Ray Surveyor is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Ray Surveyor. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef MatrixOwnerHeader
+#define MatrixOwnerHeader
+
+#include <code/Mock/constants.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/actors/Actor.h>
+
+#include <map>
+#include <iostream>
+using namespace std;
+
+class MatrixOwner : public Actor {
+private:
+
+ Parameters * m_parameters;
+ vector<string> * m_sampleNames;
+
+ map<SampleIdentifier, map<SampleIdentifier, LargeCount> > m_localGramMatrix;
+ map<SampleIdentifier, map<SampleIdentifier, LargeCount> > m_kernelDistanceMatrix;
+
+ int m_mother;
+ int m_completedStoreActors;
+ void printLocalGramMatrix(ostream & stream, map<SampleIdentifier, map<SampleIdentifier, LargeCount> > & matrix);
+
+ void computeDistanceMatrix();
+
+public:
+
+ MatrixOwner();
+ ~MatrixOwner();
+
+ void receive(Message & message);
+
+ enum {
+ FIRST_TAG = 10300,
+ GREETINGS,
+ PUSH_PAYLOAD,
+ PUSH_PAYLOAD_OK,
+ PUSH_PAYLOAD_END,
+ MATRIX_IS_READY,
+ LAST_TAG
+ };
+
+};
+
+#endif
+
diff --git a/code/Surveyor/Mother.cpp b/code/Surveyor/Mother.cpp
new file mode 100644
index 0000000..f15e161
--- /dev/null
+++ b/code/Surveyor/Mother.cpp
@@ -0,0 +1,482 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of Ray Surveyor.
+
+ Ray Surveyor is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ Ray Surveyor is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Ray Surveyor. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+// Written by <<sebhtml>>
+// 2013-10-10
+
+#include "Mother.h"
+#include "StoreKeeper.h"
+#include "GenomeGraphReader.h"
+#include "MatrixOwner.h"
+
+#include <RayPlatform/cryptography/crypto.h>
+
+#include <iostream>
+using namespace std;
+
+#define PLAN_RANK_ACTORS_PER_RANK 1
+#define PLAN_MOTHER_ACTORS_PER_RANK 1
+#define PLAN_GENOME_GRAPH_READER_ACTORS_PER_RANK 999999
+
+
+
+Mother::Mother() {
+
+ m_coalescenceManager = -1;
+ m_matrixOwner = -1;
+
+ m_parameters = NULL;
+ m_bigMother = -1;
+
+ m_finishedMothers = 0;
+ //cout << "DEBUG Mother constructor" << endl;
+}
+
+Mother::~Mother() {
+
+}
+
+void Mother::receive(Message & message) {
+
+ int tag = message.getTag();
+ int source = message.getSourceActor();
+ char * buffer = message.getBufferBytes();
+
+ if(tag == Actor::BOOT) {
+
+ boot(message);
+ } else if (tag == Mother::HELLO) {
+ hello(message);
+
+ } else if(tag == GenomeGraphReader::START_PARTY_OK) {
+
+ // spawn the next reader now !
+
+ /*
+ printName();
+ cout << "DEBUG spawnReader because START_PARTY_OK" << endl;
+*/
+ spawnReader();
+
+ } else if(tag == GenomeGraphReader::DONE) {
+
+ m_aliveReaders--;
+
+ //cout << "DEBUG received GenomeGraphReader::DONE remaining " << m_aliveReaders << endl;
+
+ if(m_aliveReaders == 0) {
+
+ notifyController();
+ }
+
+ } else if(tag == MERGE) {
+
+ int matrixOwner = -1;
+ memcpy(&matrixOwner, buffer, sizeof(matrixOwner));
+
+#ifdef CONFIG_ASSERT
+ assert(matrixOwner >= 0);
+ assert(m_storeKeepers.size() == 1);
+#endif
+
+ Message theMessage;
+ theMessage.setTag(StoreKeeper::MERGE);
+ theMessage.setBuffer(&matrixOwner);
+ theMessage.setNumberOfBytes(sizeof(matrixOwner));
+
+ int destination = m_storeKeepers[0];
+
+ send(destination, theMessage);
+
+ Message response;
+ response.setTag(MERGE_OK);
+ send(source, response);
+
+ } else if(tag == SHUTDOWN) {
+
+ Message response;
+ response.setTag(SHUTDOWN_OK);
+ send(message.getSourceActor(), response);
+
+ stop();
+
+ } else if(tag == StoreKeeper::MERGE_OK) {
+
+ /*
+ Message newMessage;
+ newMessage.setTag(MERGE_OK);
+
+ send(m_bigMother, newMessage);
+ */
+
+ } else if(tag == FINISH_JOB) {
+
+ m_finishedMothers++;
+
+ if(m_finishedMothers == getSize()) {
+
+ // all readers have finished,
+ // now tell mother to flush aggregators
+
+ sendToFirstMother(FLUSH_AGGREGATOR, FLUSH_AGGREGATOR_OK);
+ }
+ } else if(tag == FLUSH_AGGREGATOR) {
+
+ m_bigMother = source;
+
+ // forward the message to the aggregator
+
+ Message newMessage;
+ newMessage.setTag(CoalescenceManager::FLUSH_BUFFERS);
+ send(m_coalescenceManager, newMessage);
+
+ } else if(tag == CoalescenceManager::FLUSH_BUFFERS_OK) {
+
+ Message response;
+ response.setTag(FLUSH_AGGREGATOR_OK);
+ send(m_bigMother, response);
+
+ } else if(tag == m_responseTag) {
+
+ // every mother was informed.
+ if(m_motherToKill < getSize()) {
+
+ if(m_responseTag == SHUTDOWN_OK) {
+ return;
+
+ } else if(m_responseTag == FLUSH_AGGREGATOR_OK) {
+
+ // spawn the MatrixOwner here !
+
+ MatrixOwner * matrixOwner = new MatrixOwner();
+ spawn(matrixOwner);
+
+ m_matrixOwner = matrixOwner->getName();
+
+ printName();
+ cout << "Spawned MatrixOwner actor !" << endl;
+
+ // tell the StoreKeeper actors to send their stuff to the
+ // MatrixOwner actor
+ // The Mother of Mother will wait for a signal from MatrixOwner
+
+ Message greetingMessage;
+
+ vector<string> * names = & m_sampleNames;
+
+ char buffer[32];
+ int offset = 0;
+ memcpy(buffer + offset, &m_parameters, sizeof(m_parameters));
+ offset += sizeof(m_parameters);
+ memcpy(buffer + offset, &names, sizeof(names));
+ offset += sizeof(names);
+
+ greetingMessage.setBuffer(&buffer);
+ greetingMessage.setNumberOfBytes(offset);
+
+ greetingMessage.setTag(MatrixOwner::GREETINGS);
+ send(m_matrixOwner, greetingMessage);
+
+ sendToFirstMother(MERGE, MERGE_OK);
+
+ } else if(m_responseTag == MERGE_OK) {
+
+ }
+ }
+
+ sendMessageWithReply(m_motherToKill, m_forwardTag);
+ m_motherToKill--;
+
+ } else if(tag == MatrixOwner::MATRIX_IS_READY) {
+
+ sendToFirstMother(SHUTDOWN, SHUTDOWN_OK);
+ }
+}
+
+void Mother::sendToFirstMother(int forwardTag, int responseTag) {
+
+ m_forwardTag = forwardTag;
+ m_responseTag = responseTag;
+
+ m_motherToKill = 2 * getSize() - 1;
+
+ sendMessageWithReply(m_motherToKill, forwardTag);
+ m_motherToKill--;
+}
+
+void Mother::sendMessageWithReply(int & actor, int tag) {
+/*
+ printName();
+ cout << "kills Mother " << actor << endl;
+ */
+
+ Message message;
+ message.setTag(tag);
+
+ if(tag == MERGE) {
+ message.setBuffer(&m_matrixOwner);
+ message.setNumberOfBytes(sizeof(m_matrixOwner));
+ }
+
+ send(actor, message);
+}
+
+void Mother::notifyController() {
+ Message message2;
+ message2.setTag(FINISH_JOB);
+
+ // first Mother
+ int controller = getSize();
+
+ printName();
+ cout << "Mother notifies controller " << controller << endl;
+ send(controller, message2);
+
+}
+
+void Mother::stop() {
+
+ Message kill;
+ kill.setTag(CoalescenceManager::DIE);
+
+ send(m_coalescenceManager, kill);
+ m_coalescenceManager = -1;
+
+ send(m_storeKeepers[0], kill);
+ m_storeKeepers.clear();
+
+ if(m_matrixOwner >= 0) {
+ send(m_matrixOwner, kill);
+ m_matrixOwner = -1;
+ }
+
+
+ die();
+
+}
+
+void Mother::hello(Message & message) {
+ /*
+ printName();
+ cout << "received HELLO from ";
+ cout << message.getSourceActor();
+ cout << " bytes: " << message.getNumberOfBytes();
+ cout << " content: " << *((int*) message.getBufferBytes());
+ */
+
+ //char * buffer = (char*) message.getBufferBytes();
+ /*
+ uint32_t checksum = computeCyclicRedundancyCode32((uint8_t*) message.getBufferBytes(),
+ message.getNumberOfBytes());
+ */
+ //cout << "DEBUG CRC32= " << checksum << endl;
+
+ //cout << endl;
+}
+
+void Mother::boot(Message & message) {
+
+ m_aliveReaders = 0;
+
+ /*
+ printName();
+ cout << "Mother is booting and says hello" << endl;
+*/
+ Message message2;
+ /*
+ char joe[4000];
+
+ int i = 4000;
+ char value = 0;
+ while(i)
+ joe[i--]=value++;
+*/
+
+ int joe = 9921;
+
+ message2.setBuffer(&joe);
+ //message2.setNumberOfBytes(4000);
+ message2.setNumberOfBytes( sizeof(int) * 1 );
+
+ //cout << "DEBUG sending " << joe << endl;
+
+ /*
+ uint32_t checksum = computeCyclicRedundancyCode32((uint8_t*) message2.getBufferBytes(),
+ message2.getNumberOfBytes() );
+ cout << "DEBUG CRC32= " << checksum << endl;
+*/
+
+ message2.setTag(Mother::HELLO);
+
+ int next = getName() + 1;
+
+ next %= (getSize() * 2);
+
+ if(next < getSize())
+ next += getSize();
+
+ printName();
+ cout << " local Mother is " << getName() << ",";
+ cout << " friend is # " << next << endl;
+
+ send(next, message2);
+
+ if(m_parameters->hasOption("-run-surveyor")) {
+ startSurveyor();
+ } else {
+ die();
+ }
+}
+
+void Mother::startSurveyor() {
+
+ bool isRoot = (getName() % getSize()) == 0;
+
+ //cout << "DEBUG startSurveyor isRoot" << isRoot << endl;
+
+ // get a list of files.
+
+ vector<string> * commands = m_parameters->getCommands();
+
+ for(int i = 0 ; i < (int) commands->size() ; ++i) {
+
+ string & element = commands->at(i);
+
+ if(element == "-read-sample-graph") {
+
+ string sampleName = commands->at(++i);
+ string fileName = commands->at(++i);
+
+ m_sampleNames.push_back(sampleName);
+ m_graphFileNames.push_back(fileName);
+ }
+ }
+
+ if(isRoot) {
+ printName();
+ cout << "samples= " << m_sampleNames.size() << endl;
+ }
+
+
+ CoalescenceManager * coalescenceManager = new CoalescenceManager();
+ spawn(coalescenceManager);
+
+ m_coalescenceManager = coalescenceManager->getName();
+
+ // spawn the local store keeper and introduce the CoalescenceManager
+ // to the StoreKeeper
+
+ // spawn actors for storing the graph.
+ for(int i = 0 ; i < PLAN_STORE_KEEPER_ACTORS_PER_RANK; ++i) {
+
+ StoreKeeper * actor = new StoreKeeper();
+ spawn(actor);
+
+ m_storeKeepers.push_back(actor->getName());
+
+ // tell the CoalescenceManager about the local StoreKeeper
+ Message dummyMessage;
+ int localStore = actor->getName();
+
+ dummyMessage.setBuffer(&localStore);
+ dummyMessage.setNumberOfBytes( sizeof(int) );
+
+ dummyMessage.setTag(CoalescenceManager::INTRODUCE_STORE);
+
+ send(m_coalescenceManager, dummyMessage);
+
+ int kmerLength = m_parameters->getWordSize();
+
+ // send the kmer to local store
+ Message kmerMessage;
+ kmerMessage.setBuffer(&kmerLength);
+ kmerMessage.setNumberOfBytes(sizeof(kmerLength));
+ kmerMessage.setTag(CoalescenceManager::SET_KMER_LENGTH);
+ send(localStore, kmerMessage);
+
+ }
+
+
+ // spawn an actor for each file that this actor owns
+
+ for(int i = 0; i < (int) m_graphFileNames.size() ; ++i) {
+
+ int mother = getName() % getSize();
+ int fileMother = i % getSize();
+
+ if(mother == fileMother) {
+
+ m_filesToSpawn.push_back(i);
+
+ }
+ }
+
+ printName();
+ cout << " readers to spawn: " << m_filesToSpawn.size() << endl;
+
+ m_fileIterator = 0;
+ spawnReader();
+}
+
+void Mother::spawnReader() {
+
+ if(m_fileIterator < (int) m_filesToSpawn.size()) {
+
+ int sampleIdentifier = m_filesToSpawn[m_fileIterator];
+
+ string & fileName = m_graphFileNames[sampleIdentifier];
+ m_fileIterator++;
+
+ GenomeGraphReader * actor = new GenomeGraphReader();
+ spawn(actor);
+ actor->setFileName(fileName, sampleIdentifier);
+
+ int destination = actor->getName();
+ Message dummyMessage;
+
+ int coalescenceManagerName = m_coalescenceManager;
+ //cout << "DEBUG coalescenceManagerName is " << coalescenceManagerName << endl;
+
+ dummyMessage.setBuffer(&coalescenceManagerName);
+ dummyMessage.setNumberOfBytes( sizeof(int) );
+
+ dummyMessage.setTag(GenomeGraphReader::START_PARTY);
+
+ /*
+ printName();
+ cout << " sending START_PARTY " << GenomeGraphReader::START_PARTY;
+ cout << " to " << destination << endl;
+*/
+
+ m_aliveReaders ++;
+
+ send(destination, dummyMessage);
+ }
+
+ if(m_aliveReaders == 0) {
+
+ notifyController();
+ }
+}
+
+void Mother::setParameters(Parameters * parameters) {
+ m_parameters = parameters;
+}
diff --git a/code/Surveyor/Mother.h b/code/Surveyor/Mother.h
new file mode 100644
index 0000000..d45055f
--- /dev/null
+++ b/code/Surveyor/Mother.h
@@ -0,0 +1,118 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of Ray Surveyor.
+
+ Ray Surveyor is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ Ray Surveyor is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Ray Surveyor. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef MotherHeader
+#define MotherHeader
+
+#include "CoalescenceManager.h"
+
+#include <code/Mock/Parameters.h>
+#include <RayPlatform/actors/Actor.h>
+
+#include <vector>
+#include <string>
+using namespace std;
+
+/**
+ *
+ * Map (assuming N ranks)
+ *
+ * The following map is not used anymore because it is stupid.
+ * ----------------------------------------------------------
+ * Actors Quantity Role
+ * Start End
+ * ----------------------------------------------------------
+ * 0 N - 1 N ComputeCore
+ * N 2N - 1 N Mother
+ * 2N 102N - 1 100N StoreKeeper
+ * 102N 104N - 1 2N GenomeGraphReader
+ * ----------------------------------------------------------
+ *
+ * It would be nice to have a number of tokens / second that can be exchanged and also
+ * the point-to-point latency for this actor implementation.
+ *
+ * \author Sébastien Boisvert
+ */
+class Mother: public Actor {
+
+private:
+
+ int m_matrixOwner;
+
+ int m_finishedMothers;
+
+ Parameters * m_parameters;
+
+ int m_coalescenceManager;
+ int m_fileIterator;
+ vector<int> m_filesToSpawn;
+
+ vector<int> m_storeKeepers;
+
+ vector<int> m_readers;
+
+ vector<string> m_sampleNames;
+ vector<string> m_graphFileNames;
+
+ int m_bigMother;
+ int m_aliveReaders;
+ int m_motherToKill;
+
+ int m_forwardTag;
+ int m_responseTag;
+
+ void spawnReader();
+ void startSurveyor();
+ void hello(Message & message);
+ void boot(Message & message);
+ void stop();
+ void notifyController();
+ void sendMessageWithReply(int & actor, int tag);
+
+ /*
+ * Send tag to all mothers.
+ */
+ void sendToFirstMother(int forwardTag, int responseTag);
+
+public:
+
+ Mother();
+ ~Mother();
+
+ void receive(Message & message);
+
+ enum {
+ FIRST_TAG = 10100,
+ HELLO,
+ FINISH_JOB,
+ SHUTDOWN,
+ SHUTDOWN_OK,
+ FLUSH_AGGREGATOR,
+ FLUSH_AGGREGATOR_OK,
+ LAST_TAG,
+ MERGE,
+ MERGE_OK
+ };
+
+ void setParameters(Parameters * parameters);
+};
+
+#endif
+
diff --git a/code/Surveyor/StoreKeeper.cpp b/code/Surveyor/StoreKeeper.cpp
new file mode 100644
index 0000000..5c14984
--- /dev/null
+++ b/code/Surveyor/StoreKeeper.cpp
@@ -0,0 +1,460 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of Ray Surveyor.
+
+ Ray Surveyor is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ Ray Surveyor is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Ray Surveyor. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#include "StoreKeeper.h"
+#include "CoalescenceManager.h"
+#include "MatrixOwner.h"
+
+#include <code/VerticesExtractor/Vertex.h>
+
+#include <iostream>
+using namespace std;
+
+#include <string.h>
+
+#include <assert.h>
+
+StoreKeeper::StoreKeeper() {
+
+ m_receivedObjects = 0;
+
+ m_configured = false;
+ m_kmerLength = 0;
+}
+
+StoreKeeper::~StoreKeeper() {
+
+}
+
+void StoreKeeper::receive(Message & message) {
+
+ int tag = message.getTag();
+ int source = message.getSourceActor();
+ char * buffer = message.getBufferBytes();
+
+ if(!m_configured)
+ configureHashTable();
+
+ if(tag == PUSH_SAMPLE_VERTEX) {
+ pushSampleVertex(message);
+
+ } else if( tag == CoalescenceManager::DIE) {
+
+ printName();
+ cout << "(StoreKeeper) received " << m_receivedObjects << " objects in total" << endl;
+
+ // * 2 because we store pairs
+ uint64_t size = m_hashTable.size() * 2;
+
+ printName();
+ cout << "has " << size << " Kmer objects in MyHashTable instance (final)" << endl;
+
+
+ printName();
+ cout << "will now die (StoreKeeper)" << endl;
+
+ die();
+
+ } else if(tag == MERGE) {
+
+ computeLocalGramMatrix();
+
+ m_mother = source;
+
+ memcpy(&m_matrixOwner, buffer, sizeof(m_matrixOwner));
+
+ /*
+ printName();
+ cout << "DEBUG m_matrixOwner " << m_matrixOwner << endl;
+*/
+
+ m_iterator1 = m_localGramMatrix.begin();
+
+ if(m_iterator1 != m_localGramMatrix.end()) {
+
+ m_iterator2 = m_iterator1->second.begin();
+ }
+
+ /*
+ printName();
+ cout << "DEBUG printLocalGramMatrix before first sendMatrixCell" << endl;
+ printLocalGramMatrix();
+ */
+
+ sendMatrixCell();
+
+ } else if(tag == MatrixOwner::PUSH_PAYLOAD_OK) {
+
+ sendMatrixCell();
+
+ } else if(tag == CoalescenceManager::SET_KMER_LENGTH) {
+
+ int kmerLength = 0;
+ int position = 0;
+ char * buffer = (char*)message.getBufferBytes();
+ memcpy(&kmerLength, buffer + position, sizeof(kmerLength));
+ position += sizeof(kmerLength);
+
+ if(m_kmerLength == 0)
+ m_kmerLength = kmerLength;
+
+ if(kmerLength != m_kmerLength) {
+
+ cout << "Error: the k-mer value is different this time !" << endl;
+ }
+
+ // cout << "DEBUG m_kmerLength = " << m_kmerLength << endl;
+
+ // the color space mode is an artefact.
+ m_colorSpaceMode = false;
+
+#if 0
+ cout << "DEBUG StoreKeeper SET_KMER_LENGTH ";
+ cout << m_kmerLength;
+ cout << endl;
+#endif
+
+ /*
+ memcpy(&m_parameters, buffer + position, sizeof(m_parameters));
+ position += sizeof(m_parameters);
+
+ */
+ //configureHashTable();
+
+ }
+}
+
+void StoreKeeper::sendMatrixCell() {
+
+ if(m_iterator1 != m_localGramMatrix.end()) {
+
+ if(m_iterator2 != m_iterator1->second.end()) {
+
+ SampleIdentifier sample1 = m_iterator1->first;
+ SampleIdentifier sample2 = m_iterator2->first;
+ LargeCount count = m_iterator2->second;
+
+ Message message;
+ char buffer[20];
+ int offset = 0;
+ memcpy(buffer + offset, &sample1, sizeof(sample1));
+ offset += sizeof(sample1);
+ memcpy(buffer + offset, &sample2, sizeof(sample2));
+ offset += sizeof(sample2);
+ memcpy(buffer + offset, &count, sizeof(count));
+ offset += sizeof(count);
+
+ message.setBuffer(buffer);
+ message.setNumberOfBytes(offset);
+ message.setTag(MatrixOwner::PUSH_PAYLOAD);
+
+ //cout << " DEBUG send PUSH_PAYLOAD to " << m_matrixOwner << endl;
+
+ send(m_matrixOwner, message);
+
+ m_iterator2++;
+
+ // end of the line
+ if(m_iterator2 == m_iterator1->second.end()) {
+
+ m_iterator1++;
+
+ if(m_iterator1 != m_localGramMatrix.end()) {
+
+ m_iterator2 = m_iterator1->second.begin();
+ }
+ }
+
+ return;
+ }
+ }
+
+ // we processed all the matrix
+
+ // free memory.
+ m_localGramMatrix.clear();
+
+ /*
+ printName();
+ cout << "DEBUG send PUSH_PAYLOAD_END to " << m_matrixOwner << endl;
+ */
+
+ Message response;
+ response.setTag(MatrixOwner::PUSH_PAYLOAD_END);
+ send(m_matrixOwner, response);
+}
+
+void StoreKeeper::configureHashTable() {
+
+ uint64_t buckets = 268435456;
+
+ int bucketsPerGroup = 32 + 16 + 8 + 8;
+
+ // \see http://docs.oracle.com/javase/7/docs/api/java/util/HashMap.html
+ double loadFactorThreshold = 0.75;
+
+ int rank = getRank();
+
+ bool showMemoryAllocation = false;
+
+ m_hashTable.constructor(buckets,"/apps/Ray-Surveyor/actors/StoreKeeper.txt",
+ showMemoryAllocation, rank,
+ bucketsPerGroup,loadFactorThreshold
+ );
+
+ m_configured = true;
+}
+
+void StoreKeeper::printColorReport() {
+
+ printName();
+ cout << "Coloring Report:" << endl;
+
+ m_colorSet.printColors(&cout);
+}
+
+void StoreKeeper::computeLocalGramMatrix() {
+
+ // printColorReport();
+
+
+
+ // compute the local Gram matrix
+
+ int colors = m_colorSet.getTotalNumberOfVirtualColors();
+
+ for(int i = 0 ; i < colors ; ++i) {
+
+ VirtualKmerColorHandle virtualColor = i;
+
+ set<PhysicalKmerColor> * samples = m_colorSet.getPhysicalColors(virtualColor);
+
+ LargeCount hits = m_colorSet.getNumberOfReferences(virtualColor);
+
+ // since people are going to use this to check
+ // for genome size, don't duplicate counts
+ //
+ bool reportTwoDNAStrands = false;
+
+ // we have 2 DNA strands !!!
+ if(reportTwoDNAStrands)
+ hits *= 2;
+
+ // TODO: samples could be ordered and stored in a vector
+ // to stop as soon as j > i
+
+ // Complexity: quadratic in the number of samples.
+ for(set<PhysicalKmerColor>::iterator sample1 = samples->begin();
+ sample1 != samples->end();
+ ++sample1) {
+
+ for(set<PhysicalKmerColor>::iterator sample2 = samples->begin();
+ sample2 != samples->end();
+ ++sample2) {
+
+ SampleIdentifier sample1Index = *sample1;
+ SampleIdentifier sample2Index = *sample2;
+
+ //if(sample2 < sample1)
+ // this is a diagonal matrix
+
+ m_localGramMatrix[sample1Index][sample2Index] += hits;
+ //m_localGramMatrix[sample2Index][sample1Index] += hits;
+ }
+ }
+ }
+
+ //printLocalGramMatrix();
+}
+
+void StoreKeeper::printLocalGramMatrix() {
+
+ printName();
+ cout << "Local Gram Matrix: " << endl;
+ cout << endl;
+
+ for(map<SampleIdentifier, map<SampleIdentifier, LargeCount> >::iterator column = m_localGramMatrix.begin();
+ column != m_localGramMatrix.end(); ++column) {
+
+ SampleIdentifier sample = column->first;
+
+ cout << " " << sample;
+ }
+
+ cout << endl;
+
+ for(map<SampleIdentifier, map<SampleIdentifier, LargeCount> >::iterator row = m_localGramMatrix.begin();
+ row != m_localGramMatrix.end(); ++row) {
+
+ SampleIdentifier sample1 = row->first;
+
+ cout << sample1;
+ for(map<SampleIdentifier, LargeCount>::iterator cell = row->second.begin();
+ cell != row->second.end(); ++cell) {
+
+ //SampleIdentifier sample2 = cell->first;
+
+ LargeCount hits = cell->second;
+
+ cout << " " << hits;
+ }
+
+ cout << endl;
+ }
+}
+
+void StoreKeeper::pushSampleVertex(Message & message) {
+
+ char * buffer = (char*)message.getBufferBytes();
+ int bytes = message.getNumberOfBytes();
+
+ int position = 0;
+
+ int producer = -1;
+ bytes -= sizeof(producer);
+ memcpy(&producer, buffer + bytes, sizeof(producer));
+
+ /*
+ printName();
+ cout << "Received payload, last producer was " << producer << endl;
+ */
+
+ while(position < bytes) {
+ Vertex vertex;
+
+
+ position += vertex.load(buffer + position);
+
+ int sample = -1;
+ memcpy(&sample, buffer + position, sizeof(sample));
+ position += sizeof(sample);
+
+ storeData(vertex, sample);
+
+ /*
+ printName();
+ cout << " DEBUG received ";
+ cout << "(from " << message.getSourceActor();
+ cout << ") ";
+ cout << "vertex for sample " << sample;
+ cout << " with sequence ";
+ vertex.print(m_kmerLength, m_colorSpaceMode);
+ cout << endl;
+ */
+
+ m_receivedObjects ++;
+
+ if(m_receivedObjects % 1000000 == 0) {
+
+ printStatus();
+ }
+ }
+
+ int source = message.getSourceActor();
+
+ Message response;
+ response.setTag(PUSH_SAMPLE_VERTEX_OK);
+ response.setBuffer(&producer);
+ response.setNumberOfBytes(sizeof(producer));
+
+ send(source, response);
+}
+
+void StoreKeeper::printStatus() {
+
+ printName();
+ cout << "(StoreKeeper) received " << m_receivedObjects << " objects so far !" << endl;
+}
+
+void StoreKeeper::storeData(Vertex & vertex, int & sample) {
+
+ Kmer kmer = vertex.getKey();
+ Kmer lowerKey;
+ kmer.getLowerKey(&lowerKey, m_kmerLength, m_colorSpaceMode);
+
+ uint64_t before = m_hashTable.size() * 2;
+
+ ExperimentVertex * graphVertex = m_hashTable.insert(&lowerKey);
+
+ // * 2 because we store pairs
+ uint64_t size = m_hashTable.size() * 2;
+
+ if(before < size) {
+
+ set<PhysicalKmerColor> emptySet;
+ VirtualKmerColorHandle noColor = m_colorSet.findVirtualColor(&emptySet);
+
+ m_colorSet.incrementReferences(noColor);
+
+ graphVertex->setVirtualColor(noColor);
+ }
+
+ int period = 1000000;
+ if(size % period == 0 && size != m_lastSize) {
+
+ printName();
+ cout << "has " << size << " Kmer objects in MyHashTable instance" << endl;
+
+ m_lastSize = size;
+ }
+
+#if 0
+ cout << "DEBUG Growth -> " << before << " -> " << size << endl;
+#endif
+
+
+ // add the PhysicalKmerColor to the node.
+
+ PhysicalKmerColor sampleColor = sample;
+ VirtualKmerColorHandle oldVirtualColor = graphVertex->getVirtualColor();
+
+ if(m_colorSet.virtualColorHasPhysicalColor(oldVirtualColor, sampleColor)) {
+
+ // Nothing to do, we already have this color
+ return;
+ }
+
+ VirtualKmerColorHandle newVirtualColor= m_colorSet.getVirtualColorFrom(oldVirtualColor, sampleColor);
+
+ graphVertex->setVirtualColor(newVirtualColor);
+
+ m_colorSet.decrementReferences(oldVirtualColor);
+ m_colorSet.incrementReferences(newVirtualColor);
+
+ /*
+ LargeCount referencesForOld = m_colorSet.getNumberOfReferences(oldVirtualColor);
+ LargeCount referencesForNew = m_colorSet.getNumberOfReferences(newVirtualColor);
+
+#if 1
+ printName();
+ cout << "DEBUG Kmer " << kmer.idToWord(m_kmerLength, m_colorSpaceMode);
+ cout << " Sample " << sample << endl;
+ cout << " DEBUG Lower " << lowerKey.idToWord(m_kmerLength, m_colorSpaceMode) << endl;
+#endif
+
+
+
+ cout << "DEBUG referencesForOld " << referencesForOld;
+ cout << " " << " referencesForNew " << referencesForNew;
+ cout << endl;
+
+ */
+}
diff --git a/code/Surveyor/StoreKeeper.h b/code/Surveyor/StoreKeeper.h
new file mode 100644
index 0000000..c3aed94
--- /dev/null
+++ b/code/Surveyor/StoreKeeper.h
@@ -0,0 +1,99 @@
+/*
+ Copyright 2013 Sébastien Boisvert
+ Copyright 2013 Université Laval
+ Copyright 2013 Centre Hospitalier Universitaire de Québec
+
+ This file is part of Ray Surveyor.
+
+ Ray Surveyor is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+
+ Ray Surveyor is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with Ray Surveyor. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+
+#ifndef StoreKeeperHeader
+#define StoreKeeperHeader
+
+#define PLAN_STORE_KEEPER_ACTORS_PER_RANK 1
+
+#include "ExperimentVertex.h"
+
+#include <code/Searcher/ColorSet.h>
+#include <code/VerticesExtractor/Vertex.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/Mock/constants.h>
+
+#include <RayPlatform/actors/Actor.h>
+#include <RayPlatform/structures/MyHashTable.h>
+
+/**
+ * Provides genomic storage.
+ *
+ * \author Sébastien Boisvert
+ */
+class StoreKeeper: public Actor {
+
+private:
+
+ map<SampleIdentifier, map<SampleIdentifier, LargeCount> > m_localGramMatrix;
+
+ map<SampleIdentifier, map<SampleIdentifier, LargeCount> >::iterator m_iterator1;
+ map<SampleIdentifier, LargeCount>::iterator m_iterator2;
+
+ ColorSet m_colorSet;
+
+ int m_mother;
+ int m_matrixOwner;
+
+ bool m_configured;
+
+ /**
+ * MyHashTable is a space-efficient data structure provided
+ * by RayPlatform.
+ */
+ MyHashTable<Kmer,ExperimentVertex> m_hashTable;
+
+ int m_kmerLength;
+ bool m_colorSpaceMode;
+
+ uint64_t m_receivedObjects;
+ uint64_t m_lastSize;
+
+ void pushSampleVertex(Message & message);
+ void printStatus();
+
+ void storeData(Vertex & vertex, int & sample);
+ void configureHashTable();
+ void computeLocalGramMatrix();
+ void printLocalGramMatrix();
+ void printColorReport();
+
+ void sendMatrixCell();
+
+public:
+
+ StoreKeeper();
+ ~StoreKeeper();
+
+ void receive(Message & message);
+
+ enum {
+ FIRST_TAG = 10250,
+ PUSH_SAMPLE_VERTEX,
+ PUSH_SAMPLE_VERTEX_OK,
+ MERGE,
+ MERGE_OK,
+ LAST_TAG
+ };
+};
+
+#endif
diff --git a/code/plugin_PhylogenyViewer/GenomeToTaxonLoader.cpp b/code/TaxonomyViewer/GenomeToTaxonLoader.cpp
similarity index 94%
rename from code/plugin_PhylogenyViewer/GenomeToTaxonLoader.cpp
rename to code/TaxonomyViewer/GenomeToTaxonLoader.cpp
index f71661b..1e38bbf 100644
--- a/code/plugin_PhylogenyViewer/GenomeToTaxonLoader.cpp
+++ b/code/TaxonomyViewer/GenomeToTaxonLoader.cpp
@@ -18,8 +18,8 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_PhylogenyViewer/GenomeToTaxonLoader.h>
-#include <plugin_PhylogenyViewer/types.h>
+#include "GenomeToTaxonLoader.h"
+#include "types.h"
#include <assert.h>
#include <iostream>
diff --git a/code/plugin_PhylogenyViewer/GenomeToTaxonLoader.h b/code/TaxonomyViewer/GenomeToTaxonLoader.h
similarity index 91%
rename from code/plugin_PhylogenyViewer/GenomeToTaxonLoader.h
rename to code/TaxonomyViewer/GenomeToTaxonLoader.h
index 68dc28d..89e2c84 100644
--- a/code/plugin_PhylogenyViewer/GenomeToTaxonLoader.h
+++ b/code/TaxonomyViewer/GenomeToTaxonLoader.h
@@ -22,9 +22,11 @@
#define _GenomeToTaxonLoader_h
-#include <application_core/constants.h>
-#include <core/types.h>
-#include <plugin_PhylogenyViewer/types.h>
+#include "types.h"
+
+#include <code/Mock/constants.h>
+
+#include <RayPlatform/core/types.h>
#include <string>
#include <stdint.h>
diff --git a/code/TaxonomyViewer/Makefile b/code/TaxonomyViewer/Makefile
new file mode 100644
index 0000000..d3a8480
--- /dev/null
+++ b/code/TaxonomyViewer/Makefile
@@ -0,0 +1,6 @@
+TaxonomyViewer-y += code/TaxonomyViewer/TaxonomyViewer.o
+TaxonomyViewer-y += code/TaxonomyViewer/GenomeToTaxonLoader.o
+TaxonomyViewer-y += code/TaxonomyViewer/TaxonomicTreeLoader.o
+TaxonomyViewer-y += code/TaxonomyViewer/TaxonNameLoader.o
+
+obj-y += $(TaxonomyViewer-y)
diff --git a/code/plugin_PhylogenyViewer/TaxonNameLoader.cpp b/code/TaxonomyViewer/TaxonNameLoader.cpp
similarity index 95%
rename from code/plugin_PhylogenyViewer/TaxonNameLoader.cpp
rename to code/TaxonomyViewer/TaxonNameLoader.cpp
index 8336d4f..6c1f210 100644
--- a/code/plugin_PhylogenyViewer/TaxonNameLoader.cpp
+++ b/code/TaxonomyViewer/TaxonNameLoader.cpp
@@ -18,8 +18,8 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_PhylogenyViewer/TaxonNameLoader.h>
-#include <plugin_PhylogenyViewer/types.h>
+#include "TaxonNameLoader.h"
+#include "types.h"
#include <assert.h>
#include <string.h>
diff --git a/code/plugin_PhylogenyViewer/TaxonNameLoader.h b/code/TaxonomyViewer/TaxonNameLoader.h
similarity index 91%
rename from code/plugin_PhylogenyViewer/TaxonNameLoader.h
rename to code/TaxonomyViewer/TaxonNameLoader.h
index c13dedb..d98fd09 100644
--- a/code/plugin_PhylogenyViewer/TaxonNameLoader.h
+++ b/code/TaxonomyViewer/TaxonNameLoader.h
@@ -21,10 +21,11 @@
#ifndef _TaxonNameLoader_h
#define _TaxonNameLoader_h
-#include <application_core/constants.h>
-#include <core/types.h>
+#include "types.h"
-#include <plugin_PhylogenyViewer/types.h>
+#include <code/Mock/constants.h>
+
+#include <RayPlatform/core/types.h>
#include <string>
#include <stdint.h>
diff --git a/code/plugin_PhylogenyViewer/PhylogeneticTreeLoader.cpp b/code/TaxonomyViewer/TaxonomicTreeLoader.cpp
similarity index 83%
rename from code/plugin_PhylogenyViewer/PhylogeneticTreeLoader.cpp
rename to code/TaxonomyViewer/TaxonomicTreeLoader.cpp
index 973f26d..59d09a6 100644
--- a/code/plugin_PhylogenyViewer/PhylogeneticTreeLoader.cpp
+++ b/code/TaxonomyViewer/TaxonomicTreeLoader.cpp
@@ -18,14 +18,14 @@
see <http://www.gnu.org/licenses/>
*/
-#include <plugin_PhylogenyViewer/PhylogeneticTreeLoader.h>
-#include <plugin_PhylogenyViewer/types.h>
+#include "TaxonomicTreeLoader.h"
+#include "types.h"
#include <assert.h>
#include <iostream>
using namespace std;
-void PhylogeneticTreeLoader::load(string file){
+void TaxonomicTreeLoader::load(string file){
m_current=0;
m_size=0;
@@ -64,11 +64,11 @@ void PhylogeneticTreeLoader::load(string file){
cout<<"File "<<file<<" has "<<m_size<<" entries"<<endl;
}
-bool PhylogeneticTreeLoader::hasNext(){
+bool TaxonomicTreeLoader::hasNext(){
return m_current<m_size;
}
-void PhylogeneticTreeLoader::getNext(TaxonIdentifier*parent,TaxonIdentifier*child){
+void TaxonomicTreeLoader::getNext(TaxonIdentifier*parent,TaxonIdentifier*child){
TaxonIdentifier l1;
TaxonIdentifier l2;
diff --git a/code/plugin_PhylogenyViewer/PhylogeneticTreeLoader.h b/code/TaxonomyViewer/TaxonomicTreeLoader.h
similarity index 83%
rename from code/plugin_PhylogenyViewer/PhylogeneticTreeLoader.h
rename to code/TaxonomyViewer/TaxonomicTreeLoader.h
index 3b9d27e..448a6cc 100644
--- a/code/plugin_PhylogenyViewer/PhylogeneticTreeLoader.h
+++ b/code/TaxonomyViewer/TaxonomicTreeLoader.h
@@ -18,13 +18,14 @@
see <http://www.gnu.org/licenses/>
*/
-#ifndef _PhylogeneticTreeLoader_h
-#define _PhylogeneticTreeLoader_h
+#ifndef _TaxonomicTreeLoader_h
+#define _TaxonomicTreeLoader_h
+#include "types.h"
-#include <application_core/constants.h>
-#include <core/types.h>
-#include <plugin_PhylogenyViewer/types.h>
+#include <code/Mock/constants.h>
+
+#include <RayPlatform/core/types.h>
#include <string>
#include <stdint.h>
@@ -36,7 +37,7 @@ using namespace std;
*
* \author Sébastien Boisvert
*/
-class PhylogeneticTreeLoader{
+class TaxonomicTreeLoader{
ifstream m_stream;
diff --git a/code/plugin_PhylogenyViewer/PhylogenyViewer.cpp b/code/TaxonomyViewer/TaxonomyViewer.cpp
similarity index 91%
rename from code/plugin_PhylogenyViewer/PhylogenyViewer.cpp
rename to code/TaxonomyViewer/TaxonomyViewer.cpp
index f4e1323..e5fd1c3 100644
--- a/code/plugin_PhylogenyViewer/PhylogenyViewer.cpp
+++ b/code/TaxonomyViewer/TaxonomyViewer.cpp
@@ -20,28 +20,25 @@
//#define DEBUG_RECURSION
-#include <plugin_PhylogenyViewer/PhylogenyViewer.h>
-#include <plugin_VerticesExtractor/GridTableIterator.h>
-#include <plugin_PhylogenyViewer/GenomeToTaxonLoader.h>
-#include <plugin_PhylogenyViewer/PhylogeneticTreeLoader.h>
-#include <plugin_PhylogenyViewer/TaxonNameLoader.h>
-#include <core/OperatingSystem.h>
+#include "TaxonomyViewer.h"
+#include "GenomeToTaxonLoader.h"
+#include "TaxonomicTreeLoader.h"
+#include "TaxonNameLoader.h"
-__CreatePlugin(PhylogenyViewer);
+#include <code/VerticesExtractor/GridTableIterator.h>
- /**/
-__CreateMasterModeAdapter(PhylogenyViewer,RAY_MASTER_MODE_PHYLOGENY_MAIN); /**/
- /**/
-__CreateSlaveModeAdapter(PhylogenyViewer,RAY_SLAVE_MODE_PHYLOGENY_MAIN); /**/
- /**/
-__CreateMessageTagAdapter(PhylogenyViewer,RAY_MPI_TAG_TOUCH_TAXON); /**/
-__CreateMessageTagAdapter(PhylogenyViewer,RAY_MPI_TAG_TAXON_OBSERVATIONS); /**/
- /**/
+#include <RayPlatform/core/OperatingSystem.h>
+__CreatePlugin(TaxonomyViewer);
+
+__CreateMasterModeAdapter(TaxonomyViewer,RAY_MASTER_MODE_PHYLOGENY_MAIN);
+__CreateSlaveModeAdapter(TaxonomyViewer,RAY_SLAVE_MODE_PHYLOGENY_MAIN);
+__CreateMessageTagAdapter(TaxonomyViewer,RAY_MPI_TAG_TOUCH_TAXON);
+__CreateMessageTagAdapter(TaxonomyViewer,RAY_MPI_TAG_TAXON_OBSERVATIONS);
//#define DEBUG_PHYLOGENY
-void PhylogenyViewer::call_RAY_MASTER_MODE_PHYLOGENY_MAIN(){
+void TaxonomyViewer::call_RAY_MASTER_MODE_PHYLOGENY_MAIN(){
if(!m_started){
#ifdef DEBUG_PHYLOGENY
@@ -135,7 +132,7 @@ void PhylogenyViewer::call_RAY_MASTER_MODE_PHYLOGENY_MAIN(){
}
}
-void PhylogenyViewer::copyTaxonsFromSecondaryTable(){
+void TaxonomyViewer::copyTaxonsFromSecondaryTable(){
int before=m_taxonsForPhylogeny.size();
for(set<TaxonIdentifier>::iterator i=m_taxonsForPhylogenyMaster.begin();
@@ -146,7 +143,7 @@ void PhylogenyViewer::copyTaxonsFromSecondaryTable(){
int after=m_taxonsForPhylogeny.size();
- cout<<"[PhylogenyViewer::copyTaxonsFromSecondaryTable] "<<before<<" -> "<<after<<endl;
+ cout<<"[TaxonomyViewer::copyTaxonsFromSecondaryTable] "<<before<<" -> "<<after<<endl;
#ifdef ASSERT
assert(m_taxonsForPhylogeny.size() == m_taxonsForPhylogenyMaster.size());
@@ -155,7 +152,7 @@ void PhylogenyViewer::copyTaxonsFromSecondaryTable(){
m_taxonsForPhylogenyMaster.clear();
}
-void PhylogenyViewer::call_RAY_SLAVE_MODE_PHYLOGENY_MAIN(){
+void TaxonomyViewer::call_RAY_SLAVE_MODE_PHYLOGENY_MAIN(){
if(!m_extractedColorsForPhylogeny){
extractColorsForPhylogeny();
@@ -215,7 +212,7 @@ void PhylogenyViewer::call_RAY_SLAVE_MODE_PHYLOGENY_MAIN(){
}
}
-void PhylogenyViewer::sendTreeCounts(){
+void TaxonomyViewer::sendTreeCounts(){
if(m_messageReceived && m_countIterator==m_taxonObservations.end()){
m_syncedTree=true;
@@ -266,7 +263,7 @@ void PhylogenyViewer::sendTreeCounts(){
}
-void PhylogenyViewer::call_RAY_MPI_TAG_TAXON_OBSERVATIONS(Message*message){
+void TaxonomyViewer::call_RAY_MPI_TAG_TAXON_OBSERVATIONS(Message*message){
MessageUnit*buffer=message->getBuffer();
int count=message->getCount();
@@ -292,7 +289,7 @@ void PhylogenyViewer::call_RAY_MPI_TAG_TAXON_OBSERVATIONS(Message*message){
message->getSource(),RAY_MPI_TAG_TAXON_OBSERVATIONS_REPLY);
}
-void PhylogenyViewer::call_RAY_MPI_TAG_TOUCH_TAXON(Message*message){
+void TaxonomyViewer::call_RAY_MPI_TAG_TOUCH_TAXON(Message*message){
MessageUnit*buffer=message->getBuffer();
int count=message->getCount();
@@ -307,7 +304,7 @@ void PhylogenyViewer::call_RAY_MPI_TAG_TOUCH_TAXON(Message*message){
message->getSource(),RAY_MPI_TAG_TOUCH_TAXON_REPLY);
}
-void PhylogenyViewer::loadTree(){
+void TaxonomyViewer::loadTree(){
if(!m_parameters->hasOption("-with-taxonomy")){
m_loadedTree=true;
@@ -317,7 +314,7 @@ void PhylogenyViewer::loadTree(){
string treeFile=m_parameters->getTreeFile();
- PhylogeneticTreeLoader loader;
+ TaxonomicTreeLoader loader;
bool growed=true;
@@ -371,7 +368,7 @@ void PhylogenyViewer::loadTree(){
m_gatheredObservations=false;
}
-void PhylogenyViewer::gatherKmerObservations(){
+void TaxonomyViewer::gatherKmerObservations(){
/* set to true to use only assembled kmers */
bool useOnlyAssembledKmer=false;
@@ -408,7 +405,7 @@ void PhylogenyViewer::gatherKmerObservations(){
/* here, we just want to find a path with
* a good progression */
- Direction*a=node->m_directions;
+ Direction*a=node->getFirstDirection();
bool nicelyAssembled=false;
while(a!=NULL){
@@ -519,7 +516,7 @@ void PhylogenyViewer::gatherKmerObservations(){
m_countIterator=m_taxonObservations.begin();
}
-LargeCount PhylogenyViewer::getSelfCount(TaxonIdentifier taxon){
+LargeCount TaxonomyViewer::getSelfCount(TaxonIdentifier taxon){
if(m_taxonObservations.count(taxon)==0){
return 0;
}
@@ -527,7 +524,7 @@ LargeCount PhylogenyViewer::getSelfCount(TaxonIdentifier taxon){
return m_taxonObservations[taxon];
}
-void PhylogenyViewer::populateRanks(map<string,LargeCount>*rankSelfObservations,
+void TaxonomyViewer::populateRanks(map<string,LargeCount>*rankSelfObservations,
map<string,LargeCount>*rankRecursiveObservations){
for(map<TaxonIdentifier,string>::iterator i=m_taxonNames.begin();
@@ -551,7 +548,7 @@ void PhylogenyViewer::populateRanks(map<string,LargeCount>*rankSelfObservations,
}
}
-void PhylogenyViewer::showObservations_XML(ostream*stream){
+void TaxonomyViewer::showObservations_XML(ostream*stream){
ostringstream operationBuffer;
@@ -752,7 +749,7 @@ void PhylogenyViewer::showObservations_XML(ostream*stream){
}
-LargeCount PhylogenyViewer::getRecursiveCount(TaxonIdentifier taxon){
+LargeCount TaxonomyViewer::getRecursiveCount(TaxonIdentifier taxon){
if(m_taxonRecursiveObservations.count(taxon)>0){
@@ -781,7 +778,7 @@ LargeCount PhylogenyViewer::getRecursiveCount(TaxonIdentifier taxon){
return count;
}
-void PhylogenyViewer::showObservations(ostream*stream){
+void TaxonomyViewer::showObservations(ostream*stream){
(*stream)<<endl;
for(map<TaxonIdentifier,LargeCount>::iterator i=m_taxonObservations.begin();
@@ -807,7 +804,7 @@ void PhylogenyViewer::showObservations(ostream*stream){
(*stream)<<" k-mer observations: "<<m_unknown<<endl;
}
-void PhylogenyViewer::classifySignal(vector<TaxonIdentifier>*taxons,int kmerCoverage,Vertex*vertex,Kmer*key){
+void TaxonomyViewer::classifySignal(vector<TaxonIdentifier>*taxons,int kmerCoverage,Vertex*vertex,Kmer*key){
// given a list of taxon,
// place the kmer coverage somewhere in
// the tree
@@ -902,7 +899,7 @@ void PhylogenyViewer::classifySignal(vector<TaxonIdentifier>*taxons,int kmerCove
}
}
-TaxonIdentifier PhylogenyViewer::findCommonAncestor(vector<TaxonIdentifier>*taxons){
+TaxonIdentifier TaxonomyViewer::findCommonAncestor(vector<TaxonIdentifier>*taxons){
map<TaxonIdentifier,int> counts;
@@ -953,7 +950,7 @@ TaxonIdentifier PhylogenyViewer::findCommonAncestor(vector<TaxonIdentifier>*taxo
return 999999999999ULL;
}
-TaxonIdentifier PhylogenyViewer::getTaxonParent(TaxonIdentifier taxon){
+TaxonIdentifier TaxonomyViewer::getTaxonParent(TaxonIdentifier taxon){
if(m_treeParents.count(taxon)>0){
return m_treeParents[taxon];
}
@@ -961,7 +958,7 @@ TaxonIdentifier PhylogenyViewer::getTaxonParent(TaxonIdentifier taxon){
return 999999999999ULL;
}
-void PhylogenyViewer::loadTaxonNames(){
+void TaxonomyViewer::loadTaxonNames(){
if(!m_parameters->hasOption("-with-taxonomy")){
return;
@@ -990,7 +987,7 @@ void PhylogenyViewer::loadTaxonNames(){
cout<<"Rank "<<m_rank<<" loaded taxon names from "<<file<<endl;
}
-string PhylogenyViewer::getTaxonName(TaxonIdentifier taxon){
+string TaxonomyViewer::getTaxonName(TaxonIdentifier taxon){
if(m_taxonNames.count(taxon)>0){
string value=m_taxonNames[taxon];
@@ -1010,9 +1007,9 @@ string PhylogenyViewer::getTaxonName(TaxonIdentifier taxon){
return "CachingError";
}
-void PhylogenyViewer::testPaths(){
+void TaxonomyViewer::testPaths(){
cout<<endl;
- cout<<"[PhylogenyViewer::testPaths]"<<endl;
+ cout<<"[TaxonomyViewer::testPaths]"<<endl;
for(set<TaxonIdentifier>::iterator i=m_taxonsForPhylogeny.begin();i!=m_taxonsForPhylogeny.end();i++){
TaxonIdentifier taxon=*i;
@@ -1028,7 +1025,7 @@ void PhylogenyViewer::testPaths(){
cout<<endl;
}
-void PhylogenyViewer::printTaxonPath_XML(TaxonIdentifier taxon,vector<TaxonIdentifier>*path,ostream*stream){
+void TaxonomyViewer::printTaxonPath_XML(TaxonIdentifier taxon,vector<TaxonIdentifier>*path,ostream*stream){
(*stream)<<"<path>"<<endl;
//cout<<"Taxon= "<<taxon<<endl;
@@ -1043,14 +1040,14 @@ void PhylogenyViewer::printTaxonPath_XML(TaxonIdentifier taxon,vector<TaxonIdent
(*stream)<<"</path>"<<endl;
}
-void PhylogenyViewer::printTaxon_XML(TaxonIdentifier taxon,ostream*stream){
+void TaxonomyViewer::printTaxon_XML(TaxonIdentifier taxon,ostream*stream){
string name=getTaxonName(taxon);
string rank=getTaxonRank(taxon);
(*stream)<<"<taxon><name>"<<name<<"</name><rank>"<<rank<<"</rank><identifier>"<<taxon<<"</identifier></taxon>"<<endl;
}
-string PhylogenyViewer::getTaxonRank(TaxonIdentifier taxon){
+string TaxonomyViewer::getTaxonRank(TaxonIdentifier taxon){
if(m_taxonRanks.count(taxon)==0){
return "CachingError";
}
@@ -1058,7 +1055,7 @@ string PhylogenyViewer::getTaxonRank(TaxonIdentifier taxon){
return m_taxonRanks[taxon];
}
-void PhylogenyViewer::printTaxonPath(TaxonIdentifier taxon,vector<TaxonIdentifier>*path,ostream*stream){
+void TaxonomyViewer::printTaxonPath(TaxonIdentifier taxon,vector<TaxonIdentifier>*path,ostream*stream){
//cout<<"Taxon= "<<taxon<<endl;
@@ -1071,7 +1068,7 @@ void PhylogenyViewer::printTaxonPath(TaxonIdentifier taxon,vector<TaxonIdentifie
}
-void PhylogenyViewer::getTaxonPathFromRoot(TaxonIdentifier taxon,vector<TaxonIdentifier>*path){
+void TaxonomyViewer::getTaxonPathFromRoot(TaxonIdentifier taxon,vector<TaxonIdentifier>*path){
TaxonIdentifier current=taxon;
@@ -1107,7 +1104,7 @@ void PhylogenyViewer::getTaxonPathFromRoot(TaxonIdentifier taxon,vector<TaxonIde
/*
* Loads taxons by fetching pairs in a conversion file.
*/
-void PhylogenyViewer::loadTaxons(){
+void TaxonomyViewer::loadTaxons(){
if(!m_parameters->hasOption("-with-taxonomy")){
@@ -1151,7 +1148,7 @@ void PhylogenyViewer::loadTaxons(){
m_messageReceived=true;
}
-void PhylogenyViewer::sendTaxonsFromMaster(){
+void TaxonomyViewer::sendTaxonsFromMaster(){
if(m_responses == m_size && m_taxonIterator==m_taxonsForPhylogeny.end()){
@@ -1202,7 +1199,7 @@ void PhylogenyViewer::sendTaxonsFromMaster(){
-void PhylogenyViewer::sendTaxonsToMaster(){
+void TaxonomyViewer::sendTaxonsToMaster(){
if(m_messageReceived && m_taxonIterator==m_taxonsForPhylogeny.end()){
@@ -1248,7 +1245,7 @@ void PhylogenyViewer::sendTaxonsToMaster(){
/**
* here we extract the phylogeny colors
*/
-void PhylogenyViewer::extractColorsForPhylogeny(){
+void TaxonomyViewer::extractColorsForPhylogeny(){
GridTableIterator iterator;
iterator.constructor(m_subgraph,m_parameters->getWordSize(),m_parameters);
@@ -1307,22 +1304,22 @@ void PhylogenyViewer::extractColorsForPhylogeny(){
m_totalNumberOfKmerObservations=m_searcher->getTotalNumberOfKmerObservations();
}
-void PhylogenyViewer::registerPlugin(ComputeCore*core){
+void TaxonomyViewer::registerPlugin(ComputeCore*core){
m_plugin=core->allocatePluginHandle();
- core->setPluginName(m_plugin,"PhylogenyViewer");
+ core->setPluginName(m_plugin,"TaxonomyViewer");
core->setPluginDescription(m_plugin,"Get a taxonomy view of the sample.");
core->setPluginAuthors(m_plugin,"Sébastien Boisvert");
core->setPluginLicense(m_plugin,"GNU General Public License version 3");
RAY_MASTER_MODE_PHYLOGENY_MAIN=core->allocateMasterModeHandle(m_plugin);
core->setMasterModeSymbol(m_plugin,RAY_MASTER_MODE_PHYLOGENY_MAIN,"RAY_MASTER_MODE_PHYLOGENY_MAIN");
- core->setMasterModeObjectHandler(m_plugin,RAY_MASTER_MODE_PHYLOGENY_MAIN,__GetAdapter(PhylogenyViewer,RAY_MASTER_MODE_PHYLOGENY_MAIN));
+ core->setMasterModeObjectHandler(m_plugin,RAY_MASTER_MODE_PHYLOGENY_MAIN,__GetAdapter(TaxonomyViewer,RAY_MASTER_MODE_PHYLOGENY_MAIN));
RAY_SLAVE_MODE_PHYLOGENY_MAIN=core->allocateSlaveModeHandle(m_plugin);
core->setSlaveModeSymbol(m_plugin,RAY_SLAVE_MODE_PHYLOGENY_MAIN,"RAY_SLAVE_MODE_PHYLOGENY_MAIN");
- core->setSlaveModeObjectHandler(m_plugin,RAY_SLAVE_MODE_PHYLOGENY_MAIN,__GetAdapter(PhylogenyViewer,RAY_SLAVE_MODE_PHYLOGENY_MAIN));
+ core->setSlaveModeObjectHandler(m_plugin,RAY_SLAVE_MODE_PHYLOGENY_MAIN,__GetAdapter(TaxonomyViewer,RAY_SLAVE_MODE_PHYLOGENY_MAIN));
RAY_MPI_TAG_PHYLOGENY_MAIN=core->allocateMessageTagHandle(m_plugin);
core->setMessageTagSymbol(m_plugin,RAY_MPI_TAG_PHYLOGENY_MAIN,"RAY_MPI_TAG_PHYLOGENY_MAIN");
@@ -1335,7 +1332,7 @@ void PhylogenyViewer::registerPlugin(ComputeCore*core){
RAY_MPI_TAG_TOUCH_TAXON=core->allocateMessageTagHandle(m_plugin);
core->setMessageTagSymbol(m_plugin,RAY_MPI_TAG_TOUCH_TAXON,"RAY_MPI_TAG_TOUCH_TAXON");
- core->setMessageTagObjectHandler(m_plugin,RAY_MPI_TAG_TOUCH_TAXON,__GetAdapter(PhylogenyViewer,RAY_MPI_TAG_TOUCH_TAXON));
+ core->setMessageTagObjectHandler(m_plugin,RAY_MPI_TAG_TOUCH_TAXON,__GetAdapter(TaxonomyViewer,RAY_MPI_TAG_TOUCH_TAXON));
RAY_MPI_TAG_TOUCH_TAXON_REPLY=core->allocateMessageTagHandle(m_plugin);
core->setMessageTagSymbol(m_plugin,RAY_MPI_TAG_TOUCH_TAXON_REPLY,"RAY_MPI_TAG_TOUCH_TAXON_REPLY");
@@ -1352,8 +1349,8 @@ void PhylogenyViewer::registerPlugin(ComputeCore*core){
m_outboxAllocator=core->getOutboxAllocator();
m_inboxAllocator=core->getInboxAllocator();
- m_rank=core->getMessagesHandler()->getRank();
- m_size=core->getMessagesHandler()->getSize();
+ m_rank=core->getRank();
+ m_size=core->getSize();
m_extractedColorsForPhylogeny=false;
UNKNOWN_TAXON=COLOR_NAMESPACE_MULTIPLIER;
@@ -1363,7 +1360,7 @@ void PhylogenyViewer::registerPlugin(ComputeCore*core){
m_core=core;
}
-void PhylogenyViewer::resolveSymbols(ComputeCore*core){
+void TaxonomyViewer::resolveSymbols(ComputeCore*core){
RAY_MASTER_MODE_PHYLOGENY_MAIN=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_PHYLOGENY_MAIN");
RAY_MASTER_MODE_KILL_RANKS=core->getMasterModeFromSymbol(m_plugin,"RAY_MASTER_MODE_KILL_RANKS");
@@ -1390,10 +1387,14 @@ void PhylogenyViewer::resolveSymbols(ComputeCore*core){
m_colorSet=(ColorSet*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/VirtualColorManagementUnit.ray");
m_timePrinter=(TimePrinter*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/Timer.ray");
- m_searcher=(Searcher*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/plugin_Searcher.ray");
+ m_searcher=(Searcher*)core->getObjectFromSymbol(m_plugin,"/RayAssembler/ObjectStore/Searcher.ray");
m_started=false;
- __BindPlugin(PhylogenyViewer);
-}
+ __BindPlugin(TaxonomyViewer);
+ __BindAdapter(TaxonomyViewer,RAY_MASTER_MODE_PHYLOGENY_MAIN);
+ __BindAdapter(TaxonomyViewer,RAY_SLAVE_MODE_PHYLOGENY_MAIN);
+ __BindAdapter(TaxonomyViewer,RAY_MPI_TAG_TOUCH_TAXON);
+ __BindAdapter(TaxonomyViewer,RAY_MPI_TAG_TAXON_OBSERVATIONS);
+}
diff --git a/code/plugin_PhylogenyViewer/PhylogenyViewer.h b/code/TaxonomyViewer/TaxonomyViewer.h
similarity index 83%
rename from code/plugin_PhylogenyViewer/PhylogenyViewer.h
rename to code/TaxonomyViewer/TaxonomyViewer.h
index c1ca2a7..dc933b3 100644
--- a/code/plugin_PhylogenyViewer/PhylogenyViewer.h
+++ b/code/TaxonomyViewer/TaxonomyViewer.h
@@ -18,6 +18,28 @@
see <http://www.gnu.org/licenses/>
*/
+#ifndef _TaxonomyViewer_h
+#define _TaxonomyViewer_h
+
+#include "types.h"
+
+#include <code/Searcher/ColorSet.h>
+#include <code/Mock/Parameters.h>
+#include <code/VerticesExtractor/GridTable.h>
+#include <code/Searcher/Searcher.h>
+
+#include <RayPlatform/profiling/TimePrinter.h>
+#include <RayPlatform/plugins/CorePlugin.h>
+#include <RayPlatform/handlers/MasterModeHandler.h>
+#include <RayPlatform/handlers/MessageTagHandler.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/core/ComputeCore.h>
+
+#include <set>
+#include <stdint.h>
+#include <map>
+using namespace std;
+
/*
[sboisver12 at colosse1 2012-01-25]$ ls
@@ -47,33 +69,25 @@ Genome-to-Taxon.tsv Taxon-Names.tsv Taxon-Types.tsv TreeOfLife-Edges.tsv
*/
-#ifndef _PhylogenyViewer_h
-#define _PhylogenyViewer_h
-#include <core/ComputeCore.h>
-#include <plugins/CorePlugin.h>
-#include <handlers/SlaveModeHandler.h>
-#include <plugin_Searcher/ColorSet.h>
-#include <handlers/MasterModeHandler.h>
-#include <handlers/MessageTagHandler.h>
-#include <application_core/Parameters.h>
-#include <plugin_VerticesExtractor/GridTable.h>
-#include <profiling/TimePrinter.h>
-#include <plugin_Searcher/Searcher.h>
-#include <plugin_PhylogenyViewer/types.h>
-
-#include <set>
-#include <stdint.h>
-#include <map>
-using namespace std;
+__DeclarePlugin(TaxonomyViewer);
+__DeclareMasterModeAdapter(TaxonomyViewer,RAY_MASTER_MODE_PHYLOGENY_MAIN);
+__DeclareSlaveModeAdapter(TaxonomyViewer,RAY_SLAVE_MODE_PHYLOGENY_MAIN);
+__DeclareMessageTagAdapter(TaxonomyViewer,RAY_MPI_TAG_TOUCH_TAXON);
+__DeclareMessageTagAdapter(TaxonomyViewer,RAY_MPI_TAG_TAXON_OBSERVATIONS);
/**
* a plugin to know what is present in a sample
* \author Sébastien Boisvert
*/
-class PhylogenyViewer: public CorePlugin{
+class TaxonomyViewer: public CorePlugin{
+
+ __AddAdapter(TaxonomyViewer,RAY_MASTER_MODE_PHYLOGENY_MAIN);
+ __AddAdapter(TaxonomyViewer,RAY_SLAVE_MODE_PHYLOGENY_MAIN);
+ __AddAdapter(TaxonomyViewer,RAY_MPI_TAG_TOUCH_TAXON);
+ __AddAdapter(TaxonomyViewer,RAY_MPI_TAG_TAXON_OBSERVATIONS);
/* slave states */
bool m_extractedColorsForPhylogeny;
diff --git a/code/plugin_PhylogenyViewer/types.h b/code/TaxonomyViewer/types.h
similarity index 100%
rename from code/plugin_PhylogenyViewer/types.h
rename to code/TaxonomyViewer/types.h
diff --git a/code/plugin_VerticesExtractor/GridTable.cpp b/code/VerticesExtractor/GridTable.cpp
similarity index 83%
rename from code/plugin_VerticesExtractor/GridTable.cpp
rename to code/VerticesExtractor/GridTable.cpp
index 859cd09..29f0017 100644
--- a/code/plugin_VerticesExtractor/GridTable.cpp
+++ b/code/VerticesExtractor/GridTable.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,16 +19,24 @@
*/
-#include <plugin_KmerAcademyBuilder/Kmer.h>
+#include "GridTable.h"
+
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/Mock/common_functions.h>
+
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/cryptography/crypto.h>
+
#include <assert.h>
-#include <plugin_VerticesExtractor/GridTable.h>
-#include <core/OperatingSystem.h>
-#include <application_core/common_functions.h>
-#include <cryptography/crypto.h>
#include <stdlib.h>
#include <stdio.h>
void GridTable::constructor(int rank,Parameters*parameters){
+
+ #ifdef ASSERT
+ assert(parameters!=NULL);
+ #endif
+
m_parameters=parameters;
m_size=0;
@@ -38,9 +46,6 @@ void GridTable::constructor(int rank,Parameters*parameters){
int bucketsPerGroup=m_parameters->getNumberOfBucketsPerGroup();
double loadFactorThreshold=m_parameters->getLoadFactorThreshold();
- cout<<"[GridTable] buckets="<<buckets<<" bucketsPerGroup="<<bucketsPerGroup;
- cout<<" loadFactorThreshold="<<loadFactorThreshold<<endl;
-
m_hashTable.constructor(buckets,"RAY_MALLOC_TYPE_GRID_TABLE",
m_parameters->showMemoryAllocations(),m_parameters->getRank(),
bucketsPerGroup,loadFactorThreshold
@@ -60,6 +65,18 @@ void GridTable::constructor(int rank,Parameters*parameters){
m_verbose=false;
}
+void GridTable::printStatus(){
+
+ uint64_t buckets=m_parameters->getNumberOfBuckets();
+ int bucketsPerGroup=m_parameters->getNumberOfBucketsPerGroup();
+ double loadFactorThreshold=m_parameters->getLoadFactorThreshold();
+
+ if(m_parameters->hasOption("-hash-table-verbosity")){
+ cout<<"[GridTable] buckets="<<buckets<<" bucketsPerGroup="<<bucketsPerGroup;
+ cout<<" loadFactorThreshold="<<loadFactorThreshold<<endl;
+ }
+}
+
LargeCount GridTable::size(){
return m_size;
}
@@ -96,6 +113,10 @@ Vertex*GridTable::insert(Kmer*key){
assert(key!=NULL);
#endif
+ #ifdef ASSERT
+ assert(m_parameters!=NULL);
+ #endif
+
Kmer lowerKey=key->complementVertex(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
if(key->isLower(&lowerKey)){
lowerKey=*key;
@@ -213,7 +234,7 @@ vector<Direction> GridTable::getDirections(Kmer*a){
/* do a copy to track to check for a segmentation fault */
Vertex copy=*i;
- assert(copy.m_coverage_lower >= 1);
+ assert(copy.getVertexCoverage() >= 1);
#endif
return i->getDirections(a);
diff --git a/code/plugin_VerticesExtractor/GridTable.h b/code/VerticesExtractor/GridTable.h
similarity index 86%
rename from code/plugin_VerticesExtractor/GridTable.h
rename to code/VerticesExtractor/GridTable.h
index e72d58c..df9675d 100644
--- a/code/plugin_VerticesExtractor/GridTable.h
+++ b/code/VerticesExtractor/GridTable.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,11 +22,13 @@
#ifndef _GridTable
#define _GridTable
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <structures/MyHashTable.h>
-#include <memory/MyAllocator.h>
-#include <application_core/Parameters.h>
-#include <plugin_VerticesExtractor/Vertex.h>
+#include "Vertex.h"
+
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/Mock/Parameters.h>
+
+#include <RayPlatform/structures/MyHashTable.h>
+#include <RayPlatform/memory/MyAllocator.h>
/**
* The GridTable stores all the k-mers for the graph.
@@ -64,6 +66,8 @@ public:
MyHashTable<Kmer,Vertex>*getHashTable();
void printStatistics();
void completeResizing();
+
+ void printStatus();
};
#endif
diff --git a/code/plugin_VerticesExtractor/GridTableIterator.cpp b/code/VerticesExtractor/GridTableIterator.cpp
similarity index 88%
rename from code/plugin_VerticesExtractor/GridTableIterator.cpp
rename to code/VerticesExtractor/GridTableIterator.cpp
index 3463821..25e6df3 100644
--- a/code/plugin_VerticesExtractor/GridTableIterator.cpp
+++ b/code/VerticesExtractor/GridTableIterator.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,8 +19,10 @@
*/
-#include <plugin_VerticesExtractor/GridTableIterator.h>
-#include <structures/MyHashTableIterator.h>
+#include "GridTableIterator.h"
+
+#include <RayPlatform/structures/MyHashTableIterator.h>
+
#include <assert.h>
#include <iostream>
using namespace std;
@@ -46,7 +48,7 @@ Vertex*GridTableIterator::next(){
assert(hasNext());
#endif
m_currentEntry=m_iterator.next();
- m_currentKey=m_currentEntry->m_lowerKey;
+ m_currentKey=m_currentEntry->getKey();
m_mustProcessOtherKey=true;
return m_currentEntry;
}
diff --git a/code/plugin_VerticesExtractor/GridTableIterator.h b/code/VerticesExtractor/GridTableIterator.h
similarity index 82%
rename from code/plugin_VerticesExtractor/GridTableIterator.h
rename to code/VerticesExtractor/GridTableIterator.h
index 263f829..8035b07 100644
--- a/code/plugin_VerticesExtractor/GridTableIterator.h
+++ b/code/VerticesExtractor/GridTableIterator.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2011 Sébastien Boisvert
+ Copyright (C) 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,10 +22,12 @@
#ifndef _GridTableIterator
#define _GridTableIterator
-#include <application_core/Parameters.h>
-#include <application_core/common_functions.h>
-#include <structures/MyHashTableIterator.h>
-#include <plugin_VerticesExtractor/GridTable.h>
+#include "GridTable.h"
+
+#include <code/Mock/Parameters.h>
+#include <code/Mock/common_functions.h>
+
+#include <RayPlatform/structures/MyHashTableIterator.h>
/**
* \author Sébastien Boisvert
diff --git a/code/VerticesExtractor/Makefile b/code/VerticesExtractor/Makefile
new file mode 100644
index 0000000..6ef7b0b
--- /dev/null
+++ b/code/VerticesExtractor/Makefile
@@ -0,0 +1,9 @@
+VerticesExtractor-y += code/VerticesExtractor/VerticesExtractor.o
+VerticesExtractor-y += code/VerticesExtractor/GridTable.o
+VerticesExtractor-y += code/VerticesExtractor/GridTableIterator.o
+VerticesExtractor-y += code/VerticesExtractor/Vertex.o
+
+obj-y += $(VerticesExtractor-y)
+
+
+
diff --git a/code/plugin_VerticesExtractor/Vertex.cpp b/code/VerticesExtractor/Vertex.cpp
similarity index 74%
rename from code/plugin_VerticesExtractor/Vertex.cpp
rename to code/VerticesExtractor/Vertex.cpp
index ff09720..291262d 100644
--- a/code/plugin_VerticesExtractor/Vertex.cpp
+++ b/code/VerticesExtractor/Vertex.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,19 +19,30 @@
*/
-#include<assert.h>
-#include<vector>
-#include<plugin_VerticesExtractor/Vertex.h>
-#include<cstdlib>
-#include<application_core/common_functions.h>
-#include<iostream>
+#include "Vertex.h"
+#include <code/Mock/common_functions.h>
+
+#include <assert.h>
+#include <vector>
+#include <cstdlib>
+#include <iostream>
using namespace std;
#define __NO_ORIGIN -999
+Vertex::Vertex() {
+
+ constructor();
+}
+
+Vertex::~Vertex() {
+}
+
void Vertex::constructor(){
m_coverage_lower=0;
- m_edges_lower=0;
+
+ setEdgeSet(0);
+
m_readsStartingHere=NULL;
m_directions=NULL;
m_assembled=__NO_ORIGIN;
@@ -39,6 +50,11 @@ void Vertex::constructor(){
m_color=0;
}
+void Vertex::setEdgeSet(uint8_t edges) {
+
+ m_edges_lower = edges;
+}
+
void Vertex::assemble(Rank origin){
if(origin>m_assembled)
m_assembled=origin;
@@ -56,6 +72,11 @@ bool Vertex::isAssembledByGreaterRank(Rank origin){
return origin<m_assembled;
}
+void Vertex::setCoverageValue(CoverageDepth coverage) {
+
+ m_coverage_lower = coverage;
+}
+
void Vertex::setCoverage(Kmer*a,CoverageDepth coverage){
if(*a==m_lowerKey){
@@ -70,16 +91,20 @@ void Vertex::setCoverage(Kmer*a,CoverageDepth coverage){
}
}
-CoverageDepth Vertex::getCoverage(Kmer*a){
+CoverageDepth Vertex::getVertexCoverage() const{
+ return getCoverage(&m_lowerKey);
+}
+
+CoverageDepth Vertex::getCoverage(const Kmer*a) const{
return m_coverage_lower;
}
-vector<Kmer> Vertex::getIngoingEdges(Kmer *a,int k){
- return a->_getIngoingEdges(getEdges(a),k);
+vector<Kmer> Vertex::getIngoingEdges(const Kmer *a,int k) const{
+ return a->getIngoingEdges(getEdges(a),k);
}
-vector<Kmer> Vertex::getOutgoingEdges(Kmer*a,int k){
- return a->_getOutgoingEdges(getEdges(a),k);
+vector<Kmer> Vertex::getOutgoingEdges(const Kmer*a,int k) const{
+ return a->getOutgoingEdges(getEdges(a),k);
}
void Vertex::addIngoingEdge_ClassicMethod(Kmer*vertex,Kmer*a,int k){
@@ -137,7 +162,11 @@ void Vertex::setEdges(Kmer*a,uint8_t edges){
m_edges_lower=convertBitmap(edges);
}
-uint8_t Vertex::getEdges(Kmer*a){
+uint8_t Vertex::getVertexEdges() const{
+ return getEdges(&m_lowerKey);
+}
+
+uint8_t Vertex::getEdges(const Kmer*a) const{
if(*a==m_lowerKey)
return m_edges_lower;
return convertBitmap(m_edges_lower);
@@ -225,7 +254,7 @@ void Vertex::clearDirections(Kmer*a){
m_directions=NULL;
}
-void Vertex::write(Kmer*key,ofstream*f,int kmerLength){
+void Vertex::write(Kmer*key, ostream*f,int kmerLength){
int coverage=getCoverage(key);
key->write(f);
f->write((char*)&coverage,sizeof(int));
@@ -243,7 +272,7 @@ void Vertex::write(Kmer*key,ofstream*f,int kmerLength){
}
}
-void Vertex::writeAnnotations(Kmer*key,ofstream*f,int kmerLength,bool color){
+void Vertex::writeAnnotations(Kmer*key,ostream*f,int kmerLength,bool color){
key->write(f);
Kmer complement=key->complementVertex(kmerLength,color);
@@ -281,7 +310,7 @@ Kmer Vertex::getKey(){
return m_lowerKey;
}
-void Vertex::setKey(Kmer key){
+void Vertex::setKey(const Kmer & key){
m_lowerKey=key;
}
@@ -309,7 +338,7 @@ void Vertex::setKey(Kmer key){
*
* see also the .h that has more documentation for this.
*/
-uint8_t Vertex::convertBitmap(uint8_t bitMap){
+uint8_t Vertex::convertBitmap(uint8_t bitMap) const{
/*
* [Swap the 4 bits for children with the 4 bits for parents]
@@ -331,7 +360,8 @@ uint8_t Vertex::convertBitmap(uint8_t bitMap){
return baseMap;
}
-uint8_t Vertex::swapBits(uint8_t map,int bit1,int bit2){
+uint8_t Vertex::swapBits(uint8_t map,int bit1,int bit2) const {
+
int bit1Value=((uint64_t)map<<(63-bit1))>>63;
int bit2Value=((uint64_t)map<<(63-bit2))>>63;
@@ -384,3 +414,63 @@ uint8_t Vertex::swapBits(uint8_t map,int bit1,int bit2){
return map;
}
+Direction*Vertex::getFirstDirection()const{
+
+ return m_directions;
+}
+
+int Vertex::load(const char * buffer) {
+ int position = 0;
+ position += m_lowerKey.load(buffer);
+
+ int bytes = sizeof(m_coverage_lower);
+ memcpy(&m_coverage_lower, buffer + position, bytes);
+ position += bytes;
+
+ bytes = sizeof(m_edges_lower);
+ memcpy(&m_edges_lower, buffer + position, bytes);
+ position += bytes;
+
+ return position;
+
+}
+
+int Vertex::dump(char * buffer) const {
+
+ int position = 0;
+ position += m_lowerKey.dump(buffer);
+
+ int bytes = sizeof(m_coverage_lower);
+ memcpy(buffer + position, &m_coverage_lower, bytes);
+ position += bytes;
+
+ uint8_t edges = getEdgeSet();
+
+ bytes = sizeof(edges);
+ memcpy(buffer + position, &edges, bytes);
+ position += bytes;
+
+ return position;
+}
+
+uint8_t Vertex::getEdgeSet() const {
+
+ return m_edges_lower;
+}
+
+int Vertex::getRequiredNumberOfBytes() const {
+
+ //cout << "DEBUG sizeof(m_coverage_lower) is " << sizeof(m_coverage_lower) << endl;
+
+ return m_lowerKey.getRequiredNumberOfBytes() + sizeof(m_coverage_lower) + sizeof(m_edges_lower);
+}
+
+void Vertex::print(int kmerLength, bool colorSpaceMode) const {
+
+ cout << " Vertex key= ";
+ cout << m_lowerKey.idToWord(kmerLength, colorSpaceMode);
+ cout << " " << getVertexCoverage();
+ cout << " parents: " << getIngoingEdges(&m_lowerKey, kmerLength).size();
+ cout << " children: " << getOutgoingEdges(&m_lowerKey, kmerLength).size();
+ cout << endl;
+}
diff --git a/code/plugin_VerticesExtractor/Vertex.h b/code/VerticesExtractor/Vertex.h
similarity index 73%
rename from code/plugin_VerticesExtractor/Vertex.h
rename to code/VerticesExtractor/Vertex.h
index cca7ddc..a872a4a 100644
--- a/code/plugin_VerticesExtractor/Vertex.h
+++ b/code/VerticesExtractor/Vertex.h
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,11 +22,13 @@
#ifndef _Vertex
#define _Vertex
-#include <plugin_SequencesIndexer/ReadAnnotation.h>
-#include <plugin_SeedExtender/Direction.h>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-#include <application_core/common_functions.h>
-#include <plugin_Searcher/ColorSet.h>
+#include <code/SequencesIndexer/ReadAnnotation.h>
+#include <code/SeedExtender/Direction.h>
+#include <code/KmerAcademyBuilder/Kmer.h>
+#include <code/Mock/common_functions.h>
+#include <code/Searcher/ColorSet.h>
+
+#include <RayPlatform/store/CarriageableItem.h>
#include <fstream>
#include <stdint.h>
@@ -49,7 +51,9 @@ using namespace std;
* both k-mers of the pair.
*/
class Vertex{
-
+
+private:
+
/**
* This is for the colored graph subsystem.
* Right now, this subsystem stores only the origins
@@ -127,60 +131,87 @@ class Vertex{
*/
uint8_t m_edges_lower;
+/** the greatest rank that assembled the k-mer **/
+ Rank m_assembled;
+
/*
- * Below are the methods
+ * which hyperfusions go on this vertex at least once?
*/
- void setEdges(Kmer*kmer,uint8_t edgeData);
+ Direction*m_directions;
- uint8_t convertBitmap(uint8_t bitmap);
- uint8_t swapBits(uint8_t map,int bit1,int bit2);
/*
- * TODO: move these attributes in the private or protected zone
+ * read annotations
+ * which reads start here?
*/
+ ReadAnnotation*m_readsStartingHere;
-public:
- Kmer m_lowerKey;
- /*
+/*
* The coverage of the vertex
*/
CoverageDepth m_coverage_lower;
+/**
+ * TODO: there should be a sister class that does not store
+ * the Kmer object since all kmers are in the MyHashTable
+ * and the keys there are Kmer objects !!!
+ */
+ Kmer m_lowerKey;
+
+
/*
- * read annotations
- * which reads start here?
+ * Below are the methods
*/
- ReadAnnotation*m_readsStartingHere;
+ void setEdges(Kmer*kmer,uint8_t edgeData);
+
+ uint8_t convertBitmap(uint8_t bitmap) const;
+ uint8_t swapBits(uint8_t map,int bit1,int bit2) const;
+
+ uint8_t getEdgeSet() const;
+ void setEdgeSet(uint8_t edges);
+
+public:
/*
- * which hyperfusions go on this vertex at least once?
+ * TODO: move these attributes in the private or protected zone
*/
- Direction*m_directions;
-/** the greatest rank that assembled the k-mer **/
- Rank m_assembled;
+ Vertex();
+ ~Vertex();
void addOutgoingEdge_ClassicMethod(Kmer*vertex,Kmer*a,int k);
void addIngoingEdge_ClassicMethod(Kmer*vertex,Kmer*a,int k);
void constructor();
void setCoverage(Kmer*a,CoverageDepth coverage);
- CoverageDepth getCoverage(Kmer*p);
+ void setCoverageValue(CoverageDepth coverage);
+ CoverageDepth getCoverage(const Kmer*p) const;
+ CoverageDepth getVertexCoverage() const;
+
+ // TODO: add methods to add parents and children without
+ // having to provide the base kmer.
+ // Having the base kmer is required for workflows where the lexicographically-lower
+ // object is used.
void addOutgoingEdge(Kmer*vertex,Kmer*a,int k);
void addIngoingEdge(Kmer*vertex,Kmer*a,int k);
/*
* TODO, the vector should be a out parameter
*/
- vector<Kmer> getIngoingEdges(Kmer*a,int k);
+ vector<Kmer> getIngoingEdges(const Kmer*a,int k) const;
/*
* TODO, the vector should be a out parameter
*/
- vector<Kmer> getOutgoingEdges(Kmer*a,int k);
+ vector<Kmer> getOutgoingEdges(const Kmer*a,int k) const;
+
+/**
+ * get edges using a selector.
+ */
+ uint8_t getEdges(const Kmer*a) const;
- uint8_t getEdges(Kmer*a);
+ uint8_t getVertexEdges() const;
void deleteIngoingEdge(Kmer*vertex,Kmer*a,int k);
void deleteOutgoingEdge(Kmer*vertex,Kmer*a,int k);
@@ -195,15 +226,23 @@ public:
bool isAssembled();
bool isAssembledByGreaterRank(Rank origin);
- void write(Kmer*key,ofstream*f,int kmerLength);
- void writeAnnotations(Kmer*key,ofstream*f,int kmerLength,bool color);
+ void write(Kmer*key,ostream*f,int kmerLength);
+ void writeAnnotations(Kmer*key,ostream*f,int kmerLength,bool color);
VirtualKmerColorHandle getVirtualColor();
void setVirtualColor(VirtualKmerColorHandle handle);
Kmer getKey();
- void setKey(Kmer key);
+ void setKey(const Kmer & key);
+
+ Direction*getFirstDirection()const;
+
+ int load(const char * buffer);
+ int dump(char * buffer) const;
+ int getRequiredNumberOfBytes() const;
+
+ void print(int kmerLength, bool colorSpaceMode) const;
} ATTRIBUTE_PACKED;
diff --git a/code/plugin_VerticesExtractor/VerticesExtractor.cpp b/code/VerticesExtractor/VerticesExtractor.cpp
similarity index 86%
rename from code/plugin_VerticesExtractor/VerticesExtractor.cpp
rename to code/VerticesExtractor/VerticesExtractor.cpp
index b6ab316..468fdc1 100644
--- a/code/plugin_VerticesExtractor/VerticesExtractor.cpp
+++ b/code/VerticesExtractor/VerticesExtractor.cpp
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,25 +19,23 @@
*/
-#include <application_core/constants.h>
+#include "VerticesExtractor.h"
+
+#include <code/Mock/constants.h>
+#include <code/Mock/common_functions.h>
+
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/communication/Message.h>
+
#include <string.h>
+#include <time.h>
#include <stdio.h>
-#include <stdlib.h>
-#include <plugin_VerticesExtractor/VerticesExtractor.h>
#include <assert.h>
-#include <communication/Message.h>
-#include <time.h>
-#include <structures/StaticVector.h>
-#include <application_core/common_functions.h>
+#include <stdlib.h>
__CreatePlugin(VerticesExtractor);
- /**/
- /**/
-__CreateSlaveModeAdapter(VerticesExtractor,RAY_SLAVE_MODE_ADD_EDGES); /**/
- /**/
- /**/
-
+__CreateSlaveModeAdapter(VerticesExtractor,RAY_SLAVE_MODE_ADD_EDGES);
void VerticesExtractor::call_RAY_SLAVE_MODE_ADD_EDGES(){
@@ -59,7 +57,7 @@ void VerticesExtractor::call_RAY_SLAVE_MODE_ADD_EDGES(){
if(m_parameters->hasCheckpoint("GenomeGraph")){
cout<<"Rank "<<m_parameters->getRank()<<": checkpoint GenomeGraph exists, not extracting vertices."<<endl;
Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_VERTICES_DISTRIBUTED,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_finished=true;
return;
}
@@ -72,7 +70,7 @@ void VerticesExtractor::call_RAY_SLAVE_MODE_ADD_EDGES(){
return;
}
- if(m_mode_send_vertices_sequence_id%10000==0 &&m_mode_send_vertices_sequence_id_position==0
+ if(m_mode_send_vertices_sequence_id%100000==0 &&m_mode_send_vertices_sequence_id_position==0
&&m_mode_send_vertices_sequence_id<(int)m_myReads->size()){
string reverse="";
@@ -80,7 +78,6 @@ void VerticesExtractor::call_RAY_SLAVE_MODE_ADD_EDGES(){
reverse="(reverse complement) ";
}
printf("Rank %i is adding edges %s[%i/%i]\n",m_parameters->getRank(),reverse.c_str(),(int)m_mode_send_vertices_sequence_id+1,(int)m_myReads->size());
- fflush(stdout);
m_derivative.addX(m_mode_send_vertices_sequence_id);
m_derivative.printStatus(SLAVE_MODES[RAY_SLAVE_MODE_ADD_EDGES],RAY_SLAVE_MODE_ADD_EDGES);
@@ -101,10 +98,9 @@ void VerticesExtractor::call_RAY_SLAVE_MODE_ADD_EDGES(){
#endif
Message aMessage(NULL,0, MASTER_RANK, RAY_MPI_TAG_VERTICES_DISTRIBUTED,m_parameters->getRank());
- m_outbox->push_back(aMessage);
+ m_outbox->push_back(&aMessage);
m_finished=true;
printf("Rank %i is adding edges [%i/%i] (completed)\n",m_parameters->getRank(),(int)m_mode_send_vertices_sequence_id,(int)m_myReads->size());
- fflush(stdout);
m_bufferedDataForIngoingEdges.showStatistics(m_parameters->getRank());
m_bufferedDataForOutgoingEdges.showStatistics(m_parameters->getRank());
@@ -176,7 +172,7 @@ void VerticesExtractor::call_RAY_SLAVE_MODE_ADD_EDGES(){
// outgoing edge
// PreviousVertex(*) -> CurrentVertex
- Rank outgoingRank=m_parameters->_vertexRank(&m_previousVertex);
+ Rank outgoingRank=m_parameters->vertexRank(&m_previousVertex);
for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
m_bufferedDataForOutgoingEdges.addAt(outgoingRank,m_previousVertex.getU64(i));
}
@@ -191,7 +187,7 @@ void VerticesExtractor::call_RAY_SLAVE_MODE_ADD_EDGES(){
// ingoing edge
// PreviousVertex -> CurrentVertex(*)
- Rank ingoingRank=m_parameters->_vertexRank(¤tForwardKmer);
+ Rank ingoingRank=m_parameters->vertexRank(¤tForwardKmer);
for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
m_bufferedDataForIngoingEdges.addAt(ingoingRank,m_previousVertex.getU64(i));
}
@@ -218,7 +214,7 @@ void VerticesExtractor::call_RAY_SLAVE_MODE_ADD_EDGES(){
// outgoing edge
//
- Rank outgoingRank=m_parameters->_vertexRank(¤tReverseKmer);
+ Rank outgoingRank=m_parameters->vertexRank(¤tReverseKmer);
for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
m_bufferedDataForOutgoingEdges.addAt(outgoingRank,currentReverseKmer.getU64(i));
@@ -238,7 +234,7 @@ void VerticesExtractor::call_RAY_SLAVE_MODE_ADD_EDGES(){
MACRO_COLLECT_PROFILING_INFORMATION();
// ingoing edge
- Rank ingoingRank=m_parameters->_vertexRank(&m_previousVertexRC);
+ Rank ingoingRank=m_parameters->vertexRank(&m_previousVertexRC);
for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
m_bufferedDataForIngoingEdges.addAt(ingoingRank,currentReverseKmer.getU64(i));
@@ -277,6 +273,52 @@ void VerticesExtractor::call_RAY_SLAVE_MODE_ADD_EDGES(){
MACRO_COLLECT_PROFILING_INFORMATION();
}
+LargeCount VerticesExtractor::getDefaultNumberOfBitsForBloomFilter(){
+
+/*
+ * This is the product of these values:
+ *
+ * * Number of sequences on the rank;
+ * * K-mer length;
+ * * Number of strands (2);
+ * * Number of directions in one dimension (2);
+ */
+
+ uint64_t numberOfErrorsPerRead = 4;
+
+ uint64_t erroneousKmersPerError = m_parameters->getWordSize() * 2 * 2;
+
+ // the formula below is completely arbitrary.
+ // for serious cases, you should do an initial run
+ // and then you should modify -bloom-filter-bits
+
+ // furthermore, this formula does not consider
+ // the true kmers in the genome
+
+ uint64_t numberOfLocalReads = m_myReads->size();
+
+ uint64_t bits = numberOfErrorsPerRead * erroneousKmersPerError * numberOfLocalReads;
+
+
+ // This is useless because I now use a 64-bit integer.
+ if(bits < 0) {
+ bits = 2147483647;
+ }
+
+ uint64_t maximumBits = 1024;
+ maximumBits *= 1024*1024*8; // 1 GiB
+
+ if(bits > maximumBits) {
+ bits = maximumBits;
+
+ cout << "Rank " << m_core->getRank() << " ";
+ cout << "Warning: You should increase the number of ranks because";
+ cout << " the number of reads per rank is high" << endl;
+ }
+
+ return bits;
+}
+
void VerticesExtractor::constructor(int size,Parameters*parameters,GridTable*graph,
StaticVector*outbox,RingAllocator*outboxAllocator,ArrayOfReads*reads){
@@ -360,6 +402,8 @@ void VerticesExtractor::setProfiler(Profiler*profiler){
void VerticesExtractor::registerPlugin(ComputeCore*core){
m_plugin=core->allocatePluginHandle();
+ m_core = core;
+
PluginHandle plugin=m_plugin;
core->setPluginName(plugin,"VerticesExtractor");
@@ -404,4 +448,6 @@ void VerticesExtractor::resolveSymbols(ComputeCore*core){
core->setMessageTagToSlaveModeSwitch(m_plugin,RAY_MPI_TAG_WRITE_KMERS, RAY_SLAVE_MODE_WRITE_KMERS);
__BindPlugin(VerticesExtractor);
+
+ __BindAdapter(VerticesExtractor,RAY_SLAVE_MODE_ADD_EDGES);
}
diff --git a/code/plugin_VerticesExtractor/VerticesExtractor.h b/code/VerticesExtractor/VerticesExtractor.h
similarity index 76%
rename from code/plugin_VerticesExtractor/VerticesExtractor.h
rename to code/VerticesExtractor/VerticesExtractor.h
index f1eaedb..395b149 100644
--- a/code/plugin_VerticesExtractor/VerticesExtractor.h
+++ b/code/VerticesExtractor/VerticesExtractor.h
@@ -1,6 +1,6 @@
/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -22,23 +22,28 @@
#ifndef _VerticesExtractor
#define _VerticesExtractor
-#include <structures/StaticVector.h>
-#include <communication/BufferedData.h>
-#include <application_core/Parameters.h>
-#include <application_core/common_functions.h>
-#include <plugin_SequencesLoader/ArrayOfReads.h>
-#include <communication/Message.h>
-#include <plugin_VerticesExtractor/GridTable.h>
-#include <memory/RingAllocator.h>
-#include <plugin_SequencesLoader/Read.h>
-#include <profiling/Derivative.h>
-#include <profiling/Profiler.h>
-#include <core/ComputeCore.h>
+#include "GridTable.h"
+
+#include <code/Mock/Parameters.h>
+#include <code/Mock/common_functions.h>
+#include <code/SequencesLoader/ArrayOfReads.h>
+#include <code/SequencesLoader/Read.h>
+
+#include <RayPlatform/profiling/Derivative.h>
+#include <RayPlatform/profiling/Profiler.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/communication/BufferedData.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/core/ComputeCore.h>
+#include <RayPlatform/structures/StaticVector.h>
#include <set>
#include <vector>
using namespace std;
+__DeclarePlugin(VerticesExtractor);
+
+__DeclareSlaveModeAdapter(VerticesExtractor,RAY_SLAVE_MODE_ADD_EDGES);
/*
* Any MPI rank has some reads to process.
@@ -46,10 +51,13 @@ using namespace std;
* It also computes arcs between k-mers.
* These bits are then sent (buffered) to
* their respective owners.
+ *
* \author Sébastien Boisvert
*/
class VerticesExtractor: public CorePlugin{
+ __AddAdapter(VerticesExtractor,RAY_SLAVE_MODE_ADD_EDGES);
+
MessageTag RAY_MPI_TAG_VERTEX_INFO_REPLY;
MessageTag RAY_MPI_TAG_BUILD_GRAPH;
MessageTag RAY_MPI_TAG_WRITE_KMERS;
@@ -105,6 +113,9 @@ public:
StaticVector*outbox,RingAllocator*outboxAllocator,ArrayOfReads*myReads
);
void call_RAY_SLAVE_MODE_ADD_EDGES();
+
+ LargeCount getDefaultNumberOfBitsForBloomFilter();
+
void setReadiness();
bool finished();
diff --git a/code/application_core/Machine.cpp b/code/application_core/Machine.cpp
index 7d69eeb..2f19d5e 100644
--- a/code/application_core/Machine.cpp
+++ b/code/application_core/Machine.cpp
@@ -1,6 +1,7 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert
+ Copyright (C) 2013 Charles Joly Beauparlant
http://DeNovoAssembler.SourceForge.Net/
@@ -14,38 +15,41 @@
GNU General Public License for more details.
You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
+ along with this program (gpl-3.0.txt)
see <http://www.gnu.org/licenses/>
*/
// TODO -- there should not be any plugin construction in this file
-#include <plugin_VerticesExtractor/GridTableIterator.h>
-#include <cryptography/crypto.h>
-#include <core/OperatingSystem.h>
-#include <structures/SplayNode.h>
-#include <application_core/Machine.h>
-#include <core/statistics.h>
-#include <plugin_VerticesExtractor/VerticesExtractor.h>
-#include <profiling/Profiler.h>
-#include <sstream>
-#include <communication/Message.h>
-#include <time.h>
-#include <plugin_SeedExtender/TipWatchdog.h>
-#include <plugin_SeedExtender/BubbleTool.h>
-#include <assert.h>
-#include <application_core/common_functions.h>
+#include "Machine.h"
+
+#include <code/VerticesExtractor/GridTableIterator.h>
+#include <code/VerticesExtractor/VerticesExtractor.h>
+#include <code/SeedExtender/TipWatchdog.h>
+#include <code/SeedExtender/BubbleTool.h>
+#include <code/Mock/common_functions.h>
+#include <code/Mock/constants.h>
+#include <code/CoverageGatherer/CoverageDistribution.h>
+#include <code/SequencesLoader/Read.h>
+#include <code/SequencesLoader/Loader.h>
+
+#include <RayPlatform/profiling/Profiler.h>
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/structures/SplayTreeIterator.h>
+#include <RayPlatform/structures/SplayNode.h>
+#include <RayPlatform/core/statistics.h>
+#include <RayPlatform/core/OperatingSystem.h>
+#include <RayPlatform/memory/MyAllocator.h>
+#include <RayPlatform/cryptography/crypto.h>
+#include <RayPlatform/core/types.h>
+
#include <iostream>
#include <fstream>
-#include <plugin_CoverageGatherer/CoverageDistribution.h>
#include <string.h>
-#include <structures/SplayTreeIterator.h>
-#include <plugin_SequencesLoader/Read.h>
-#include <plugin_SequencesLoader/Loader.h>
-#include <memory/MyAllocator.h>
-#include <application_core/constants.h>
+#include <assert.h>
+#include <time.h>
#include <algorithm>
-
+#include <sstream>
using namespace std;
/* Pick-up the option -show-communication-events */
@@ -55,14 +59,17 @@ using namespace std;
*/
Machine::Machine(int argc,char**argv){
- void constructor(int*argc,char**argv);
+ m_argc=argc;
+ m_argv=argv;
+
+}
- m_computeCore.constructor(&argc,&argv);
- m_messagesHandler=m_computeCore.getMessagesHandler();
+void Machine::init(int argc,char**argv){
+ m_rank=m_computeCore.getRank();
+ m_size=m_computeCore.getSize();
+
m_alive=m_computeCore.getLife();
- m_rank=m_messagesHandler->getRank();
- m_size=m_messagesHandler->getSize();
m_inbox=m_computeCore.getInbox();
m_outbox=m_computeCore.getOutbox();
@@ -87,21 +94,25 @@ Machine::Machine(int argc,char**argv){
if(param.find("help")!=string::npos){
if(isMaster())
m_parameters.showUsage();
+
m_computeCore.destructor();
m_aborted=true;
}else if(param.find("usage")!=string::npos){
if(isMaster())
m_parameters.showUsage();
+
m_computeCore.destructor();
m_aborted=true;
}else if(param.find("man")!=string::npos){
if(isMaster())
m_parameters.showUsage();
+
m_computeCore.destructor();
m_aborted=true;
}else if(param.find("-version")!=string::npos){
if(isMaster())
showRayVersionShort();
+
m_computeCore.destructor();
m_aborted=true;
}
@@ -110,7 +121,9 @@ Machine::Machine(int argc,char**argv){
if(m_aborted)
return;
- cout<<"Rank "<<m_rank<<": Rank= "<<m_rank<<" Size= "<<m_size<<" ProcessIdentifier= "<<portableProcessId()<<" ProcessorName= "<<*(m_messagesHandler->getName())<<endl;
+ #if 0
+ " ProcessorName= "<<*(m_messagesHandler->getName())<<endl;
+ #endif
m_argc=argc;
m_argv=argv;
@@ -129,7 +142,7 @@ Machine::Machine(int argc,char**argv){
m_helper.constructor(argc,argv,&m_parameters,m_switchMan,m_outboxAllocator,m_outbox,&m_aborted,
&m_coverageDistribution,&m_numberOfMachinesDoneSendingCoverage,&m_numberOfRanksWithCoverageData,
&m_reductionOccured,m_ed,m_fusionData,m_profiler,&m_networkTest,m_seedingData,
- &m_timePrinter,&m_seedExtender,&m_scaffolder,m_messagesHandler,m_inbox,
+ &m_timePrinter,&m_seedExtender,&m_scaffolder,m_inbox,
&m_oa,&m_isFinalFusion,m_bubbleData,m_alive,
&m_CLEAR_n,&m_DISTRIBUTE_n,&m_FINISH_n,&m_searcher,
&m_numberOfRanksDoneSeeding,&m_numberOfRanksDoneDetectingDistances,&m_numberOfRanksDoneSendingDistances,
@@ -154,12 +167,13 @@ m_virtualCommunicator,&m_kmerAcademyBuilder,
*/
void Machine::start(){
+ init(m_argc,m_argv);
+
+
if(m_aborted)
return;
- m_partitioner.constructor(m_outboxAllocator,m_inbox,m_outbox,&m_parameters,m_switchMan);
-
- // TODO: move this in plugin Searcher
+ // TODO: move this in plugin Searcher
m_searcher.constructor(&m_parameters,m_outbox,&m_timePrinter,m_switchMan,m_virtualCommunicator,m_inbox,
m_outboxAllocator,&m_subgraph);
@@ -176,7 +190,6 @@ void Machine::start(){
m_numberOfMachinesReadyForEdgesDistribution=0;
m_aborted=false;
m_readyToSeed=0;
- m_wordSize=-1;
m_last_value=0;
m_mode_send_ingoing_edges=false;
m_mode_send_vertices=false;
@@ -191,37 +204,41 @@ void Machine::start(){
m_sequence_ready_machines=0;
m_isFinalFusion=false;
- m_messagesHandler->barrier();
if(isMaster()){
cout<<endl<<"**************************************************"<<endl;
cout<<"This program comes with ABSOLUTELY NO WARRANTY."<<endl;
cout<<"This is free software, and you are welcome to redistribute it"<<endl;
- cout<<"under certain conditions; see \"LICENSE\" for details."<<endl;
+ cout<<"under certain conditions; see \"LICENSE.txt\" for details."<<endl;
cout<<"**************************************************"<<endl;
cout<<endl;
- cout<<"Ray Copyright (C) 2010, 2011, 2012 Sébastien Boisvert"<<endl;
+ cout<<"Ray Copyright (C) 2010, 2011, 2012, 2013 Sébastien Boisvert"<<endl;
cout<<"Centre de recherche en infectiologie de l'Université Laval"<<endl;
cout<<"Project funded by the Canadian Institutes of Health Research (Doctoral award 200902CGM-204212-172830 to S.B.)"<<endl;
cout<<"http://denovoassembler.sf.net/"<<endl<<endl;
- cout<<"Reference to cite: "<<endl<<endl;
+ cout<<"References to cite: "<<endl<<endl;
+
+ cout << "Sébastien Boisvert, Frédéric Raymond, Élénie Godzaridis, François Laviolette and Jacques Corbeil." <<endl;
+ cout << "Ray Meta: scalable de novo metagenome assembly and profiling."<<endl;
+ cout<<"Genome Biology (BioMed Central Ltd)."<<endl;
+ cout<<"13:R122, Published: 22 December 2012"<<endl;
+ cout<<"http://dx.doi.org/doi:10.1186/gb-2012-13-12-r122"<<endl;
+ cout << endl;
+
cout<<"Sébastien Boisvert, François Laviolette & Jacques Corbeil."<<endl;
cout<<"Ray: simultaneous assembly of reads from a mix of high-throughput sequencing technologies."<<endl;
cout<<"Journal of Computational Biology (Mary Ann Liebert, Inc. publishers, New York, U.S.A.)."<<endl;
cout<<"November 2010, Volume 17, Issue 11, Pages 1519-1533."<<endl;
- cout<<"doi:10.1089/cmb.2009.0238"<<endl;
cout<<"http://dx.doi.org/doi:10.1089/cmb.2009.0238"<<endl;
cout<<endl;
}
- m_messagesHandler->barrier();
-
m_parameters.setSize(getSize());
// this peak is attained in VerticesExtractor::deleteVertices
// 2012-03-20: which peaks ?
-
+ // 2012-11-06: probably memory usage (Virtual Memory)
/**
* build the plugins now */
@@ -235,7 +252,6 @@ void Machine::start(){
m_coverageGatherer.constructor(&m_parameters,m_inbox,m_outbox,m_switchMan->getSlaveModePointer(),&m_subgraph,
m_outboxAllocator);
-
m_fusionTaskCreator.constructor(m_virtualProcessor,m_outbox,
m_outboxAllocator,m_switchMan->getSlaveModePointer(),&m_parameters,&(m_ed->m_EXTENSION_contigs),
&(m_ed->m_EXTENSION_identifiers),&(m_fusionData->m_FUSION_eliminated),
@@ -246,7 +262,6 @@ void Machine::start(){
&(m_ed->m_EXTENSION_identifiers),&(m_fusionData->m_FUSION_eliminated),
m_virtualCommunicator,&(m_fusionData->m_FINISH_newFusions));
-
m_amos.constructor(&m_parameters,m_outboxAllocator,m_outbox,m_fusionData,m_ed,m_switchMan->getMasterModePointer(),m_switchMan->getSlaveModePointer(),&m_scaffolder,
m_inbox,m_virtualCommunicator);
@@ -259,15 +274,13 @@ void Machine::start(){
m_ranksDoneAttachingReads=0;
- m_messagesHandler->barrier();
-
// TODO: check if 65536 is really a limit.
// the limit is probably in a header somewhere in application_core or in RayPlatform.
// with routing enabled, the limit is 4096 compute cores.
int maximumNumberOfProcesses=65536;
if(getSize()>maximumNumberOfProcesses){
- cout<<"The maximum number of processes is "<<maximumNumberOfProcesses<<" (this can be changed in the code)"<<endl;
+ cout<<"Error: The maximum number of processes is "<<maximumNumberOfProcesses<<" (this can be changed in the code)"<<endl;
}
assert(getSize()<=maximumNumberOfProcesses);
@@ -281,32 +294,44 @@ void Machine::start(){
}
if(isMaster()){
- showRayVersion(m_messagesHandler,fullReport);
+ showRayVersion(fullReport);
if(!fullReport){
cout<<endl;
}
}
+ int miniRanksPerRank=m_computeCore.getMiniRanksPerRank();
+
/** only show the version. */
if(fullReport){
m_computeCore.destructor();
return;
}
- m_parameters.constructor(m_argc,m_argv,getRank(),m_size);
+ m_parameters.constructor(m_argc,m_argv,getRank(),m_size,miniRanksPerRank);
+
+ m_partitioner.constructor(m_outboxAllocator,m_inbox,m_outbox,&m_parameters,m_switchMan);
- if(m_parameters.runProfiler())
+ if(m_parameters.runProfiler()) {
m_computeCore.enableProfiler();
+ }
if(m_parameters.hasOption("-with-profiler-details"))
m_computeCore.enableProfilerVerbosity();
+ if(m_parameters.hasOption("-debug")) {
+ m_computeCore.enableProfiler();
+ m_computeCore.enableProfilerVerbosity();
+ }
+
if(m_parameters.showCommunicationEvents())
m_computeCore.showCommunicationEvents();
+ string*name=m_computeCore.getMessagesHandler()->getName();
+
// initiate the network test.
- m_networkTest.constructor(m_rank,m_size,m_inbox,m_outbox,&m_parameters,m_outboxAllocator,m_messagesHandler->getName(),
+ m_networkTest.constructor(m_rank,m_size,m_inbox,m_outbox,&m_parameters,m_outboxAllocator,name,
&m_timePrinter);
m_networkTest.setSwitchMan(m_switchMan);
@@ -315,48 +340,82 @@ void Machine::start(){
m_persistentAllocator.constructor(PERSISTENT_ALLOCATOR_CHUNK_SIZE,"RAY_MALLOC_TYPE_PERSISTENT_DATA_ALLOCATOR",
m_parameters.showMemoryAllocations());
- int directionAllocatorChunkSize=4194304; // 4 MiB
+ // options are loaded from here
+ // plus the directory exists now
+ if(m_parameters.hasOption("-route-messages")){
- m_directionsAllocator.constructor(directionAllocatorChunkSize,"RAY_MALLOC_TYPE_WAVE_ALLOCATOR",
- m_parameters.showMemoryAllocations());
+ ostringstream routingPrefix;
+ routingPrefix<<m_parameters.getPrefix()<<"/Routing/";
+
+ m_router->enable(m_inbox,m_outbox,m_outboxAllocator,m_parameters.getRank(),
+ routingPrefix.str(),m_parameters.getSize(),
+ m_parameters.getConnectionType(),m_parameters.getRoutingDegree());
+
+ // update the connections
+ vector<int> connections;
+ m_router->getGraph()->getIncomingConnections(m_parameters.getRank(),&connections);
+
+ }
+
+ if(m_parameters.hasOption("-verify-message-integrity")) {
+ m_computeCore.enableCheckSums();
+ }
+
+/*
+ * We need to know if we must abort. All the RayPlatform
+ * engine will start anyway on each core, so we need to stop
+ * them if necessary.
+ */
+ bool oldDirectoryExists=false;
- /** create the directory for the assembly */
-
string directory=m_parameters.getPrefix();
if(fileExists(directory.c_str())){
- if(m_parameters.getRank() == MASTER_RANK)
+ if(m_parameters.getRank() == MASTER_RANK){
cout<<"Error, "<<directory<<" already exists, change the -o parameter to another value."<<endl;
+ cout<<endl;
- m_computeCore.destructor();
- return;
- }
+/*
+ * Tell the first plugin that the job will not run.
+ */
+ m_helper.notifyThatOldDirectoryExists();
- m_messagesHandler->barrier();
+ oldDirectoryExists=true;
+ }
+ }
// create the directory
- if(m_parameters.getRank() == MASTER_RANK){
+ /** create the directory for the assembly */
+ if(!oldDirectoryExists && m_parameters.getRank() == MASTER_RANK){
createDirectory(directory.c_str());
m_parameters.writeCommandFile();
m_timePrinter.setFile(m_parameters.getPrefix());
- }
- cout<<endl;
+/*
+ * Write routing information only if the directory is sane.
+ */
+ if(m_router->isEnabled())
+ m_router->writeFiles();
+
+ }
// register the plugins.
registerPlugins();
- cout<<endl;
-
// write a report about plugins
- if(m_parameters.getRank()==MASTER_RANK){
- ostringstream directory;
- directory<<m_parameters.getPrefix()<<"/Plugins";
+ if(!oldDirectoryExists && m_parameters.getRank()==MASTER_RANK){
+
+ bool mustWritePluginData = m_parameters.hasOption("-write-plugin-data");
+
+ if(mustWritePluginData) {
+ ostringstream directory;
+ directory<<m_parameters.getPrefix()<<"/Plugins";
- string file=directory.str();
- createDirectory(file.c_str());
+ string file=directory.str();
+ createDirectory(file.c_str());
- m_computeCore.printPlugins(file);
+ m_computeCore.printPlugins(file);
+ }
// write the version of RayPlatform
@@ -368,28 +427,8 @@ void Machine::start(){
f7.close();
}
- ostringstream routingPrefix;
- routingPrefix<<m_parameters.getPrefix()<<"/Routing/";
-
- // options are loaded from here
- // plus the directory exists now
- if(m_parameters.hasOption("-route-messages")){
-
- if(m_parameters.getRank()== MASTER_RANK)
- createDirectory(routingPrefix.str().c_str());
-
- m_router->enable(m_inbox,m_outbox,m_outboxAllocator,m_parameters.getRank(),
- routingPrefix.str(),m_parameters.getSize(),
- m_parameters.getConnectionType(),m_parameters.getRoutingDegree());
-
- // update the connections
- vector<int> connections;
- m_router->getGraph()->getIncomingConnections(m_parameters.getRank(),&connections);
- m_messagesHandler->setConnections(&connections);
- }
-
// set the attributes of the seed extender.
- m_seedExtender.constructor(&m_parameters,&m_directionsAllocator,m_ed,&m_subgraph,m_inbox,m_profiler,
+ m_seedExtender.constructor(&m_parameters,m_ed,&m_subgraph,m_inbox,m_profiler,
m_outbox,m_seedingData,m_switchMan->getSlaveModePointer(),&(m_seedingData->m_SEEDING_vertexCoverageRequested),
&(m_seedingData->m_SEEDING_vertexCoverageReceived),m_outboxAllocator,m_fusionData,
&(m_seedingData->m_SEEDING_seeds),m_bubbleData,&(m_seedingData->m_SEEDING_edgesRequested),
@@ -397,15 +436,12 @@ void Machine::start(){
&(m_seedingData->m_SEEDING_currentVertex),&(m_seedingData->m_SEEDING_receivedVertexCoverage),
&(m_seedingData->m_SEEDING_receivedOutgoingEdges),&m_c,&m_oa);
-
m_kmerAcademyBuilder.constructor(m_parameters.getSize(),&m_parameters,&m_subgraph,
&m_myReads,m_inbox,m_outbox,m_switchMan->getSlaveModePointer(),m_outboxAllocator);
m_si.constructor(&m_parameters,m_outboxAllocator,m_inbox,m_outbox,m_virtualCommunicator,
m_switchMan->getSlaveModePointer(),&m_myReads);
-
-
m_profiler = m_computeCore.getProfiler();
m_profiler->constructor(m_parameters.runProfiler());
@@ -438,24 +474,18 @@ void Machine::start(){
&m_parameters,m_seedingData,m_inbox,m_virtualCommunicator);
m_subgraph.constructor(getRank(),&m_parameters);
+
+ if(!oldDirectoryExists)
+ m_subgraph.printStatus();
- m_seedingData->constructor(&m_seedExtender,getRank(),getSize(),m_outbox,m_outboxAllocator,m_switchMan->getSlaveModePointer(),&m_parameters,&m_wordSize,&m_subgraph,m_inbox,m_virtualCommunicator);
+ m_seedingData->constructor(&m_seedExtender,getRank(),getSize(),m_outbox,m_outboxAllocator,m_switchMan->getSlaveModePointer(),&m_parameters,&m_subgraph,m_inbox,m_virtualCommunicator);
(*m_alive)=true;
m_totalLetters=0;
- m_messagesHandler->barrier();
-
- if(m_parameters.showMemoryUsage()){
- showMemoryUsage(getRank());
- }
-
- m_messagesHandler->barrier();
-
- if(isMaster()){
- cout<<endl;
- }
-
+/*
+ * Build the monstruous object.
+ */
m_mp.constructor(
m_router,
m_seedingData,
@@ -471,7 +501,6 @@ m_seedingData,
getRank(),
&m_numberOfMachinesDoneSendingEdges,
m_fusionData,
- &m_wordSize,
&m_myReads,
getSize(),
m_inboxAllocator,
@@ -489,7 +518,6 @@ m_seedingData,
&m_readyToSeed,
&m_FINISH_n,
&m_reductionOccured,
- &m_directionsAllocator,
&m_mode_send_coverage_iterator,
&m_coverageDistribution,
&m_sequence_ready_machines,
@@ -505,6 +533,23 @@ m_seedingData,
&m_numberOfRanksWithCoverageData,&m_seedExtender,
m_switchMan->getMasterModePointer(),&m_isFinalFusion,&m_si);
+ bool mustWriteSchedulingInformation = m_parameters.hasOption("-write-scheduling-data");
+
+/*
+ * Create the Scheduling before calling run().
+ * run() is like a barrier.
+ */
+ // log ticks
+ if(!oldDirectoryExists && m_parameters.getRank()==MASTER_RANK
+ && mustWriteSchedulingInformation){
+
+ ostringstream scheduling;
+ scheduling<<m_parameters.getPrefix()<<"/Scheduling/";
+ createDirectory(scheduling.str().c_str());
+ }
+
+ cout<<"Rank "<<m_rank<<": Rank= "<<m_rank<<" Size= "<<m_size<<" ProcessIdentifier= "<<portableProcessId()<<endl;
+
if(m_argc==1||((string)m_argv[1])=="--help"){
if(isMaster()){
m_aborted=true;
@@ -515,68 +560,46 @@ m_seedingData,
m_computeCore.run();
}
- if(isMaster() && !m_aborted){
+ /*if(isMaster() && !m_aborted){
m_timePrinter.printDurations();
-
cout<<endl;
- }
-
- m_messagesHandler->barrier();
-
- if(m_parameters.showMemoryUsage()){
- showMemoryUsage(getRank());
- }
-
- m_messagesHandler->barrier();
+ }*/
+/*
+ * TODO: the code above should go in a plugin, not here.
+ */
if(isMaster() && !m_aborted && !m_parameters.hasOption("-test-network-only")){
m_scaffolder.printFinalMessage();
- cout<<endl;
- cout<<"Rank "<<getRank()<<" wrote "<<m_parameters.getOutputFile()<<endl;
- cout<<"Rank "<<getRank()<<" wrote "<<m_parameters.getScaffoldFile()<<endl;
- cout<<"Check for "<<m_parameters.getPrefix()<<"*"<<endl;
- cout<<endl;
- if(m_parameters.useAmos()){
- cout<<"Rank "<<getRank()<<" wrote "<<m_parameters.getAmosFile()<<" (reads mapped onto contiguous sequences in AMOS format)"<<endl;
-
- }
- cout<<endl;
}
- m_messagesHandler->barrier();
+ if(!oldDirectoryExists && !m_aborted
+ && mustWriteSchedulingInformation){
+ ostringstream masterTicks;
+ masterTicks<<m_parameters.getPrefix()<<"/Scheduling/"<<m_parameters.getRank()<<".MasterTicks.txt";
+ ofstream f1(masterTicks.str().c_str());
+ m_tickLogger->printMasterTicks(&f1);
+ f1.close();
- // log ticks
- if(m_parameters.getRank()==MASTER_RANK){
- ostringstream scheduling;
- scheduling<<m_parameters.getPrefix()<<"/Scheduling/";
- createDirectory(scheduling.str().c_str());
+ ostringstream slaveTicks;
+ slaveTicks<<m_parameters.getPrefix()<<"/Scheduling/"<<m_parameters.getRank()<<".SlaveTicks.txt";
+ ofstream f2(slaveTicks.str().c_str());
+ m_tickLogger->printSlaveTicks(&f2);
+ f2.close();
}
- // wait for master to create the directory.
- m_messagesHandler->barrier();
-
- ostringstream masterTicks;
- masterTicks<<m_parameters.getPrefix()<<"/Scheduling/"<<m_parameters.getRank()<<".MasterTicks.txt";
- ofstream f1(masterTicks.str().c_str());
- m_tickLogger->printMasterTicks(&f1);
- f1.close();
-
- ostringstream slaveTicks;
- slaveTicks<<m_parameters.getPrefix()<<"/Scheduling/"<<m_parameters.getRank()<<".SlaveTicks.txt";
- ofstream f2(slaveTicks.str().c_str());
- m_tickLogger->printSlaveTicks(&f2);
- f2.close();
-
-
- m_messagesHandler->freeLeftovers();
m_persistentAllocator.clear();
- m_directionsAllocator.clear();
- m_inboxAllocator->clear();
- m_outboxAllocator->clear();
m_diskAllocator.clear();
+/*
+ * The app is responsible for destroying the core.
+ * It should work also without this call because it
+ * is RayPlatform that destroys the MessagesHandler.
+ */
+
+// TODO: it would be nice to have a callback to deregister plugins
+
m_computeCore.destructor();
}
@@ -599,7 +622,7 @@ Machine::~Machine(){
void Machine::showRayVersionShort(){
- cout<<"Ray version "<<RAY_VERSION<<endl;
+ cout<<"Ray version "<<CONFIG_RAY_VERSION<<endl;
cout<<"License for Ray: GNU General Public License version 3"<<endl;
cout<<"RayPlatform version: "<<m_computeCore.getRayPlatformVersion()<<endl;
@@ -609,14 +632,13 @@ void Machine::showRayVersionShort(){
CoverageDepth maximumCoverageDepth=0;
maximumCoverageDepth--; // underflow it
- cout<<"MAXKMERLENGTH: "<<MAXKMERLENGTH<<endl;
- cout<<"KMER_U64_ARRAY_SIZE: "<<KMER_U64_ARRAY_SIZE<<endl;
+ cout<<"MAXKMERLENGTH: "<<CONFIG_MAXKMERLENGTH<<endl;
cout<<"Maximum coverage depth stored by CoverageDepth: "<<maximumCoverageDepth<<endl;
cout<<"MAXIMUM_MESSAGE_SIZE_IN_BYTES: "<<MAXIMUM_MESSAGE_SIZE_IN_BYTES<<" bytes"<<endl;
/*
content
- cout<<"option";
+ cout<<"option";
#ifdef option
cout<<"y";
#else
@@ -625,16 +647,16 @@ content
cout<<endl;
list
- FORCE_PACKING
- ASSERT
- HAVE_LIBZ
- HAVE_LIBBZ2
- CONFIG_PROFILER_COLLECT
- CONFIG_CLOCK_GETTIME
- __linux__
- _MSC_VER
- __GNUC__
- RAY_32_BITS
+ FORCE_PACKING
+ ASSERT
+ CONFIG_HAVE_LIBZ
+ HAVE_LIBBZ2
+ CONFIG_PROFILER_COLLECT
+ CONFIG_CLOCK_GETTIME
+ __linux__
+ _MSC_VER
+ __GNUC__
+ RAY_32_BITS
RAY_64_BITS
for i in $(cat list ); do exp="s/option/$i/g"; sed $exp content; done > list2
@@ -643,7 +665,7 @@ for i in $(cat list ); do exp="s/option/$i/g"; sed $exp content; done > list2
/* generated code */
cout<<"FORCE_PACKING = ";
- #ifdef FORCE_PACKING
+ #ifdef CONFIG_FORCE_PACKING
cout<<"y";
#else
cout<<"n";
@@ -659,7 +681,7 @@ for i in $(cat list ); do exp="s/option/$i/g"; sed $exp content; done > list2
cout<<endl;
cout<<"HAVE_LIBZ = ";
- #ifdef HAVE_LIBZ
+ #ifdef CONFIG_HAVE_LIBZ
cout<<"y";
#else
cout<<"n";
@@ -667,14 +689,14 @@ for i in $(cat list ); do exp="s/option/$i/g"; sed $exp content; done > list2
cout<<endl;
cout<<"HAVE_LIBBZ2 = ";
- #ifdef HAVE_LIBBZ2
+ #ifdef CONFIG_HAVE_LIBBZ2
cout<<"y";
#else
cout<<"n";
#endif
cout<<endl;
- cout<<"CONFIG_PROFILER_COLLECT = ";
+ cout<<"PROFILER_COLLECT = ";
#ifdef CONFIG_PROFILER_COLLECT
cout<<"y";
#else
@@ -682,7 +704,15 @@ for i in $(cat list ); do exp="s/option/$i/g"; sed $exp content; done > list2
#endif
cout<<endl;
- cout<<"CONFIG_CLOCK_GETTIME = ";
+ cout<<"MINIRANKS = ";
+#ifdef CONFIG_MINI_RANKS
+ cout<<"y";
+#else
+ cout<<"n";
+#endif
+ cout<<endl;
+
+ cout<<"CLOCK_GETTIME = ";
#ifdef CONFIG_CLOCK_GETTIME
cout<<"y";
#else
@@ -749,6 +779,17 @@ for i in $(cat list ); do exp="s/option/$i/g"; sed $exp content; done > list2
cout<<"Compiler: GNU gcc/g++ "<<__VERSION__<<endl;
#endif
+/**
+ * Report availability of MPI I/O
+ */
+ cout<<"MPI I/O: ";
+
+#ifdef CONFIG_MPI_IO
+ cout<<"y";
+#else
+ cout<<"n";
+#endif
+ cout<<endl;
#ifdef __SSE4_2__
cout<<"With SSE 4.2"<<endl;
@@ -759,39 +800,9 @@ for i in $(cat list ); do exp="s/option/$i/g"; sed $exp content; done > list2
#endif
}
-void Machine::showRayVersion(MessagesHandler*messagesHandler,bool fullReport){
+void Machine::showRayVersion(bool fullReport){
showRayVersionShort();
- cout<<endl;
- cout<<"Rank "<<MASTER_RANK<<": Operating System: ";
- cout<<getOperatingSystem()<<endl;
-
-
- cout<<"Message-passing interface"<<endl;
- cout<<endl;
- cout<<"Rank "<<MASTER_RANK<<": Message-Passing Interface implementation: ";
- cout<<messagesHandler->getMessagePassingInterfaceImplementation()<<endl;
-
- int version;
- int subversion;
- messagesHandler->version(&version,&subversion);
-
- cout<<"Rank "<<MASTER_RANK<<": Message-Passing Interface standard version: "<<version<<"."<<subversion<<""<<endl;
-
-
- cout<<endl;
-
- if(!fullReport)
- return;
-
- cout<<endl;
-
- cout<<"Number of MPI ranks: "<<messagesHandler->getSize()<<endl;
- cout<<"Ray master MPI rank: "<<MASTER_RANK<<endl;
- cout<<"Ray slave MPI ranks: 0-"<<messagesHandler->getSize()-1<<endl;
- cout<<endl;
-
- cout<<endl;
}
void Machine::registerPlugins(){
@@ -818,6 +829,10 @@ void Machine::registerPlugins(){
m_computeCore.registerPlugin(&m_phylogeny);
m_computeCore.registerPlugin(&m_genomeNeighbourhood);
m_computeCore.registerPlugin(&m_ontologyPlugin);
+ m_computeCore.registerPlugin(&m_mock);
+ m_computeCore.registerPlugin(&m_example);
+ m_computeCore.registerPlugin(&m_pathEvaluator);
+ m_computeCore.registerPlugin(&m_spuriousSeedAnnihilator);
// resolve the symbols
// this is done here because we want to write a summary for
@@ -825,3 +840,12 @@ void Machine::registerPlugins(){
// otherwise, symbols are resolved when ComputeCore::run() is called.
m_computeCore.resolveSymbols();
}
+
+void Machine::run(){
+
+ start();
+}
+
+
+
+
diff --git a/code/application_core/Machine.h b/code/application_core/Machine.h
index bf2e9a8..99aa34b 100644
--- a/code/application_core/Machine.h
+++ b/code/application_core/Machine.h
@@ -1,6 +1,7 @@
/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Ray -- Parallel genome assemblies for parallel DNA sequencing
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
+ Copyright (C) 2013 Charles Joly Beauparlant
http://DeNovoAssembler.SourceForge.Net/
@@ -22,94 +23,94 @@
#ifndef _Machine
#define _Machine
+/** Chapter II. stuff related to the Ray Genome Assembler **/
+/** stuff specific to the Ray assembler that don't fit elsewhere */
+
+#include <code/Mock/common_functions.h> /* common code for all plugins */
+#include <code/Mock/Parameters.h> /* command-line parsing */
+
+// also in this category is application_core/Machine.h
+
+/** list of RayPlatform CorePlugin objects for the Ray genome software suite. */
+/* (in no particular order) */
+
+#include <code/SeedExtender/DepthFirstSearchData.h>
+#include <code/GenomeNeighbourhood/GenomeNeighbourhood.h>
+#include <code/Scaffolder/Scaffolder.h>
+#include <code/VerticesExtractor/GridTable.h>
+#include <code/Partitioner/Partitioner.h>
+#include <code/SequencesLoader/ArrayOfReads.h>
+#include <code/SeedingData/SeedingData.h>
+#include <code/SequencesIndexer/SequencesIndexer.h>
+#include <code/SeedExtender/SeedExtender.h>
+#include <code/SequencesLoader/SequencesLoader.h>
+#include <code/Library/Library.h>
+#include <code/CoverageGatherer/CoverageGatherer.h>
+#include <code/MessageProcessor/MessageProcessor.h>
+#include <code/VerticesExtractor/Vertex.h>
+#include <code/SeedExtender/OpenAssemblerChooser.h>
+#include <code/SeedExtender/BubbleData.h>
+#include <code/VerticesExtractor/VerticesExtractor.h>
+#include <code/Amos/Amos.h>
+#include <code/KmerAcademyBuilder/KmerAcademyBuilder.h>
+#include <code/EdgePurger/EdgePurger.h>
+#include <code/NetworkTest/NetworkTest.h>
+#include <code/FusionTaskCreator/FusionTaskCreator.h>
+#include <code/JoinerTaskCreator/JoinerTaskCreator.h>
+#include <code/SequencesLoader/Read.h>
+#include <code/Searcher/Searcher.h>
+#include <code/MachineHelper/MachineHelper.h>
+#include <code/TaxonomyViewer/TaxonomyViewer.h>
+#include <code/GenomeNeighbourhood/GenomeNeighbourhood.h>
+#include <code/GeneOntology/GeneOntology.h>
+#include <code/Mock/Mock.h>
+#include <code/Example/Example.h>
+#include <code/PathEvaluator/PathEvaluator.h>
+#include <code/SpuriousSeedAnnihilator/SpuriousSeedAnnihilator.h>
+
/** Chapter I. stuff pulled from RayPlatform, called The Platform hereafter */
/** the heart of RayPlatform **/
/** ComputeCore is a facade (a design pattern) **/
-#include <core/ComputeCore.h>
+#include <RayPlatform/core/ComputeCore.h>
+#include "RayPlatform/core/MiniRank.h"
/** communication */
-#include <communication/VirtualCommunicator.h> /** virtual stuff */
-#include <communication/Message.h>
-#include <communication/MessageRouter.h>
-#include <communication/MessagesHandler.h> /** MPI wrapper **/
+#include <RayPlatform/communication/VirtualCommunicator.h> /** virtual stuff */
+#include <RayPlatform/communication/Message.h>
+#include <RayPlatform/communication/MessageRouter.h>
/** scheduling, virtual pools */
-#include <scheduling/VirtualProcessor.h> /** thread pool **/
+#include <RayPlatform/scheduling/VirtualProcessor.h> /** thread pool **/
/** distributed scheduler, this is a state machine */
/* design pattern: state */
-#include <scheduling/SwitchMan.h>
+#include <RayPlatform/scheduling/SwitchMan.h>
/** memory stuff */
-#include <memory/RingAllocator.h>
-#include <memory/MyAllocator.h>
+#include <RayPlatform/memory/RingAllocator.h>
+#include <RayPlatform/memory/MyAllocator.h>
/** data structure */
-#include <structures/StaticVector.h>
-#include <structures/SplayTree.h>
-#include <structures/SplayTreeIterator.h>
+#include <RayPlatform/structures/StaticVector.h>
+#include <RayPlatform/structures/SplayTree.h>
+#include <RayPlatform/structures/SplayTreeIterator.h>
/** run-time code profiling */
-#include <profiling/TimePrinter.h>
-#include <profiling/Profiler.h>
-#include <profiling/TickLogger.h>
+#include <RayPlatform/profiling/TimePrinter.h>
+#include <RayPlatform/profiling/Profiler.h>
+#include <RayPlatform/profiling/TickLogger.h>
/** interfaces to define adapters compatible with RayPlatform **/
-#include <handlers/MessageTagHandler.h>
-#include <handlers/SlaveModeHandler.h>
-#include <handlers/MessageTagHandler.h>
-
-
-/** Chapter II. stuff related to the Ray Genome Assembler **/
-
-/** stuff specific to the Ray assembler that don't fit elsewhere */
-
-#include <application_core/common_functions.h> /* common code for all plugins */
-#include <application_core/Parameters.h> /* command-line parsing */
-
-// also in this category is application_core/Machine.h
-
-
-/** list of RayPlatform CorePlugin objects for the Ray genome software suite. */
-/* (in no particular order) */
-
-#include <plugin_SeedExtender/DepthFirstSearchData.h>
-#include <plugin_GenomeNeighbourhood/GenomeNeighbourhood.h>
-#include <plugin_Scaffolder/Scaffolder.h>
-#include <plugin_VerticesExtractor/GridTable.h>
-#include <plugin_Partitioner/Partitioner.h>
-#include <plugin_SequencesLoader/ArrayOfReads.h>
-#include <plugin_SeedingData/SeedingData.h>
-#include <plugin_SequencesIndexer/SequencesIndexer.h>
-#include <plugin_SeedExtender/SeedExtender.h>
-#include <plugin_SequencesLoader/SequencesLoader.h>
-#include <plugin_Library/Library.h>
-#include <plugin_CoverageGatherer/CoverageGatherer.h>
-#include <plugin_MessageProcessor/MessageProcessor.h>
-#include <plugin_VerticesExtractor/Vertex.h>
-#include <plugin_SeedExtender/OpenAssemblerChooser.h>
-#include <plugin_SeedExtender/BubbleData.h>
-#include <plugin_VerticesExtractor/VerticesExtractor.h>
-#include <plugin_Amos/Amos.h>
-#include <plugin_KmerAcademyBuilder/KmerAcademyBuilder.h>
-#include <plugin_EdgePurger/EdgePurger.h>
-#include <plugin_NetworkTest/NetworkTest.h>
-#include <plugin_FusionTaskCreator/FusionTaskCreator.h>
-#include <plugin_JoinerTaskCreator/JoinerTaskCreator.h>
-#include <plugin_SequencesLoader/Read.h>
-#include <plugin_Searcher/Searcher.h>
-#include <plugin_MachineHelper/MachineHelper.h>
-#include <plugin_PhylogenyViewer/PhylogenyViewer.h>
-#include <plugin_GenomeNeighbourhood/GenomeNeighbourhood.h>
-#include <plugin_GeneOntology/GeneOntology.h>
+#include <RayPlatform/handlers/MessageTagHandler.h>
+#include <RayPlatform/handlers/SlaveModeHandler.h>
+#include <RayPlatform/handlers/MessageTagHandler.h>
/** C++ bits **/
#include <map>
#include <vector>
#include <set>
#include <time.h>
-
using namespace std;
/**
@@ -122,9 +123,13 @@ using namespace std;
*
* \author Sébastien Boisvert
*/
-class Machine{
+class Machine : public MiniRank{
+
+/*
+ * Useless placeholder.
+ */
+ Mock m_mock;
- ComputeCore m_computeCore;
Chooser m_c;
MachineHelper m_helper;
@@ -132,7 +137,8 @@ class Machine{
* transcriptome) studied
*/
GenomeNeighbourhood m_genomeNeighbourhood;
-
+ PathEvaluator m_pathEvaluator;
+ SpuriousSeedAnnihilator m_spuriousSeedAnnihilator;
Searcher m_searcher;
@@ -154,8 +160,9 @@ class Machine{
Partitioner m_partitioner;
NetworkTest m_networkTest;
EdgePurger m_edgePurger;
- PhylogenyViewer m_phylogeny;
+ TaxonomyViewer m_phylogeny;
GeneOntology m_ontologyPlugin;
+ Example m_example;
KmerAcademyBuilder m_kmerAcademyBuilder;
bool m_initialisedAcademy;
@@ -164,7 +171,6 @@ class Machine{
Amos m_amos;
Library m_library;
- MessagesHandler*m_messagesHandler;
MyAllocator m_diskAllocator;
int m_numberOfRanksWithCoverageData;
@@ -173,7 +179,6 @@ class Machine{
MessageProcessor m_mp;
int m_argc;
char**m_argv;
- int m_wordSize;
int m_last_value;
time_t m_lastTime;
bool m_mode_send_outgoing_edges;
@@ -247,9 +252,6 @@ class Machine{
// allocator for persistent data
MyAllocator m_persistentAllocator;
- // allocator for directions in the de Bruijn graph
- MyAllocator m_directionsAllocator;
-
ArrayOfReads m_myReads;
bool m_mode_send_vertices;
@@ -304,19 +306,29 @@ class Machine{
int getRank();
- void showRayVersion(MessagesHandler*messagesHandler,bool fullReport);
+ void showRayVersion(bool fullReport);
void showRayVersionShort();
void registerPlugins();
+ void start();
+
+ StaticVector*getInbox();
+ StaticVector*getOutbox();
+ RingAllocator*getOutboxAllocator();
+
+ void init(int argc,char**argv);
+
public:
/*
* this is the only public bit
*/
Machine(int argc,char**argv);
- void start();
~Machine();
+
+ void run();
+
};
#endif
diff --git a/code/application_core/Makefile b/code/application_core/Makefile
index 6ddd7ad..9ff746b 100644
--- a/code/application_core/Makefile
+++ b/code/application_core/Makefile
@@ -1,5 +1,4 @@
-
-application_core-y += application_core/ray_main.o application_core/Machine.o
-application_core-y += application_core/Parameters.o application_core/common_functions.o
+#application_core-y += code/application_core/ray_main.o
+application_core-y += code/application_core/Machine.o
obj-y += $(application_core-y)
diff --git a/code/application_core/ray_main.cpp b/code/application_core/ray_main.cpp
index cbe8b74..c2223c3 100644
--- a/code/application_core/ray_main.cpp
+++ b/code/application_core/ray_main.cpp
@@ -1,6 +1,6 @@
/*
Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
+ Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
http://DeNovoAssembler.SourceForge.Net/
@@ -19,14 +19,18 @@
*/
-#include <iostream>
-#include <application_core/Machine.h>
-#include <stdlib.h>
+#include "Machine.h"
+
+#include <RayPlatform/core/RankProcess.h>
+
using namespace std;
int main(int argc,char**argv){
- Machine*m=new Machine(argc,argv);
- m->start();
+
+ RankProcess<Machine> applicationContainer;
+ applicationContainer.constructor(&argc,&argv);
+ applicationContainer.run();
+
return EXIT_SUCCESS;
}
diff --git a/code/plugin_Amos/Makefile b/code/plugin_Amos/Makefile
deleted file mode 100644
index 4d3b9d1..0000000
--- a/code/plugin_Amos/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-
-Amos-y += plugin_Amos/Amos.o
-
-obj-y += $(Amos-y)
-
-
-
diff --git a/code/plugin_CoverageGatherer/Makefile b/code/plugin_CoverageGatherer/Makefile
deleted file mode 100644
index ae6b63c..0000000
--- a/code/plugin_CoverageGatherer/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-
-CoverageGatherer-y += plugin_CoverageGatherer/CoverageGatherer.o
-CoverageGatherer-y += plugin_CoverageGatherer/CoverageDistribution.o
-
-obj-y += $(CoverageGatherer-y)
-
-
-
diff --git a/code/plugin_EdgePurger/Makefile b/code/plugin_EdgePurger/Makefile
deleted file mode 100644
index af1d536..0000000
--- a/code/plugin_EdgePurger/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-# plugin EdgePurger
-
-EdgePurger-y += plugin_EdgePurger/EdgePurger.o plugin_EdgePurger/EdgePurgerWorker.o
-
-obj-y += $(EdgePurger-y)
-
-
-
diff --git a/code/plugin_FusionData/Makefile b/code/plugin_FusionData/Makefile
deleted file mode 100644
index 84902ea..0000000
--- a/code/plugin_FusionData/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-
-FusionData-y += plugin_FusionData/FusionData.o
-
-obj-y += $(FusionData-y)
-
diff --git a/code/plugin_FusionTaskCreator/Makefile b/code/plugin_FusionTaskCreator/Makefile
deleted file mode 100644
index 62e21b1..0000000
--- a/code/plugin_FusionTaskCreator/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-
-FusionTaskCreator-y += plugin_FusionTaskCreator/FusionWorker.o
-FusionTaskCreator-y += plugin_FusionTaskCreator/FusionTaskCreator.o
-
-obj-y += $(FusionTaskCreator-y)
-
diff --git a/code/plugin_GeneOntology/Makefile b/code/plugin_GeneOntology/Makefile
deleted file mode 100644
index 507abd1..0000000
--- a/code/plugin_GeneOntology/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-
-GeneOntology-y += plugin_GeneOntology/KeyEncoder.o
-GeneOntology-y += plugin_GeneOntology/GeneOntology.o
-
-obj-y += $(GeneOntology-y)
-
diff --git a/code/plugin_GenomeNeighbourhood/Makefile b/code/plugin_GenomeNeighbourhood/Makefile
deleted file mode 100644
index d555efc..0000000
--- a/code/plugin_GenomeNeighbourhood/Makefile
+++ /dev/null
@@ -1,11 +0,0 @@
-# this file contains the blueprints for
-# the plugin GenomeNeighbourhood
-# \author Sébastien Boisvert
-# \date 2012-03-20 initial version
-
-GenomeNeighbourhood-y += plugin_GenomeNeighbourhood/GenomeNeighbourhood.o
-GenomeNeighbourhood-y += plugin_GenomeNeighbourhood/Neighbour.o
-GenomeNeighbourhood-y += plugin_GenomeNeighbourhood/NeighbourPair.o
-
-obj-y += $(GenomeNeighbourhood-y)
-
diff --git a/code/plugin_JoinerTaskCreator/Makefile b/code/plugin_JoinerTaskCreator/Makefile
deleted file mode 100644
index 40db8ae..0000000
--- a/code/plugin_JoinerTaskCreator/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-
-JoinerTaskCreator-y += plugin_JoinerTaskCreator/JoinerWorker.o
-JoinerTaskCreator-y += plugin_JoinerTaskCreator/JoinerTaskCreator.o
-
-obj-y += $(JoinerTaskCreator-y)
-
-
-
diff --git a/code/plugin_KmerAcademyBuilder/Makefile b/code/plugin_KmerAcademyBuilder/Makefile
deleted file mode 100644
index 3b18b60..0000000
--- a/code/plugin_KmerAcademyBuilder/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-KmerAcademyBuilder-y += plugin_KmerAcademyBuilder/KmerAcademyBuilder.o
-KmerAcademyBuilder-y += plugin_KmerAcademyBuilder/BloomFilter.o
-KmerAcademyBuilder-y += plugin_KmerAcademyBuilder/Kmer.o
-
-obj-y += $(KmerAcademyBuilder-y)
-
diff --git a/code/plugin_Library/Makefile b/code/plugin_Library/Makefile
deleted file mode 100644
index dfa7156..0000000
--- a/code/plugin_Library/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-
-Library-y += plugin_Library/LibraryPeakFinder.o plugin_Library/LibraryWorker.o
-Library-y += plugin_Library/Library.o
-
-obj-y += $(Library-y)
-
-
-
diff --git a/code/plugin_MachineHelper/Makefile b/code/plugin_MachineHelper/Makefile
deleted file mode 100644
index 1988d5d..0000000
--- a/code/plugin_MachineHelper/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-
-MachineHelper-y += plugin_MachineHelper/MachineHelper.o
-
-obj-y += $(MachineHelper-y)
-
-
-
diff --git a/code/plugin_MessageProcessor/Makefile b/code/plugin_MessageProcessor/Makefile
deleted file mode 100644
index 0c29e56..0000000
--- a/code/plugin_MessageProcessor/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-
-MessageProcessor-y += plugin_MessageProcessor/MessageProcessor.o
-
-obj-y += $(MessageProcessor-y)
-
-
-
diff --git a/code/plugin_NetworkTest/Makefile b/code/plugin_NetworkTest/Makefile
deleted file mode 100644
index 0ee8b66..0000000
--- a/code/plugin_NetworkTest/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-
-NetworkTest-y += plugin_NetworkTest/NetworkTest.o
-
-obj-y += $(NetworkTest-y)
-
-
-
diff --git a/code/plugin_Partitioner/Makefile b/code/plugin_Partitioner/Makefile
deleted file mode 100644
index 402ddbd..0000000
--- a/code/plugin_Partitioner/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-
-Partitioner-y += plugin_Partitioner/Partitioner.o
-
-obj-y += $(Partitioner-y)
-
-
diff --git a/code/plugin_PhylogenyViewer/Makefile b/code/plugin_PhylogenyViewer/Makefile
deleted file mode 100644
index 933c6a6..0000000
--- a/code/plugin_PhylogenyViewer/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-
-PhylogenyViewer-y += plugin_PhylogenyViewer/PhylogenyViewer.o
-PhylogenyViewer-y += plugin_PhylogenyViewer/GenomeToTaxonLoader.o
-PhylogenyViewer-y += plugin_PhylogenyViewer/PhylogeneticTreeLoader.o
-PhylogenyViewer-y += plugin_PhylogenyViewer/TaxonNameLoader.o
-
-obj-y += $(PhylogenyViewer-y)
-
diff --git a/code/plugin_Scaffolder/Makefile b/code/plugin_Scaffolder/Makefile
deleted file mode 100644
index c513198..0000000
--- a/code/plugin_Scaffolder/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-
-Scaffolder-y += plugin_Scaffolder/Scaffolder.o
-Scaffolder-y += plugin_Scaffolder/ScaffoldingLink.o
-Scaffolder-y += plugin_Scaffolder/SummarizedLink.o
-Scaffolder-y += plugin_Scaffolder/ScaffoldingAlgorithm.o
-Scaffolder-y += plugin_Scaffolder/ScaffoldingVertex.o
-Scaffolder-y += plugin_Scaffolder/ScaffoldingEdge.o
-
-
-obj-y += $(Scaffolder-y)
-
-
-
diff --git a/code/plugin_Searcher/Makefile b/code/plugin_Searcher/Makefile
deleted file mode 100644
index 80f99c7..0000000
--- a/code/plugin_Searcher/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-
-Searcher-y += plugin_Searcher/Searcher.o
-Searcher-y += plugin_Searcher/SearchDirectory.o
-Searcher-y += plugin_Searcher/ContigSearchEntry.o
-Searcher-y += plugin_Searcher/ContigHit.o
-Searcher-y += plugin_Searcher/ColorSet.o
-Searcher-y += plugin_Searcher/VirtualKmerColor.o
-Searcher-y += plugin_Searcher/QualityCaller.o
-Searcher-y += plugin_Searcher/DistributionWriter.o
-Searcher-y += plugin_Searcher/ColoredPeakFinder.o
-
-obj-y += $(Searcher-y)
-
diff --git a/code/plugin_SeedExtender/Makefile b/code/plugin_SeedExtender/Makefile
deleted file mode 100644
index e29eb7e..0000000
--- a/code/plugin_SeedExtender/Makefile
+++ /dev/null
@@ -1,14 +0,0 @@
-
-SeedExtender-y += plugin_SeedExtender/SeedExtender.o
-SeedExtender-y += plugin_SeedExtender/Direction.o
-SeedExtender-y += plugin_SeedExtender/VertexMessenger.o plugin_SeedExtender/ReadFetcher.o
-SeedExtender-y += plugin_SeedExtender/BubbleTool.o plugin_SeedExtender/Chooser.o plugin_SeedExtender/OpenAssemblerChooser.o
-SeedExtender-y += plugin_SeedExtender/TipWatchdog.o plugin_SeedExtender/NovaEngine.o
-SeedExtender-y += plugin_SeedExtender/ExtensionElement.o
-SeedExtender-y += plugin_SeedExtender/DepthFirstSearchData.o
-SeedExtender-y += plugin_SeedExtender/ExtensionData.o
-
-obj-y += $(SeedExtender-y)
-
-
-
diff --git a/code/plugin_SeedingData/AssemblySeed.cpp b/code/plugin_SeedingData/AssemblySeed.cpp
deleted file mode 100644
index 8b82b2c..0000000
--- a/code/plugin_SeedingData/AssemblySeed.cpp
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
-
- http://DeNovoAssembler.SourceForge.Net/
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, version 3 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
- see <http://www.gnu.org/licenses/>
-*/
-
-#include <plugin_SeedingData/AssemblySeed.h>
-#include <application_core/constants.h>
-#include <core/statistics.h>
-#include <iostream>
-#include <map>
-using namespace std;
-
-AssemblySeed::AssemblySeed(){
-}
-
-int AssemblySeed::size()const{
- return m_vertices.size();
-}
-
-Kmer*AssemblySeed::at(int i){
- return &(m_vertices.at(i));
-}
-
-CoverageDepth AssemblySeed::getCoverageAt(int position){
-
- if(m_coverageValues.size()==0)
- return 0;
-
- return m_coverageValues[position];
-}
-
-void AssemblySeed::push_back(Kmer*a){
- m_vertices.push_back(*a);
-}
-
-vector<Kmer>*AssemblySeed::getVertices(){
- return &m_vertices;
-}
-
-void AssemblySeed::clear(){
- m_vertices.clear();
-}
-
-void AssemblySeed::resetCoverageValues(){
- m_coverageValues.clear();
-}
-
-void AssemblySeed::computePeakCoverage(){
-
- bool useMode=false;
- bool useMean=true;
-
- #ifdef ASSERT
- assert(m_coverageValues.size() == m_vertices.size());
- assert((useMode || useMean) && !(useMode && useMean));
- #endif
-
- // the default is to use the weighted mean algorithm
-
- #ifdef ASSERT
- assert(useMean==true);
- assert(useMode==false);
- #endif
-
- if(useMode){
- computePeakCoverageUsingMode();
- }else if(useMean){
- computePeakCoverageUsingMean();
- }
-}
-
-CoverageDepth AssemblySeed::getPeakCoverage(){
- return m_peakCoverage;
-}
-
-void AssemblySeed::addCoverageValue(CoverageDepth value){
- m_coverageValues.push_back(value);
-}
-
-void AssemblySeed::computePeakCoverageUsingMode(){
-
- map<CoverageDepth,int> frequencies;
-
- for(int i=0;i<(int)m_coverageValues.size();i++){
- frequencies[m_coverageValues[i]]++;
- }
-
- int best=-1;
-
- for(map<CoverageDepth,int>::iterator i=frequencies.begin();
- i!=frequencies.end();i++){
-
- if(frequencies.count(best)==0 || i->second > frequencies[best]){
- best=i->first;
- }
-
- #ifdef CONFIG_VERBOSITY_FOR_SEEDS
- cout<<i->first<<" "<<i->second<<endl;
- #endif
- }
-
- cout<<"mode= "<<best<<" length= "<<m_vertices.size()<<endl;
-
- m_peakCoverage=best;
-
-}
-
-void AssemblySeed::computePeakCoverageUsingMean(){
-
- map<CoverageDepth,int> frequencies;
-
- for(int i=0;i<(int)m_coverageValues.size();i++){
- frequencies[m_coverageValues[i]]++;
- }
-
- LargeCount sum=0;
- LargeCount count=0;
-
- for(map<CoverageDepth,int>::iterator i=frequencies.begin();
- i!=frequencies.end();i++){
-
- CoverageDepth coverage=i->first;
- LargeCount frequency=i->second;
-
- #ifdef CONFIG_VERBOSITY_FOR_SEEDS
- cout<<coverage<<" "<<frequency<<endl;
- #endif
-
- sum+=coverage*frequency;
- count+=frequency;
- }
-
- #ifdef ASSERT
- assert(m_coverageValues.size()>=1);
- assert(count!=0);
- assert(count>0);
- assert(sum > 0);
- #endif
-
- CoverageDepth mean=( sum / count );
-
- cout<<"mean= "<<mean <<" length= "<<m_vertices.size()<<endl;
-
- m_peakCoverage=mean;
-}
diff --git a/code/plugin_SeedingData/AssemblySeed.h b/code/plugin_SeedingData/AssemblySeed.h
deleted file mode 100644
index 06f44eb..0000000
--- a/code/plugin_SeedingData/AssemblySeed.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- Ray
- Copyright (C) 2010, 2011, 2012 Sébastien Boisvert
-
- http://DeNovoAssembler.SourceForge.Net/
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, version 3 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
- see <http://www.gnu.org/licenses/>
-*/
-
-#ifndef _AssemblySeed_h
-#define _AssemblySeed_h
-
-#include <vector>
-#include <plugin_KmerAcademyBuilder/Kmer.h>
-using namespace std;
-
-/**
- * This class describes objects representing assembly seeds.
- * An assembly seed is a path in the de Bruijn graph.
- * It is likely unique in the target genome (hopefully !)
- * \author pro grammer: Sébastien Boisvert (good with grammar)
- * \author co-designer: The Ray committee of wise people (E. God.)
- * \date updated 2012-06-21 for the release of v2.0.0
- */
-class AssemblySeed{
- vector<Kmer> m_vertices;
- vector<CoverageDepth> m_coverageValues;
-
- CoverageDepth m_peakCoverage;
-
-/** computes locality with a weighted mean (locality object is peak coverage) **/
- void computePeakCoverageUsingMean();
-
-/** computes locality with a mode (locality object is peak coverage) **/
- void computePeakCoverageUsingMode();
-
-public:
- AssemblySeed();
-
- int size()const;
- Kmer*at(int i);
- CoverageDepth getCoverageAt(int i);
-
- void push_back(Kmer*a);
- vector<Kmer>*getVertices();
- void clear();
-
- void addCoverageValue(CoverageDepth value);
- CoverageDepth getPeakCoverage();
- void resetCoverageValues();
-
- void computePeakCoverage();
-};
-
-#endif
diff --git a/code/plugin_SeedingData/Makefile b/code/plugin_SeedingData/Makefile
deleted file mode 100644
index 979395f..0000000
--- a/code/plugin_SeedingData/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-
-SeedingData-y += plugin_SeedingData/AssemblySeed.o plugin_SeedingData/SeedWorker.o
-SeedingData-y += plugin_SeedingData/SeedingData.o
-
-obj-y += $(SeedingData-y)
-
-
-
diff --git a/code/plugin_SequencesIndexer/Makefile b/code/plugin_SequencesIndexer/Makefile
deleted file mode 100644
index f9761a3..0000000
--- a/code/plugin_SequencesIndexer/Makefile
+++ /dev/null
@@ -1,9 +0,0 @@
-
-SequencesIndexer-y += plugin_SequencesIndexer/SequencesIndexer.o
-SequencesIndexer-y += plugin_SequencesIndexer/IndexerWorker.o
-SequencesIndexer-y += plugin_SequencesIndexer/PairedRead.o plugin_SequencesIndexer/ReadAnnotation.o
-
-obj-y += $(SequencesIndexer-y)
-
-
-
diff --git a/code/plugin_SequencesLoader/FastqGzLoader.cpp b/code/plugin_SequencesLoader/FastqGzLoader.cpp
deleted file mode 100644
index b9ceb48..0000000
--- a/code/plugin_SequencesLoader/FastqGzLoader.cpp
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
-
- http://DeNovoAssembler.SourceForge.Net/
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, version 3 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
- see <http://www.gnu.org/licenses/>
-
-*/
-
-#ifdef HAVE_LIBZ
-
-#include<plugin_SequencesLoader/FastqGzLoader.h>
-#include<fstream>
-#include<zlib.h>
-#include <stdlib.h>
-
-int FastqGzLoader::open(string file,int period){
- m_f=gzopen(file.c_str(),"r");
- char buffer[4096];
- m_size=0;
- m_loaded=0;
-
- int rotatingVariable=0;
- while(Z_NULL!=gzgets(m_f,buffer,4096)){
- if(rotatingVariable==1){
- m_size++;
- }
- rotatingVariable++;
- if(rotatingVariable==period){
- rotatingVariable=0;
- }
- }
- gzclose(m_f);
- m_f=gzopen(file.c_str(),"r");
- return EXIT_SUCCESS;
-}
-
-// a very simple and compact fastq.gz reader
-void FastqGzLoader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator,int period){
- char buffer[4096];
- int rotatingVariable=0;
- int loadedSequences=0;
-
- while(loadedSequences<maxToLoad && Z_NULL!=gzgets(m_f,buffer,4096)){
- if(rotatingVariable==1){
- Read t;
- t.constructor(buffer,seqMyAllocator,true);
- reads->push_back(&t);
- }
- rotatingVariable++;
-
- // a period is reached for each read.
- if(rotatingVariable==period){
- rotatingVariable=0;
- loadedSequences++;
- m_loaded++;
- }
- }
- if(m_loaded==m_size){
- gzclose(m_f);
- }
-}
-
-int FastqGzLoader::getSize(){
- return m_size;
-}
-
-#endif
-
diff --git a/code/plugin_SequencesLoader/FastqLoader.cpp b/code/plugin_SequencesLoader/FastqLoader.cpp
deleted file mode 100644
index 17a5fc2..0000000
--- a/code/plugin_SequencesLoader/FastqLoader.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
-
- http://DeNovoAssembler.SourceForge.Net/
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, version 3 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
- see <http://www.gnu.org/licenses/>
-
-*/
-
-#include<plugin_SequencesLoader/FastqLoader.h>
-#include <stdlib.h>
-#include <application_core/constants.h>
-#include<fstream>
-
-int FastqLoader::open(string file,int period){
- m_f=fopen(file.c_str(),"r");
- m_size=0;
- m_loaded=0;
- int rotatingVariable=0;
- char buffer[RAY_MAXIMUM_READ_LENGTH];
- while(NULL!=fgets(buffer,RAY_MAXIMUM_READ_LENGTH,m_f)){
- if(rotatingVariable==1){
- m_size++;
- }
- rotatingVariable++;
- if(rotatingVariable==period){
- rotatingVariable=0;
- }
- }
-
- fclose(m_f);
- m_f=fopen(file.c_str(),"r");
- return EXIT_SUCCESS;
-}
-
-void FastqLoader::load(int maxToLoad,ArrayOfReads*reads,MyAllocator*seqMyAllocator,int period){
- char buffer[RAY_MAXIMUM_READ_LENGTH];
- int rotatingVariable=0;
- int loadedSequences=0;
-
- while(loadedSequences<maxToLoad && NULL!=fgets(buffer,RAY_MAXIMUM_READ_LENGTH,m_f)){
- if(rotatingVariable==1){
- Read t;
- t.constructor(buffer,seqMyAllocator,true);
- reads->push_back(&t);
- }
- rotatingVariable++;
-
- // a period is reached for each read.
- if(rotatingVariable==period){
- rotatingVariable=0;
- loadedSequences++;
- m_loaded++;
- }
- }
-
- if(m_loaded==m_size){
- fclose(m_f);
- }
-}
-
-int FastqLoader::getSize(){
- return m_size;
-}
diff --git a/code/plugin_SequencesLoader/Loader.cpp b/code/plugin_SequencesLoader/Loader.cpp
deleted file mode 100644
index 48b352f..0000000
--- a/code/plugin_SequencesLoader/Loader.cpp
+++ /dev/null
@@ -1,207 +0,0 @@
-
-/*
- Ray
- Copyright (C) 2010, 2011 Sébastien Boisvert
-
- http://DeNovoAssembler.SourceForge.Net/
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, version 3 of the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You have received a copy of the GNU General Public License
- along with this program (gpl-3.0.txt).
- see <http://www.gnu.org/licenses/>
-
-*/
-
-#include <sstream>
-#include <iostream>
-#include <assert.h>
-#include <string>
-#include <vector>
-#include <plugin_SequencesLoader/Loader.h>
-#include <stdlib.h>
-#include <plugin_SequencesLoader/Read.h>
-using namespace std;
-
-void Loader::constructor(const char*prefix,bool show){
- m_maxToLoad=500000;
- m_currentOffset=0;
- m_type=FORMAT_NULL;
- ostringstream prefixFull;
- prefixFull<<prefix<<"_Loader";
- m_show=show;
- m_allocator.constructor(4194304,"RAY_MALLOC_TYPE_LOADER_ALLOCATOR",m_show);
- m_reads.constructor(&m_allocator);
-}
-
-int Loader::load(string file,bool isGenome){
- ifstream f(file.c_str());
- bool exists=f;
- f.close();
- if(!exists){
- cout<<"Ray: cannot access '"<<file<<"': No such file or directory"<<endl;
- return EXIT_FAILURE;// ERROR
- }
-
- if(file.length()<4){
- (cout)<<"Error: "<<file<<endl;
- return EXIT_FAILURE;
- }
-
- if(isGenome){
- m_fasta.load(file,&m_reads,&m_allocator);
- m_size=m_reads.size();
- return EXIT_SUCCESS;
- }
-
- cout<<"[Loader::load] File: "<<file<<" (please wait...)"<<endl;
-
- string csfastaExtension=".csfasta";
- if(file.length()>=csfastaExtension.length() &&
- file.substr(file.length()-csfastaExtension.length(),csfastaExtension.length())==csfastaExtension){
- m_type=FORMAT_CSFASTA;
- int ret=m_color.open(file);
- m_size=m_color.getSize();
- return ret;
- }
- if(file.substr(file.length()-4,4)==".sff"){
- m_type=FORMAT_SFF;
- int ret=m_sff.open(file);
- m_size=m_sff.getSize();
- return ret;
-
- }
- if(file.substr(file.length()-6,6)==".fasta"){
- m_type=FORMAT_FASTA;
- int ret=m_fastq.open(file,2);
- m_size=m_fastq.getSize();
- return ret;
- }
-
- if(file.substr(file.length()-6,6)==".fastq"){
- m_type=FORMAT_FASTQ;
- int ret=m_fastq.open(file,4);
- m_size=m_fastq.getSize();
- return ret;
- }
-
- #ifdef HAVE_LIBZ
- if(file.substr(file.length()-9,9)==".fastq.gz"){
- m_type=FORMAT_FASTQ_GZ;
- int ret=m_fastqgz.open(file,4);
- m_size=m_fastqgz.getSize();
- return ret;
- }
-
- if(file.substr(file.length()-9,9)==".fasta.gz"){
- m_type=FORMAT_FASTA_GZ;
- int ret=m_fastqgz.open(file,2);
- m_size=m_fastqgz.getSize();
- return ret;
- }
- #endif
-
- #ifdef HAVE_LIBBZ2
- if(file.substr(file.length()-10,10)==".fastq.bz2"){
- m_type=FORMAT_FASTQ_BZ2;
- int ret=m_fastqbz2.open(file,4);
- m_size=m_fastqbz2.getSize();
- return ret;
- }
-
- if(file.substr(file.length()-10,10)==".fasta.bz2"){
- m_type=FORMAT_FASTA_BZ2;
- int ret=m_fastqbz2.open(file,2);
- m_size=m_fastqbz2.getSize();
- return ret;
- }
- #endif
-
- cout<<"Error: "<<file<<": unknown extension, exiting. (see Ray --help for valid extensions)"<<endl;
-
- return EXIT_FAILURE;
-}
-
-Read*Loader::at(LargeIndex i){
- #ifdef ASSERT
- assert(i<m_size);
- #endif
-
- if(i>=m_currentOffset+m_reads.size()){
- loadSequences();
- }
- #ifdef ASSERT
- if(i>=m_currentOffset+m_reads.size()){
- cout<<"i="<<i<<" offset="<<m_currentOffset<<" Loaded="<<m_reads.size()<<endl;
- }
- assert(i<m_currentOffset+m_reads.size());
- #endif
- return m_reads.at(i-m_currentOffset);
-}
-
-LargeCount Loader::size(){
- return m_size;
-}
-
-void Loader::clear(){
- m_reads.clear();
- m_allocator.clear();
- m_size=0;
- m_currentOffset=0;
- m_type=FORMAT_NULL;
-
- #ifdef ASSERT
- assert(m_reads.size()==0);
- #endif
-}
-
-void Loader::loadSequences(){
- m_currentOffset+=m_reads.size();
- m_allocator.reset();
- m_reads.reset();
-
- if(m_type==FORMAT_FASTQ_GZ){
- #ifdef HAVE_LIBZ
- m_fastqgz.load(m_maxToLoad,&m_reads,&m_allocator,4);
- #endif
- }else if(m_type==FORMAT_FASTQ){
- m_fastq.load(m_maxToLoad,&m_reads,&m_allocator,4);
- }else if(m_type==FORMAT_FASTQ_BZ2){
- #ifdef HAVE_LIBBZ2
- m_fastqbz2.load(m_maxToLoad,&m_reads,&m_allocator,4);
- #endif
- }else if(m_type==FORMAT_CSFASTA){
- m_color.load(m_maxToLoad,&m_reads,&m_allocator);
- }else if(m_type==FORMAT_SFF){
- m_sff.load(m_maxToLoad,&m_reads,&m_allocator);
- }else if(m_type==FORMAT_FASTA){
- m_fastq.load(m_maxToLoad,&m_reads,&m_allocator,2);
- }else if(m_type==FORMAT_FASTA_BZ2){
- #ifdef HAVE_LIBBZ2
- m_fastqbz2.load(m_maxToLoad,&m_reads,&m_allocator,2);
- #endif
- }else if(m_type==FORMAT_FASTA_GZ){
- #ifdef HAVE_LIBZ
- m_fastqgz.load(m_maxToLoad,&m_reads,&m_allocator,2);
- #endif
- }
-}
-
-void Loader::reset(){
- m_allocator.reset();
- m_reads.reset();
- m_size=0;
- m_currentOffset=0;
- m_type=FORMAT_NULL;
-
- #ifdef ASSERT
- assert(m_reads.size()==0);
- #endif
-}
diff --git a/code/plugin_SequencesLoader/Makefile b/code/plugin_SequencesLoader/Makefile
deleted file mode 100644
index 7124e57..0000000
--- a/code/plugin_SequencesLoader/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-
-SequencesLoader-y += plugin_SequencesLoader/SequencesLoader.o
-SequencesLoader-y += plugin_SequencesLoader/Read.o plugin_SequencesLoader/ArrayOfReads.o
-SequencesLoader-y += plugin_SequencesLoader/ColorSpaceDecoder.o plugin_SequencesLoader/ColorSpaceLoader.o
-SequencesLoader-y += plugin_SequencesLoader/FastaLoader.o plugin_SequencesLoader/FastqLoader.o plugin_SequencesLoader/SffLoader.o plugin_SequencesLoader/Loader.o
-SequencesLoader-$(HAVE_LIBBZ2) += plugin_SequencesLoader/BzReader.o
-SequencesLoader-$(HAVE_LIBBZ2) += plugin_SequencesLoader/FastqBz2Loader.o
-SequencesLoader-$(HAVE_LIBZ) += plugin_SequencesLoader/FastqGzLoader.o
-
-obj-y += $(SequencesLoader-y)
-
-
-
diff --git a/code/plugin_VerticesExtractor/Makefile b/code/plugin_VerticesExtractor/Makefile
deleted file mode 100644
index 2d7d827..0000000
--- a/code/plugin_VerticesExtractor/Makefile
+++ /dev/null
@@ -1,9 +0,0 @@
-
-VerticesExtractor-y += plugin_VerticesExtractor/VerticesExtractor.o
-VerticesExtractor-y += plugin_VerticesExtractor/GridTable.o plugin_VerticesExtractor/GridTableIterator.o
-VerticesExtractor-y += plugin_VerticesExtractor/Vertex.o
-
-obj-y += $(VerticesExtractor-y)
-
-
-
diff --git a/git.txt b/git.txt
deleted file mode 100644
index edf226f..0000000
--- a/git.txt
+++ /dev/null
@@ -1 +0,0 @@
-git://github.com/sebhtml/ray.git
diff --git a/github.txt b/github.txt
deleted file mode 100644
index 843b1d3..0000000
--- a/github.txt
+++ /dev/null
@@ -1 +0,0 @@
-https://github.com/sebhtml/ray
diff --git a/scripts/Build-Link-Time-Optimization.sh b/scripts/Build-Link-Time-Optimization.sh
index 35e494d..5ae071e 100755
--- a/scripts/Build-Link-Time-Optimization.sh
+++ b/scripts/Build-Link-Time-Optimization.sh
@@ -2,12 +2,12 @@
mpicxx \
--Wall -std=c++98 -O3 -march=native -flto -fwhole-program \
- -D HAVE_LIBZ -D HAVE_LIBBZ2 -lz -lbz2 \
+-Wall -std=c++98 -Os -march=native -flto -fwhole-program \
+ -D CONFIG_HAVE_LIBZ -D CONFIG_HAVE_LIBBZ2 -lz -lbz2 \
-flto-report \
- -D MAXKMERLENGTH=32 \
- -D RAY_VERSION=\"2.1.0-devel\" -D RAYPLATFORM_VERSION=\"1.1.0-devel\" \
- -I. -Icode -IRayPlatform \
+ -D CONFIG_MAXKMERLENGTH=32 \
+ -D CONFIG_RAY_VERSION=\"x.y.z\" -D CONFIG_RAYPLATFORM_VERSION=\"x.y.z\" \
+ -I. -I RayPlatform \
$(find code/|grep .cpp$;find RayPlatform/|grep .cpp$) \
-o Ray
diff --git a/scripts/ShipProduct.sh b/scripts/ShipProduct.sh
index dccfc60..88a51da 100755
--- a/scripts/ShipProduct.sh
+++ b/scripts/ShipProduct.sh
@@ -6,21 +6,7 @@ echo "Welcome to Ray Technologies."
echo "Today we ship a product."
echo ""
-# you need
-
-# - git
-# - mkdir (coreutils)
-# - bash
-# - xz
-# - tar
-# - gzip
-# - bzip2
-# - zip
-# - md5sum
-# - sha1sum
-
release=$1
-
base=~/git-clones
RayGit=$base/ray
@@ -39,38 +25,22 @@ echo "Assembling product parts"
mkdir -p $productDirectory
cd $productDirectory
-echo "Cloning $RayGit"
git clone $RayGit $productName &> /dev/null
cd $productName
-git log|grep ^commit|head -n1|awk '{print $2}' > tag.txt
-echo https://github.com/sebhtml/ray > github.txt
-echo git://github.com/sebhtml/ray.git > git.txt
rm -rf .git
rm -rf RayPlatform
-echo "Cloning $RayPlatformGit"
git clone $RayPlatformGit &> /dev/null
cd RayPlatform
-git log|grep ^commit|head -n1|awk '{print $2}' > tag.txt
-echo https://github.com/sebhtml/RayPlatform > github.txt
-echo git://github.com/sebhtml/RayPlatform.git > git.txt
rm -rf .git
cd ..
-
-
-
-# create archives
+cd ..
echo "Creating product releases."
-
-cd ..
-echo ".tar.bz2"
-tar cjf $productName.tar.bz2 $productName &> /dev/null
+tar -cjf $productName.tar.bz2 $productName &> /dev/null
rm -rf $productName
-# compute checksums
-
for i in $(ls Ray-$release.*)
do
sha1sum $i > $i.sha1sum.txt
diff --git a/scripts/install.sh b/scripts/install.sh
deleted file mode 100755
index 5ef0027..0000000
--- a/scripts/install.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-
-TARGETS=$(cat TARGETS)
-PREFIX=$(cat PREFIX)
-
-mkdir -p $PREFIX
-
-echo ""
-echo "Installing Ray to $PREFIX"
-echo ""
-
-# legal stuff
-
-cp LICENSE.txt $PREFIX
-cp gpl-3.0.txt $PREFIX
-cp RayPlatform/lgpl-3.0.txt $PREFIX
-
-
-
-cp $TARGETS $PREFIX
-
-cp -r Documentation $PREFIX
-cp README.md $PREFIX
-cp MANUAL_PAGE.txt $PREFIX
-cp INSTALL.txt $PREFIX
-cp AUTHORS $PREFIX
-
-#cp code/Application/libRayApplication.a $PREFIX
-#cp code/Platform/libRayPlatform.a $PREFIX
-
-cp -r scripts/xsl-xml $PREFIX
-
-ls $PREFIX
diff --git a/tag.txt b/tag.txt
deleted file mode 100644
index bd3efae..0000000
--- a/tag.txt
+++ /dev/null
@@ -1 +0,0 @@
-c8953003b7b3bdfda399fc49c11c0e15889332cd
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/ray.git
More information about the debian-med-commit
mailing list