[med-svn] [Git][med-team/htsjdk][master] 5 commits: New upstream version 4.1.0+dfsg
Pierre Gruet (@pgt)
gitlab at salsa.debian.org
Sun Feb 11 22:07:24 GMT 2024
Pierre Gruet pushed to branch master at Debian Med / htsjdk
Commits:
97ded439 by Pierre Gruet at 2024-02-11T20:57:54+01:00
New upstream version 4.1.0+dfsg
- - - - -
b65c8791 by Pierre Gruet at 2024-02-11T20:58:18+01:00
Update upstream source from tag 'upstream/4.1.0+dfsg'
Update to upstream version '4.1.0+dfsg'
with Debian dir 6e32e134aa41d60b2d1042deae0aa5cd963885d7
- - - - -
805e6290 by Pierre Gruet at 2024-02-11T21:02:40+01:00
Refreshing patches
- - - - -
097d9f71 by Pierre Gruet at 2024-02-11T21:03:10+01:00
Updating changelog
- - - - -
80d13b32 by Pierre Gruet at 2024-02-11T22:41:49+01:00
Upload to unstable
- - - - -
13 changed files:
- debian/changelog
- debian/patches/13-skip_network_tests
- src/main/java/htsjdk/io/HtsPath.java
- src/main/java/htsjdk/samtools/SAMRecordSetBuilder.java
- src/main/java/htsjdk/samtools/seekablestream/SeekableStreamFactory.java
- src/main/java/htsjdk/tribble/FeatureCodec.java
- src/main/java/htsjdk/tribble/TribbleIndexedFeatureReader.java
- src/main/java/htsjdk/tribble/util/FTPHelper.java
- src/main/java/htsjdk/tribble/util/ParsingUtils.java
- src/test/java/htsjdk/io/HtsPathUnitTest.java
- src/test/java/htsjdk/samtools/seekablestream/SeekableStreamFactoryTest.java
- src/test/java/htsjdk/tribble/AbstractFeatureReaderTest.java
- src/test/java/htsjdk/tribble/util/ParsingUtilsTest.java
Changes:
=====================================
debian/changelog
=====================================
@@ -1,3 +1,10 @@
+htsjdk (4.1.0+dfsg-1) unstable; urgency=medium
+
+ * New upstream version 4.1.0+dfsg
+ * Refreshing patches
+
+ -- Pierre Gruet <pgt at debian.org> Sun, 11 Feb 2024 22:41:39 +0100
+
htsjdk (4.0.2+dfsg-2) unstable; urgency=medium
* Excluding one more test needing the network
=====================================
debian/patches/13-skip_network_tests
=====================================
@@ -1194,73 +1194,74 @@ Forwarded: not-needed
}
--- a/src/test/java/htsjdk/tribble/util/ParsingUtilsTest.java
+++ b/src/test/java/htsjdk/tribble/util/ParsingUtilsTest.java
-@@ -153,67 +153,12 @@
- tstExists(file.toUri().toString(), false);
+@@ -156,68 +156,7 @@
+ }
}
- @Test(groups = "ftp")
- public void testFTPDoesExist() throws IOException{
-- tstExists(AVAILABLE_FTP_URL, true);
+- testExists(AVAILABLE_FTP_URL, true);
- }
-
- @Test(groups = "ftp")
- public void testFTPNotExist() throws IOException{
-- tstExists(UNAVAILABLE_FTP_URL, false);
+- testExists(UNAVAILABLE_FTP_URL, false);
- }
-
- @Test
- public void testHTTPDoesExist() throws IOException{
-- tstExists(AVAILABLE_HTTP_URL, true);
+- testExists(AVAILABLE_HTTP_URL, true);
- }
-
- @Test
- public void testHTTPNotExist() throws IOException{
-- tstExists(UNAVAILABLE_HTTP_URL, false);
+- testExists(UNAVAILABLE_HTTP_URL, false);
- }
-
- private void tstExists(String path, boolean expectExists) throws IOException{
- boolean exists = ParsingUtils.resourceExists(path);
- Assert.assertEquals(exists, expectExists);
+-
+-
+ private static void testExists(String path, boolean expectExists) throws IOException{
+ Assert.assertEquals(ParsingUtils.resourceExists(path), expectExists);
}
-
+-
- @Test
- public void testFileOpenInputStream() throws IOException{
- File tempFile = File.createTempFile(getClass().getSimpleName(), ".tmp");
- tempFile.deleteOnExit();
-- OutputStream os = IOUtil.openFileForWriting(tempFile);
-- BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os));
-- writer.write("hello");
-- writer.close();
-- tstStream(tempFile.getAbsolutePath());
-- tstStream(tempFile.toURI().toString());
+- try(Writer writer = new BufferedWriter(new OutputStreamWriter(IOUtil.openFileForWriting(tempFile)))) {
+- writer.write("hello");
+- }
+- testStream(tempFile.getAbsolutePath());
+- testStream(tempFile.toURI().toString());
- }
-
- @Test
- public void testInMemoryNioFileOpenInputStream() throws IOException{
-- FileSystem fs = Jimfs.newFileSystem(Configuration.unix());
-- Path file = fs.getPath("/file");
-- Files.write(file, "hello".getBytes("UTF-8"));
-- tstStream(file.toUri().toString());
+- try(FileSystem fs = Jimfs.newFileSystem(Configuration.unix())) {
+- Path file = fs.getPath("/file");
+- Files.write(file, "hello".getBytes(StandardCharsets.UTF_8));
+- testStream(file.toUri().toString());
+- }
- }
-
- @Test(groups = "ftp")
- public void testFTPOpenInputStream() throws IOException{
-- tstStream(AVAILABLE_FTP_URL);
+- testStream(AVAILABLE_FTP_URL);
- }
-
- @Test
- public void testHTTPOpenInputStream() throws IOException{
-- tstStream(AVAILABLE_HTTP_URL);
+- testStream(AVAILABLE_HTTP_URL);
- }
-
-- private void tstStream(String path) throws IOException{
-- InputStream is = ParsingUtils.openInputStream(path);
-- Assert.assertNotNull(is, "InputStream is null for " + path);
-- int b = is.read();
-- Assert.assertNotSame(b, -1);
+- private static void testStream(String path) throws IOException{
+- try(InputStream is = ParsingUtils.openInputStream(path)) {
+- Assert.assertNotNull(is, "InputStream is null for " + path);
+- int b = is.read();
+- Assert.assertNotSame(b, -1);
+- }
- }
-
-
+-
+-
}
--- a/src/test/java/htsjdk/tribble/AbstractFeatureReaderTest.java
+++ b/src/test/java/htsjdk/tribble/AbstractFeatureReaderTest.java
@@ -1287,7 +1288,7 @@ Forwarded: not-needed
-
- @Test(groups = "ftp")
- public void testLoadBEDFTP() throws Exception {
-- final String path = "ftp://ftp.broadinstitute.org/distribution/igv/TEST/cpgIslands with spaces.hg18.bed";
+- final String path = "ftp://ftp.broadinstitute.org/distribution/igv/TEST/cpgIslands%20with%20spaces.hg18.bed";
- final BEDCodec codec = new BEDCodec();
- final AbstractFeatureReader<BEDFeature, LineIterator> bfs = AbstractFeatureReader.getFeatureReader(path, codec, false);
- for (final Feature feat : bfs.iterator()) {
=====================================
src/main/java/htsjdk/io/HtsPath.java
=====================================
@@ -27,7 +27,7 @@ import java.nio.file.spi.FileSystemProvider;
* encoding/escaping.
*
* For example, a URI that contains a scheme and has an embedded "#" in the path will be treated as a URI
- * having a fragment delimiter. If the URI contains an scheme, the "#" will be escaped and the encoded "#"
+ * having a fragment delimiter. If the URI contains no scheme, the "#" will be escaped and the encoded "#"
* will be interpreted as part of the URI path.
*
* There are 3 succeeding levels of input validation/conversion:
@@ -69,6 +69,7 @@ import java.nio.file.spi.FileSystemProvider;
*/
public class HtsPath implements IOPath, Serializable {
private static final long serialVersionUID = 1L;
+ private static final String HIERARCHICAL_SCHEME_SEPARATOR = "://";
private final String rawInputString; // raw input string provided by th user; may or may not have a scheme
private final URI uri; // working URI; always has a scheme ("file" if not otherwise specified)
@@ -253,6 +254,10 @@ public class HtsPath implements IOPath, Serializable {
tempURI = getCachedPath().toUri();
}
} catch (URISyntaxException uriException) {
+ //check that the uri wasn't a badly encoded absolute uri of some sort
+ //if you don't do this it will be treated as a badly formed file:// url
+ assertNoProblematicScheme(pathString, uriException);
+
// the input string isn't a valid URI; assume its a local (non-URI) file reference, and
// use the URI resulting from the corresponding Path
try {
@@ -276,5 +281,42 @@ public class HtsPath implements IOPath, Serializable {
return tempURI;
}
+
+ /**
+ * Check for problems associated with the presence of a hierarchical scheme.
+ *
+ * It's better to reject cases like `://` or `ftp://I forgot to encode this` than to treat them as relative file paths
+ * It's almost certainly an error on the users part instead of an atttempt to intentionally reference a file named
+ * `file:///workingidr/ftp:/I forgot to encode this`
+ *
+ * Note this is only meant to be called in the case of a URLSyntaxException already having occured during initial
+ * parsing of the URI
+ *
+ * @param pathString the path being examined
+ * @param cause the original failure reason
+ */
+ static void assertNoProblematicScheme(String pathString, URISyntaxException cause){
+ if(pathString.equals(HIERARCHICAL_SCHEME_SEPARATOR)){
+ throw new IllegalArgumentException(HIERARCHICAL_SCHEME_SEPARATOR + " is not a valid path.", cause);
+ }
+
+ final String[] split = pathString.split(HIERARCHICAL_SCHEME_SEPARATOR, -1);
+ final String scheme = split[0];
+
+ if(split.length == 2 && pathString.endsWith(HIERARCHICAL_SCHEME_SEPARATOR)) {
+ throw new IllegalArgumentException("A path consisting of only a scheme is not allowed: " + pathString, cause);
+ }
+
+ if(split.length > 1){
+ if(scheme == null || scheme.isEmpty()){
+ throw new IllegalArgumentException("Malformed path " + pathString + " includes an empty scheme.", cause);
+ }
+ if(!scheme.equals("file")){
+ throw new IllegalArgumentException("Malformed path " + pathString + " includes a scheme: " + scheme + ":// but was an invalid URI." +
+ "\nCheck that it is fully encoded.", cause);
+ }
+ }
+
+ }
}
=====================================
src/main/java/htsjdk/samtools/SAMRecordSetBuilder.java
=====================================
@@ -54,6 +54,7 @@ public class SAMRecordSetBuilder implements Iterable<SAMRecord> {
"chr21", "chr22", "chrX", "chrY", "chrM"
};
+
private static final String READ_GROUP_ID = "1";
private static final String SAMPLE = "FREE_SAMPLE";
private final Random random = new Random(TestUtil.RANDOM_SEED);
=====================================
src/main/java/htsjdk/samtools/seekablestream/SeekableStreamFactory.java
=====================================
@@ -23,12 +23,15 @@
*/
package htsjdk.samtools.seekablestream;
-import htsjdk.samtools.util.IOUtil;
-import java.io.File;
+import htsjdk.io.HtsPath;
+import htsjdk.io.IOPath;
+import htsjdk.tribble.TribbleException;
+
import java.io.IOException;
-import java.net.URI;
import java.net.URL;
import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Path;
+import java.util.Set;
import java.util.function.Function;
/**
@@ -40,6 +43,14 @@ import java.util.function.Function;
public class SeekableStreamFactory{
private static final ISeekableStreamFactory DEFAULT_FACTORY;
+ private static final String HTTP = "http";
+ private static final String HTTPS = "https";
+ private static final String FTP = "ftp";
+ /**
+ * the set of url schemes that have special support in htsjdk that isn't through a FileSystemProvider
+ */
+ private static final Set<String> URL_SCHEMES_WITH_LEGACY_SUPPORT = Set.of(HTTP, FTP, HTTPS);
+ public static final String FILE_SCHEME = "file";
private static ISeekableStreamFactory currentFactory;
static{
@@ -61,9 +72,28 @@ public class SeekableStreamFactory{
* Does this path point to a regular file on disk and not something like a URL?
* @param path the path to test
* @return true if the path is to a file on disk
+ * @deprecated this method is simplistic and no longer particularly useful since IOPath allows similar access to
+ * various non-file data sources, internal use has been replaced with {@link #isBeingHandledByLegacyUrlSupport(String)}
*/
+ @Deprecated
public static boolean isFilePath(final String path) {
- return ! ( path.startsWith("http:") || path.startsWith("https:") || path.startsWith("ftp:") );
+ return !canBeHandledByLegacyUrlSupport(path);
+ }
+
+ /**
+ * is this path being handled by one of the legacy SeekableStream types (http(s) / ftp)
+ *
+ * @param path a path to check
+ * @return if the path is not being handled by a FileSystemProvider and it can be read by legacy streams
+ */
+ public static boolean isBeingHandledByLegacyUrlSupport(final String path){
+ return !new HtsPath(path).hasFileSystemProvider() //if we have a provider for it that's what we'll use
+ && canBeHandledByLegacyUrlSupport(path); // otherwise we fall back to the special handlers
+ }
+
+ //is this one of the url types that has legacy htsjdk support built in?
+ public static boolean canBeHandledByLegacyUrlSupport(final String path) {
+ return URL_SCHEMES_WITH_LEGACY_SUPPORT.stream().anyMatch(scheme-> path.startsWith(scheme +"://"));
}
private static class DefaultSeekableStreamFactory implements ISeekableStreamFactory {
@@ -79,7 +109,7 @@ public class SeekableStreamFactory{
}
/**
- * The wrapper will only be applied to the stream if the stream is treated as a {@link java.nio.file.Path}
+ * The wrapper will only be applied to the stream if the stream is treated as a {@link Path}
*
* This currently means any uri with a scheme that is not http, https, ftp, or file will have the wrapper applied to it
*
@@ -89,26 +119,30 @@ public class SeekableStreamFactory{
@Override
public SeekableStream getStreamFor(final String path,
Function<SeekableByteChannel, SeekableByteChannel> wrapper) throws IOException {
- // todo -- add support for SeekableBlockInputStream
-
- if (path.startsWith("http:") || path.startsWith("https:")) {
- final URL url = new URL(path);
- return new SeekableHTTPStream(url);
- } else if (path.startsWith("ftp:")) {
- return new SeekableFTPStream(new URL(path));
- } else if (path.startsWith("file:")) {
- try {
- // convert to URI in order to obtain a decoded version of the path string suitable
- // for use with the File constructor
- final String decodedPath = new URI(path).getPath();
- return new SeekableFileStream(new File(decodedPath));
- } catch (java.net.URISyntaxException e) {
- throw new IllegalArgumentException(String.format("The input string %s contains a URI scheme but is not a valid URI", path), e);
- }
- } else if (IOUtil.hasScheme(path)) {
- return new SeekablePathStream(IOUtil.getPath(path), wrapper);
+ return getStreamFor(new HtsPath(path), wrapper);
+ }
+
+
+ /**
+ * The wrapper will only be applied to the stream if the stream is treated as a non file:// {@link Path}
+ *
+ * This has a fall back to htsjdk's built in http and ftp providers if no FileSystemProvder is available for them
+ *
+ * @param path an IOPath to be opened
+ * @param wrapper a wrapper to apply to the stream allowing direct transformations on the byte stream to be applied
+ * @throws IOException
+ */
+ public static SeekableStream getStreamFor(final IOPath path, Function<SeekableByteChannel, SeekableByteChannel> wrapper) throws IOException {
+ if(path.hasFileSystemProvider()) {
+ return path.getScheme().equals(FILE_SCHEME)
+ ? new SeekableFileStream(path.toPath().toFile()) //don't apply the wrapper to local files
+ : new SeekablePathStream(path.toPath(), wrapper);
} else {
- return new SeekableFileStream(new File(path));
+ return switch(path.getScheme()){
+ case HTTP, HTTPS -> new SeekableHTTPStream(new URL(path.getRawInputString()));
+ case FTP -> new SeekableFTPStream((new URL(path.getRawInputString())));
+ default -> throw new TribbleException("Unknown path type. No FileSystemProvider available for " + path.getRawInputString());
+ };
}
}
=====================================
src/main/java/htsjdk/tribble/FeatureCodec.java
=====================================
@@ -18,6 +18,7 @@
package htsjdk.tribble;
+import htsjdk.io.IOPath;
import htsjdk.samtools.util.LocationAware;
import htsjdk.tribble.index.tabix.TabixFormat;
=====================================
src/main/java/htsjdk/tribble/TribbleIndexedFeatureReader.java
=====================================
@@ -23,6 +23,7 @@
*/
package htsjdk.tribble;
+import htsjdk.io.HtsPath;
import htsjdk.samtools.seekablestream.SeekableStream;
import htsjdk.samtools.seekablestream.SeekableStreamFactory;
import htsjdk.samtools.util.IOUtil;
@@ -40,6 +41,7 @@ import java.io.InputStream;
import java.net.URI;
import java.net.URLEncoder;
import java.nio.channels.SeekableByteChannel;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -60,9 +62,9 @@ public class TribbleIndexedFeatureReader<T extends Feature, SOURCE> extends Abst
private Index index;
/**
- * is the path pointing to our source data a regular file?
+ * is the path backed by old style built in http(s) / ftp support instead of a FileSystemProvider
*/
- private final boolean pathIsRegularFile;
+ private final boolean pathIsOldStyleHttpOrFtp;
/**
* a potentially reusable seekable stream for queries over regular files
@@ -97,8 +99,7 @@ public class TribbleIndexedFeatureReader<T extends Feature, SOURCE> extends Abst
}
}
- // does path point to a regular file?
- this.pathIsRegularFile = SeekableStreamFactory.isFilePath(path);
+ this.pathIsOldStyleHttpOrFtp = SeekableStreamFactory.isBeingHandledByLegacyUrlSupport(path);
readHeader();
}
@@ -203,7 +204,7 @@ public class TribbleIndexedFeatureReader<T extends Feature, SOURCE> extends Abst
* @return true if
*/
private boolean reuseStreamInQuery() {
- return pathIsRegularFile;
+ return !pathIsOldStyleHttpOrFtp;
}
@Override
@@ -252,7 +253,7 @@ public class TribbleIndexedFeatureReader<T extends Feature, SOURCE> extends Abst
PositionalBufferedStream pbs = null;
try {
is = ParsingUtils.openInputStream(path, wrapper);
- if (IOUtil.hasBlockCompressedExtension(new URI(URLEncoder.encode(path, "UTF-8")))) {
+ if (IOUtil.hasBlockCompressedExtension(new HtsPath(path).getURI())) {
// TODO: TEST/FIX THIS! https://github.com/samtools/htsjdk/issues/944
// TODO -- warning I don't think this can work, the buffered input stream screws up position
is = new GZIPInputStream(new BufferedInputStream(is));
=====================================
src/main/java/htsjdk/tribble/util/FTPHelper.java
=====================================
@@ -6,6 +6,7 @@ import htsjdk.samtools.util.ftp.FTPUtils;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URISyntaxException;
import java.net.URL;
/**
@@ -35,7 +36,12 @@ public class FTPHelper implements URLHelper {
@Override
public InputStream openInputStream() throws IOException {
- String file = url.getPath();
+ String file = null;
+ try {
+ file = url.toURI().getPath();
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
FTPClient ftp = FTPUtils.connect(url.getHost(), url.getUserInfo(), null);
ftp.pasv();
ftp.retr(file);
=====================================
src/main/java/htsjdk/tribble/util/ParsingUtils.java
=====================================
@@ -23,17 +23,20 @@
*/
package htsjdk.tribble.util;
+import htsjdk.io.HtsPath;
+import htsjdk.io.IOPath;
import htsjdk.samtools.seekablestream.SeekablePathStream;
+import htsjdk.samtools.seekablestream.SeekableStreamFactory;
import htsjdk.samtools.util.IOUtil;
+
import java.awt.Color;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
+import java.net.URLEncoder;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.util.*;
@@ -49,7 +52,7 @@ public class ParsingUtils {
private static URLHelperFactory urlHelperFactory = RemoteURLHelper::new;
// HTML 4.1 color table, + orange and magenta
- private static Map<String, String> colorSymbols = new HashMap();
+ private static final Map<String, String> colorSymbols = new HashMap<>();
static {
colorSymbols.put("white", "FFFFFF");
@@ -81,13 +84,13 @@ public class ParsingUtils {
return openInputStream(path, null);
}
- static private final Set<String> URL_SCHEMES = new HashSet<>(Arrays.asList("http", "ftp", "https"));
-
/**
* open an input stream from the given path and wrap the raw byte stream with a wrapper if given
*
- * the wrapper will only be applied to paths that are not http, https, ftp, or file, i.e. any {@link java.nio.file.Path}
- * using a custom filesystem plugin
+ * the wrapper will only be applied to paths that are
+ * 1. not local files
+ * 2. not being handled by the legacy http(s)/ftp providers
+ * i.e. any {@link java.nio.file.Path} using a custom FileSystem plugin
* @param uri a uri like string
* @param wrapper to wrap the input stream in, may be used to implement caching or prefetching, etc
* @return An inputStream appropriately created from uri and conditionally wrapped with wrapper (only in certain cases)
@@ -95,18 +98,21 @@ public class ParsingUtils {
*/
public static InputStream openInputStream(final String uri, final Function<SeekableByteChannel, SeekableByteChannel> wrapper)
throws IOException {
-
- final InputStream inputStream;
-
- if (URL_SCHEMES.stream().anyMatch(uri::startsWith)) {
- inputStream = getURLHelper(new URL(uri)).openInputStream();
- } else if (!IOUtil.hasScheme(uri)) {
- File file = new File(uri);
- inputStream = Files.newInputStream(file.toPath());
+ final IOPath path = new HtsPath(uri);
+ if(path.hasFileSystemProvider()){
+ if(path.isPath()) {
+ return path.getScheme().equals("file")
+ ? Files.newInputStream(path.toPath())
+ : new SeekablePathStream(path.toPath(), wrapper);
+ } else {
+ throw new IOException("FileSystemProvider for path " + path.getRawInputString() + " exits but failed to " +
+ " create path. \n" + path.getToPathFailureReason());
+ }
+ } else if( SeekableStreamFactory.canBeHandledByLegacyUrlSupport(uri)){
+ return getURLHelper(new URL(uri)).openInputStream();
} else {
- inputStream = new SeekablePathStream(IOUtil.getPath(uri), wrapper);
+ throw new IOException("No FileSystemProvider available to handle path: " + path.getRawInputString());
}
- return inputStream;
}
public static <T> String join(String separator, Collection<T> objects) {
@@ -402,10 +408,9 @@ public class ParsingUtils {
}
public static boolean resourceExists(String resource) throws IOException{
-
- boolean remoteFile = resource.startsWith("http://") || resource.startsWith("https://") || resource.startsWith("ftp://");
+ boolean remoteFile = SeekableStreamFactory.isBeingHandledByLegacyUrlSupport(resource);
if (remoteFile) {
- URL url = null;
+ URL url;
try {
url = new URL(resource);
} catch (MalformedURLException e) {
=====================================
src/test/java/htsjdk/io/HtsPathUnitTest.java
=====================================
@@ -21,8 +21,6 @@ import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.ProviderNotFoundException;
-import java.util.HashMap;
-import java.util.Map;
import java.util.Optional;
public class HtsPathUnitTest extends HtsjdkTest {
@@ -92,6 +90,8 @@ public class HtsPathUnitTest extends HtsjdkTest {
{"gcs://abucket/bucket", "gcs://abucket/bucket", false, false},
{"gendb://somegdb", "gendb://somegdb", false, false},
{"chr1:1-100", "chr1:1-100", false, false},
+ {"ftp://broad.org/file", "ftp://broad.org/file", false, false},
+ {"ftp://broad.org/with%20space", "ftp://broad.org/with%20space", false, false},
//**********************************************************************************************
// Valid URIs which ARE valid NIO URIs (there *IS* an installed file system provider), but are
@@ -112,7 +112,7 @@ public class HtsPathUnitTest extends HtsjdkTest {
//*****************************************************************************************
// Reference that contain characters that require URI-encoding. If the input string is presented
- // with no scheme, it will be be automatically encoded by HtsPath, otherwise it
+ // with no scheme, it will be automatically encoded by HtsPath, otherwise it
// must already be URI-encoded.
//*****************************************************************************************
@@ -124,6 +124,8 @@ public class HtsPathUnitTest extends HtsjdkTest {
{"file:project/gvcf-pcr/23232_1#1/1.g.vcf.gz", "file:project/gvcf-pcr/23232_1#1/1.g.vcf.gz", true, false},
{"file:/project/gvcf-pcr/23232_1#1/1.g.vcf.gz", "file:/project/gvcf-pcr/23232_1#1/1.g.vcf.gz", true, false},
{"file:///project/gvcf-pcr/23232_1%231/1.g.vcf.g", "file:///project/gvcf-pcr/23232_1%231/1.g.vcf.g", true, true},
+
+ {"http://host/path://", "http://host/path://", false, false}
};
}
@@ -167,6 +169,15 @@ public class HtsPathUnitTest extends HtsjdkTest {
// the nul character is rejected on all of the supported platforms in both local
// filenames and URIs, so use it to test HtsPath constructor failure on all platforms
{"\0"},
+
+ // this has a non-file scheme but isn't encoded properly, best to reject these with
+ // a helpful error message than to continue on and treat it as a file:// path
+ {"ftp://broad.org/file with space"},
+
+ // if you have a scheme you need something with it
+ {"file://"},
+ {"http://"}
+
};
}
@@ -190,11 +201,14 @@ public class HtsPathUnitTest extends HtsjdkTest {
{"unknownscheme://foobar"},
{"gendb://adb"},
+
{"gcs://abucket/bucket"},
// URIs with schemes that are backed by an valid NIO provider, but for which the
// scheme-specific part is not valid.
{"file://nonexistent_authority/path/to/file.bam"}, // unknown authority "nonexistent_authority"
+
+
};
}
@@ -646,4 +660,62 @@ public class HtsPathUnitTest extends HtsjdkTest {
return localPath.toUri().normalize().getPath();
}
+ @DataProvider
+ public Object[][] getNonProblematic() {
+ return new Object[][]{
+ // URI is unencoded but no problems with a scheme
+ {"file/ name-sare/ :wierd-"},
+ {"hello there"},
+
+ //schemes schemes everywheret
+ {"file://://"},
+ {"file://://://"},
+ {"file://something://"},
+ {"file://something://somoethingelse"},
+
+ // file scheme is deliberately ignored here since it's handled specially later
+ {"file://unencoded file names/ are/ok!"},
+
+ //these aren't invalid URIs so they should never be tested by this method, they'll pass it though
+
+ {"eep/ee:e:e:ep"},
+ {"file:///"}
+ };
+ }
+ @Test(dataProvider = "getNonProblematic")
+ public void testAssertNoProblematicScheme(String path){
+ HtsPath.assertNoProblematicScheme(path, null);
+ }
+
+ @DataProvider
+ public Object[][] getProblematic(){
+ return new Object[][]{
+
+ // This is the primary use case, to detect unencoded uris that were intended to be encoded.
+ // Note that assertNoProblematicScheme doesn't check for issues constructing the URI itself
+ // it is only called after a URI parsing exception has already occured.
+ {"http://forgot.com/to encode"},
+ {"ftp://server.com/file name.txt"},
+
+ {"://"},
+ {"://://"},
+ {"://://://"},
+
+ //this is technically a valid file name, but it seems very unlikely that anyone would do this delierately
+ //better to call it out
+ {"://forgotmyscheme"},
+
+ //invalid URI, needs the rest of the path
+ {"file://"},
+ {"http://"},
+
+ //This gets rejected but it should never reach here because it's not an invalid URI
+ {"http://thisIsOkButItwouldNeverGetHere/something?file=thisone#righthere"},
+ };
+ }
+
+ @Test(dataProvider = "getProblematic", expectedExceptions = IllegalArgumentException.class)
+ public void testAssertNoProblematicSchemeRejectedCases(String path){
+ HtsPath.assertNoProblematicScheme(path, null);
+ }
}
=====================================
src/test/java/htsjdk/samtools/seekablestream/SeekableStreamFactoryTest.java
=====================================
@@ -1,5 +1,7 @@
package htsjdk.samtools.seekablestream;
+import com.google.common.jimfs.Configuration;
+import com.google.common.jimfs.Jimfs;
import htsjdk.HtsjdkTest;
import htsjdk.samtools.util.IOUtil;
import htsjdk.samtools.util.TestUtil;
@@ -10,20 +12,45 @@ import org.testng.annotations.Test;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
-import java.net.URISyntaxException;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
public class SeekableStreamFactoryTest extends HtsjdkTest {
private static final File TEST_DATA_DIR = new File("src/test/resources/htsjdk/samtools");
- @Test
- public void testIsFilePath() {
- Assert.assertEquals(SeekableStreamFactory.isFilePath("x"), true);
- Assert.assertEquals(SeekableStreamFactory.isFilePath(""), true);
- Assert.assertEquals(SeekableStreamFactory.isFilePath("http://broadinstitute.org"), false);
- Assert.assertEquals(SeekableStreamFactory.isFilePath("https://broadinstitute.org"), false);
- Assert.assertEquals(SeekableStreamFactory.isFilePath("ftp://broadinstitute.org"), false);
+ @DataProvider
+ public Object[][] getSpecialCasePaths(){
+ return new Object[][]{
+ {"x", true},
+ {"", true},
+ {"http://broadinstitute.org", false},
+ {"https://broadinstitute.org", false},
+ {"ftp://broadinstitute.org", false},
+ {"ftp://broadinstitute.org/file%20with%20spaces", false}
+ };
+ }
+
+ @Test(dataProvider = "getSpecialCasePaths")
+ @Deprecated
+ public void testIsFilePath(String path, boolean expected) {
+ Assert.assertEquals(SeekableStreamFactory.isFilePath(path), expected);
+ }
+
+
+ // this test isn't particularly useful since we're not testing the meaninful case of having the http-nio provider
+ // installed
+ @Test(dataProvider = "getSpecialCasePaths")
+ public void testIsBeingHandledByLegacyUrlSupport(String path, boolean notExpected) {
+ Assert.assertEquals(SeekableStreamFactory.isBeingHandledByLegacyUrlSupport(path), !notExpected);
+ }
+
+ @Test(dataProvider = "getSpecialCasePaths")
+ public void testCanBeHandledByLegacyUrlSupport(String path, boolean notExpected){
+ Assert.assertEquals(SeekableStreamFactory.canBeHandledByLegacyUrlSupport(path), !notExpected);
}
@DataProvider(name="getStreamForData")
@@ -38,7 +65,7 @@ public class SeekableStreamFactoryTest extends HtsjdkTest {
{ new URL(TestUtil.BASE_URL_FOR_HTTP_TESTS + "index_test.bam").toExternalForm(),
new URL(TestUtil.BASE_URL_FOR_HTTP_TESTS + "index_test.bam").toExternalForm() },
{ new URL(TestUtil.BASE_URL_FOR_HTTP_TESTS + "index_test.bam.bai").toExternalForm(),
- new URL(TestUtil.BASE_URL_FOR_HTTP_TESTS + "index_test.bam.bai").toExternalForm() }
+ new URL(TestUtil.BASE_URL_FOR_HTTP_TESTS + "index_test.bam.bai").toExternalForm() },
};
}
@@ -47,6 +74,7 @@ public class SeekableStreamFactoryTest extends HtsjdkTest {
Assert.assertEquals(SeekableStreamFactory.getInstance().getStreamFor(path).getSource(), expectedPath);
}
+
@Test
public void testPathWithEmbeddedSpace() throws IOException {
final File testBam = new File(TEST_DATA_DIR, "BAMFileIndexTest/index_test.bam");
@@ -69,7 +97,17 @@ public class SeekableStreamFactoryTest extends HtsjdkTest {
final int BYTES_TO_READ = 10;
Assert.assertEquals(seekableStream.read(new byte[BYTES_TO_READ], 0,BYTES_TO_READ), BYTES_TO_READ);
}
-
}
+ @Test
+ public void testGetSeekableStreamWorksOnJimfs() throws IOException {
+ try(final FileSystem fs = Jimfs.newFileSystem(Configuration.unix())) {
+ final Path file = fs.getPath("/file");
+ Files.writeString(file,"hello");
+ try(final SeekableStream stream = SeekableStreamFactory.getInstance().getStreamFor(file.toUri().toString())){
+ final byte[] bytes = stream.readAllBytes();
+ Assert.assertEquals(new String(bytes, StandardCharsets.UTF_8), "hello");
+ }
+ }
+ }
}
=====================================
src/test/java/htsjdk/tribble/AbstractFeatureReaderTest.java
=====================================
@@ -72,7 +72,7 @@ public class AbstractFeatureReaderTest extends HtsjdkTest {
@Test(groups = "ftp")
public void testLoadBEDFTP() throws Exception {
- final String path = "ftp://ftp.broadinstitute.org/distribution/igv/TEST/cpgIslands with spaces.hg18.bed";
+ final String path = "ftp://ftp.broadinstitute.org/distribution/igv/TEST/cpgIslands%20with%20spaces.hg18.bed";
final BEDCodec codec = new BEDCodec();
final AbstractFeatureReader<BEDFeature, LineIterator> bfs = AbstractFeatureReader.getFeatureReader(path, codec, false);
for (final Feature feat : bfs.iterator()) {
=====================================
src/test/java/htsjdk/tribble/util/ParsingUtilsTest.java
=====================================
@@ -9,6 +9,7 @@ import org.testng.Assert;
import org.testng.annotations.Test;
import java.io.*;
+import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -126,93 +127,96 @@ public class ParsingUtilsTest extends HtsjdkTest {
public void testFileDoesExist() throws IOException{
File tempFile = File.createTempFile(getClass().getSimpleName(), ".tmp");
tempFile.deleteOnExit();
- tstExists(tempFile.getAbsolutePath(), true);
- tstExists(tempFile.toURI().toString(), true);
+ testExists(tempFile.getAbsolutePath(), true);
+ testExists(tempFile.toURI().toString(), true);;
}
@Test
public void testFileDoesNotExist() throws IOException{
File tempFile = File.createTempFile(getClass().getSimpleName(), ".tmp");
tempFile.delete();
- tstExists(tempFile.getAbsolutePath(), false);
- tstExists(tempFile.toURI().toString(), false);
+ testExists(tempFile.getAbsolutePath(), false);
+ testExists(tempFile.toURI().toString(), false);
}
@Test
public void testInMemoryNioFileDoesExist() throws IOException{
- FileSystem fs = Jimfs.newFileSystem(Configuration.unix());
- Path file = fs.getPath("/file");
- Files.createFile(file);
- tstExists(file.toUri().toString(), true);
+ try(FileSystem fs = Jimfs.newFileSystem(Configuration.unix())) {
+ Path file = fs.getPath("/file");
+ Files.createFile(file);
+ testExists(file.toUri().toString(), true);
+ }
}
@Test
public void testInMemoryNioFileDoesNotExist() throws IOException{
- FileSystem fs = Jimfs.newFileSystem(Configuration.unix());
- Path file = fs.getPath("/file");
- tstExists(file.toUri().toString(), false);
+ try(FileSystem fs = Jimfs.newFileSystem(Configuration.unix())) {
+ Path file = fs.getPath("/file");
+ testExists(file.toUri().toString(), false);
+ }
}
@Test(groups = "ftp")
public void testFTPDoesExist() throws IOException{
- tstExists(AVAILABLE_FTP_URL, true);
+ testExists(AVAILABLE_FTP_URL, true);
}
@Test(groups = "ftp")
public void testFTPNotExist() throws IOException{
- tstExists(UNAVAILABLE_FTP_URL, false);
+ testExists(UNAVAILABLE_FTP_URL, false);
}
@Test
public void testHTTPDoesExist() throws IOException{
- tstExists(AVAILABLE_HTTP_URL, true);
+ testExists(AVAILABLE_HTTP_URL, true);
}
@Test
public void testHTTPNotExist() throws IOException{
- tstExists(UNAVAILABLE_HTTP_URL, false);
+ testExists(UNAVAILABLE_HTTP_URL, false);
}
- private void tstExists(String path, boolean expectExists) throws IOException{
- boolean exists = ParsingUtils.resourceExists(path);
- Assert.assertEquals(exists, expectExists);
+
+ private static void testExists(String path, boolean expectExists) throws IOException{
+ Assert.assertEquals(ParsingUtils.resourceExists(path), expectExists);
}
@Test
public void testFileOpenInputStream() throws IOException{
File tempFile = File.createTempFile(getClass().getSimpleName(), ".tmp");
tempFile.deleteOnExit();
- OutputStream os = IOUtil.openFileForWriting(tempFile);
- BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os));
- writer.write("hello");
- writer.close();
- tstStream(tempFile.getAbsolutePath());
- tstStream(tempFile.toURI().toString());
+ try(Writer writer = new BufferedWriter(new OutputStreamWriter(IOUtil.openFileForWriting(tempFile)))) {
+ writer.write("hello");
+ }
+ testStream(tempFile.getAbsolutePath());
+ testStream(tempFile.toURI().toString());
}
@Test
public void testInMemoryNioFileOpenInputStream() throws IOException{
- FileSystem fs = Jimfs.newFileSystem(Configuration.unix());
- Path file = fs.getPath("/file");
- Files.write(file, "hello".getBytes("UTF-8"));
- tstStream(file.toUri().toString());
+ try(FileSystem fs = Jimfs.newFileSystem(Configuration.unix())) {
+ Path file = fs.getPath("/file");
+ Files.write(file, "hello".getBytes(StandardCharsets.UTF_8));
+ testStream(file.toUri().toString());
+ }
}
@Test(groups = "ftp")
public void testFTPOpenInputStream() throws IOException{
- tstStream(AVAILABLE_FTP_URL);
+ testStream(AVAILABLE_FTP_URL);
}
@Test
public void testHTTPOpenInputStream() throws IOException{
- tstStream(AVAILABLE_HTTP_URL);
+ testStream(AVAILABLE_HTTP_URL);
}
- private void tstStream(String path) throws IOException{
- InputStream is = ParsingUtils.openInputStream(path);
- Assert.assertNotNull(is, "InputStream is null for " + path);
- int b = is.read();
- Assert.assertNotSame(b, -1);
+ private static void testStream(String path) throws IOException{
+ try(InputStream is = ParsingUtils.openInputStream(path)) {
+ Assert.assertNotNull(is, "InputStream is null for " + path);
+ int b = is.read();
+ Assert.assertNotSame(b, -1);
+ }
}
View it on GitLab: https://salsa.debian.org/med-team/htsjdk/-/compare/673d6642d01090c8a9c070dd2d390abb8986c6e1...80d13b32682bcff805d8acbf5b86add4677c9491
--
View it on GitLab: https://salsa.debian.org/med-team/htsjdk/-/compare/673d6642d01090c8a9c070dd2d390abb8986c6e1...80d13b32682bcff805d8acbf5b86add4677c9491
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20240211/9d75cb05/attachment-0001.htm>
More information about the debian-med-commit
mailing list