[Git][java-team/libexternalsortinginjava-java][master] 8 commits: New upstream version 0.2.4
Andreas Tille
gitlab at salsa.debian.org
Wed May 2 07:42:16 BST 2018
Andreas Tille pushed to branch master at Debian Java Maintainers / libexternalsortinginjava-java
Commits:
51baca56 by Andreas Tille at 2018-05-02T08:03:42+02:00
New upstream version 0.2.4
- - - - -
78afbe58 by Andreas Tille at 2018-05-02T08:03:42+02:00
Update upstream source from tag 'upstream/0.2.4'
Update to upstream version '0.2.4'
with Debian dir cbbe302bced2888d005e80d0714becdc9d15634b
- - - - -
5b91ea0f by Andreas Tille at 2018-05-02T08:03:42+02:00
New upstream version
- - - - -
87f9109d by Andreas Tille at 2018-05-02T08:04:09+02:00
Standards-Version: 4.1.4
- - - - -
8ccec409 by Andreas Tille at 2018-05-02T08:05:07+02:00
Point Vcs-fields to Salsa
- - - - -
4f82bba2 by Andreas Tille at 2018-05-02T08:05:24+02:00
debhelper 11
- - - - -
97b2c707 by Andreas Tille at 2018-05-02T08:05:42+02:00
Refresh patches
- - - - -
0817d910 by Andreas Tille at 2018-05-02T08:41:48+02:00
Drop notice about failed build in d/changelog
- - - - -
8 changed files:
- README.md
- debian/changelog
- debian/compat
- debian/control
- debian/patches/exclude_test_using_jamm.patch
- pom.xml
- src/main/java/com/google/code/externalsorting/ExternalSort.java
- src/test/java/com/google/code/externalsorting/ExternalSortTest.java
Changes:
=====================================
README.md
=====================================
--- a/README.md
+++ b/README.md
@@ -12,6 +12,8 @@ External-Memory Sorting in Java: useful to sort very large files using multiple
The versions 0.1 of the library are compatible with Java 6 and above. Versions 0.2 and above
require at least Java 8.
+This code is used in [Apache Jackrabbit Oak](https://github.com/apache/jackrabbit-oak).
+
Code sample
------------
=====================================
debian/changelog
=====================================
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,17 @@
+libexternalsortinginjava-java (0.2.4-1) UNRELEASED; urgency=medium
+
+ * New upstream version
+ * Standards-Version: 4.1.4
+ * Point Vcs-fields to Salsa
+ * debhelper 11
+ TODO: build fails
+ [ERROR] Failed to execute goal org.apache.maven.plugins:maven-javadoc-plugin:3.0.0:jar (default-cli) on project externalsortinginjava: Execution default-cli of goal org.apache.maven.plugins:maven-javadoc-plugin:3.0.0:jar failed.: NullPointerException -> [Help 1]
+ [ERROR]
+ [ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch.
+ [ERROR] Re-run Maven using the -X switch to enable full debug logging.
+
+ -- Andreas Tille <tille at debian.org> Wed, 02 May 2018 08:03:42 +0200
+
libexternalsortinginjava-java (0.2.3-1) unstable; urgency=medium
* Initial release (Closes: #861939)
=====================================
debian/compat
=====================================
--- a/debian/compat
+++ b/debian/compat
@@ -1 +1 @@
-10
+11
=====================================
debian/control
=====================================
--- a/debian/control
+++ b/debian/control
@@ -3,17 +3,17 @@ Maintainer: Debian Java Maintainers <pkg-java-maintainers at lists.alioth.debian.or
Uploaders: Andreas Tille <tille at debian.org>
Section: java
Priority: optional
-Build-Depends: debhelper (>= 10),
+Build-Depends: debhelper (>= 11~),
default-jdk,
maven-debian-helper (>= 2.1)
Build-Depends-Indep: libmaven-dependency-plugin-java,
junit4,
default-jdk-doc,
libmaven-javadoc-plugin-java
-Standards-Version: 3.9.8
-Vcs-Browser: https://anonscm.debian.org/cgit/pkg-java/libexternalsortinginjava-java.git
-Vcs-Git: https://anonscm.debian.org/pkg-java/libexternalsortinginjava-java.git
-Homepage: http://github.com/lemire/externalsortinginjava/
+Standards-Version: 4.1.4
+Vcs-Browser: https://salsa.debian.org/java-team/libexternalsortinginjava-java
+Vcs-Git: https://salsa.debian.org/java-team/libexternalsortinginjava-java.git
+HHomepage: http://github.com/lemire/externalsortinginjava/
Package: libexternalsortinginjava-java
Architecture: all
=====================================
debian/patches/exclude_test_using_jamm.patch
=====================================
--- a/debian/patches/exclude_test_using_jamm.patch
+++ b/debian/patches/exclude_test_using_jamm.patch
@@ -4,15 +4,15 @@ Description: Skip test requiring inpackaged class jamm
--- a/src/test/java/com/google/code/externalsorting/ExternalSortTest.java
+++ b/src/test/java/com/google/code/externalsorting/ExternalSortTest.java
-@@ -21,7 +21,6 @@ import java.util.Scanner;
- import org.junit.After;
+@@ -29,7 +29,6 @@ import org.junit.After;
import org.junit.Before;
+ import org.junit.Ignore;
import org.junit.Test;
-import org.github.jamm.*;
/**
* Unit test for simple App.
-@@ -131,25 +130,6 @@ public class ExternalSortTest {
+@@ -139,25 +138,6 @@ public class ExternalSortTest {
System.out.println("[performance] String size estimator uses "+bestdiff * 1.0 / N + " ns per string");
}
=====================================
pom.xml
=====================================
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
<groupId>com.google.code.externalsortinginjava</groupId>
<artifactId>externalsortinginjava</artifactId>
<packaging>jar</packaging>
- <version>0.2.3</version>
+ <version>0.2.4</version>
<name>externalsortinginjava</name>
<url>http://github.com/lemire/externalsortinginjava/</url>
<description>Sometimes, you want to sort large file without first loading them into memory. The solution is to use External Sorting. You divide the files into small blocks, sort each block in RAM, and then merge the result.
@@ -198,6 +198,6 @@
<connection>scm:git:git at github.com:lemire/externalsortinginjava.git</connection>
<url>scm:git:git at github.com:lemire/externalsortinginjava.git</url>
<developerConnection>scm:git:git at github.com:lemire/externalsortinginjava.git</developerConnection>
- <tag>externalsortinginjava-0.2.3</tag>
+ <tag>externalsortinginjava-0.2.4</tag>
</scm>
</project>
=====================================
src/main/java/com/google/code/externalsorting/ExternalSort.java
=====================================
--- a/src/main/java/com/google/code/externalsorting/ExternalSort.java
+++ b/src/main/java/com/google/code/externalsorting/ExternalSort.java
@@ -210,7 +210,7 @@ public class ExternalSort {
* @throws IOException generic IO exception
*
*/
- public static int mergeSortedFiles(BufferedWriter fbw,
+ public static long mergeSortedFiles(BufferedWriter fbw,
final Comparator<String> cmp, boolean distinct,
List<BinaryFileBuffer> buffers) throws IOException {
PriorityQueue<BinaryFileBuffer> pq = new PriorityQueue<>(
@@ -226,7 +226,7 @@ public class ExternalSort {
pq.add(bfb);
}
}
- int rowcounter = 0;
+ long rowcounter = 0;
try {
if (!distinct) {
while (pq.size() > 0) {
@@ -290,7 +290,7 @@ public class ExternalSort {
* @return The number of lines sorted.
* @throws IOException generic IO exception
*/
- public static int mergeSortedFiles(List<File> files, File outputfile)
+ public static long mergeSortedFiles(List<File> files, File outputfile)
throws IOException {
return mergeSortedFiles(files, outputfile, defaultcomparator,
Charset.defaultCharset());
@@ -306,7 +306,7 @@ public class ExternalSort {
* @return The number of lines sorted.
* @throws IOException generic IO exception
*/
- public static int mergeSortedFiles(List<File> files, File outputfile,
+ public static long mergeSortedFiles(List<File> files, File outputfile,
final Comparator<String> cmp) throws IOException {
return mergeSortedFiles(files, outputfile, cmp,
Charset.defaultCharset());
@@ -324,7 +324,7 @@ public class ExternalSort {
* @return The number of lines sorted.
* @throws IOException generic IO exception
*/
- public static int mergeSortedFiles(List<File> files, File outputfile,
+ public static long mergeSortedFiles(List<File> files, File outputfile,
final Comparator<String> cmp, boolean distinct)
throws IOException {
return mergeSortedFiles(files, outputfile, cmp,
@@ -343,7 +343,7 @@ public class ExternalSort {
* @return The number of lines sorted.
* @throws IOException generic IO exception
*/
- public static int mergeSortedFiles(List<File> files, File outputfile,
+ public static long mergeSortedFiles(List<File> files, File outputfile,
final Comparator<String> cmp, Charset cs) throws IOException {
return mergeSortedFiles(files, outputfile, cmp, cs, false);
}
@@ -363,7 +363,7 @@ public class ExternalSort {
* @throws IOException generic IO exception
* @since v0.1.2
*/
- public static int mergeSortedFiles(List<File> files, File outputfile,
+ public static long mergeSortedFiles(List<File> files, File outputfile,
final Comparator<String> cmp, Charset cs, boolean distinct)
throws IOException {
return mergeSortedFiles(files, outputfile, cmp, cs, distinct,
@@ -389,7 +389,7 @@ public class ExternalSort {
* @throws IOException generic IO exception
* @since v0.1.4
*/
- public static int mergeSortedFiles(List<File> files, File outputfile,
+ public static long mergeSortedFiles(List<File> files, File outputfile,
final Comparator<String> cmp, Charset cs, boolean distinct,
boolean append, boolean usegzip) throws IOException {
ArrayList<BinaryFileBuffer> bfbs = new ArrayList<>();
@@ -412,7 +412,7 @@ public class ExternalSort {
}
BufferedWriter fbw = new BufferedWriter(new OutputStreamWriter(
new FileOutputStream(outputfile, append), cs));
- int rowcounter = mergeSortedFiles(fbw, cmp, distinct, bfbs);
+ long rowcounter = mergeSortedFiles(fbw, cmp, distinct, bfbs);
for (File f : files) {
f.delete();
}
=====================================
src/test/java/com/google/code/externalsorting/ExternalSortTest.java
=====================================
--- a/src/test/java/com/google/code/externalsorting/ExternalSortTest.java
+++ b/src/test/java/com/google/code/externalsorting/ExternalSortTest.java
@@ -13,13 +13,21 @@ import java.io.FileReader;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Scanner;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.github.jamm.*;
@@ -407,4 +415,47 @@ public class ExternalSortTest {
}
}
+ /**
+ * Sort a text file with lines greater than {@link Integer#MAX_VALUE}.
+ *
+ * @throws IOException
+ */
+ @Ignore("This test takes too long to execute")
+ @Test
+ public void sortVeryLargeFile() throws IOException {
+ final Path veryLargeFile = getTestFile();
+ final Path outputFile = Files.createTempFile("Merged-File", ".tmp");
+ final long sortedLines = ExternalSort.mergeSortedFiles(ExternalSort.sortInBatch(veryLargeFile.toFile()), outputFile.toFile());
+ final long expectedLines = 2148L * 1000000L;
+ assertEquals(expectedLines, sortedLines);
+ }
+
+ /**
+ * Generate a test file with 2148 million lines.
+ *
+ * @throws IOException
+ */
+ private Path getTestFile() throws IOException {
+ System.out.println("Temp File Creation: Started");
+ final Path path = Files.createTempFile("IntegrationTestFile", ".txt");
+ final List<String> idList = new ArrayList<>();
+ final int saneLimit = 1000000;
+ IntStream.range(0, saneLimit)
+ .forEach(i -> idList.add("A"));
+ final String content = idList.stream().collect(Collectors.joining("\n"));
+ Files.write(path, content.getBytes(StandardCharsets.UTF_8), StandardOpenOption.TRUNCATE_EXISTING);
+ final String newLine = "\n";
+ IntStream.range(1, 2148)
+ .forEach(i -> {
+ try {
+ Files.write(path, newLine.getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND);
+ Files.write(path, content.getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND);
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+ });
+ System.out.println("Temp File Creation: Finished");
+ return path;
+ }
+
}
View it on GitLab: https://salsa.debian.org/java-team/libexternalsortinginjava-java/compare/42ba13c5a9362c78930731e6afb0c6caa5cc0291...0817d910119deebc648c54236df719344d6f98ee
---
View it on GitLab: https://salsa.debian.org/java-team/libexternalsortinginjava-java/compare/42ba13c5a9362c78930731e6afb0c6caa5cc0291...0817d910119deebc648c54236df719344d6f98ee
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-java-commits/attachments/20180502/017d2a76/attachment.html>
More information about the pkg-java-commits
mailing list