[Git][java-team/jackson-core][upstream] 4 commits: New upstream version 2.9.5
Emmanuel Bourg
gitlab at salsa.debian.org
Fri Dec 28 23:00:47 GMT 2018
Emmanuel Bourg pushed to branch upstream at Debian Java Maintainers / jackson-core
Commits:
c66353ff by Emmanuel Bourg at 2018-12-28T22:21:53Z
New upstream version 2.9.5
- - - - -
0b5233e1 by Emmanuel Bourg at 2018-12-28T22:28:41Z
New upstream version 2.9.6
- - - - -
c24d39ef by Emmanuel Bourg at 2018-12-28T22:36:05Z
New upstream version 2.9.7
- - - - -
91563028 by Emmanuel Bourg at 2018-12-28T22:38:40Z
New upstream version 2.9.8
- - - - -
21 changed files:
- pom.xml
- release-notes/CREDITS → release-notes/CREDITS-2.x
- release-notes/VERSION → release-notes/VERSION-2.x
- src/main/java/com/fasterxml/jackson/core/Base64Variant.java
- src/main/java/com/fasterxml/jackson/core/Base64Variants.java
- src/main/java/com/fasterxml/jackson/core/JsonFactory.java
- src/main/java/com/fasterxml/jackson/core/base/ParserBase.java
- src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java
- src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java
- src/main/java/com/fasterxml/jackson/core/json/UTF8DataInputJsonParser.java
- src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java
- src/main/java/com/fasterxml/jackson/core/util/BufferRecyclers.java
- + src/main/java/com/fasterxml/jackson/core/util/ThreadLocalBufferManager.java
- src/test/java/com/fasterxml/jackson/core/BaseTest.java
- src/test/java/com/fasterxml/jackson/core/VersionTest.java
- src/test/java/com/fasterxml/jackson/core/base64/Base64BinaryParsingTest.java
- src/test/java/com/fasterxml/jackson/core/json/JsonFactoryTest.java
- + src/test/java/com/fasterxml/jackson/core/json/async/AsyncConcurrencyTest.java
- + src/test/java/com/fasterxml/jackson/core/json/async/AsyncTokenFilterTest.java
- + src/test/java/com/fasterxml/jackson/core/read/NumberOverflowTest.java
- src/test/java/com/fasterxml/jackson/core/read/NumberParsingTest.java
Changes:
=====================================
pom.xml
=====================================
@@ -4,13 +4,13 @@
<groupId>com.fasterxml.jackson</groupId>
<!-- For 2.9.2 and beyond, new parent pom; extends jackson-bom -->
<artifactId>jackson-base</artifactId>
- <version>2.9.4</version>
+ <version>2.9.8</version>
</parent>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<name>Jackson-core</name>
- <version>2.9.4</version>
+ <version>2.9.8</version>
<packaging>bundle</packaging>
<description>Core Jackson processing abstractions (aka Streaming API), implementation for JSON</description>
<inceptionYear>2008</inceptionYear>
@@ -20,7 +20,7 @@
<connection>scm:git:git at github.com:FasterXML/jackson-core.git</connection>
<developerConnection>scm:git:git at github.com:FasterXML/jackson-core.git</developerConnection>
<url>http://github.com/FasterXML/jackson-core</url>
- <tag>jackson-core-2.9.4</tag>
+ <tag>jackson-core-2.9.8</tag>
</scm>
<properties>
@@ -45,7 +45,17 @@ com.fasterxml.jackson.core.*;version=${project.version}
<jdk.module.name>com.fasterxml.jackson.core</jdk.module.name>
</properties>
- <!-- parent provides junit dep, not repeated here -->
+ <!-- Alas, need to include snapshot reference since otherwise can not find
+ snapshot of parent... -->
+ <repositories>
+ <repository>
+ <id>sonatype-nexus-snapshots</id>
+ <name>Sonatype Nexus Snapshots</name>
+ <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+ <releases><enabled>false</enabled></releases>
+ <snapshots><enabled>true</enabled></snapshots>
+ </repository>
+ </repositories>
<build>
<plugins>
@@ -74,6 +84,11 @@ com.fasterxml.jackson.core.*;version=${project.version}
<excludes>
<exclude>**/failing/**/*.java</exclude>
</excludes>
+<!-- 13-Apr-2018, tatu: for debugging [core#400]
+ <systemPropertyVariables>
+<com.fasterxml.jackson.core.util.BufferRecyclers.trackReusableBuffers>true</com.fasterxml.jackson.core.util.BufferRecyclers.trackReusableBuffers>
+ </systemPropertyVariables>
+-->
</configuration>
</plugin>
<!-- settings are fine, but needed to trigger execution! -->
=====================================
release-notes/CREDITS → release-notes/CREDITS-2.x
=====================================
@@ -145,3 +145,14 @@ Rafal Foltynski (rfoltyns at github)
(2.9.0)
* Contributed#208: Make use of `_matchCount` in `FilteringParserDelegate`
(2.9.0)
+
+Jeroen Borgers (jborgers at github)
+ * Reported, contributed impl for #400: Add mechanism for forcing `BufferRecycler` released
+ (to call on shutdown)
+ (2.9.6)
+
+Doug Roper (htmldoug at github)
+ * Suggested #463: Ensure that `skipChildren()` of non-blocking `JsonParser` will throw
+ exception if not enough input
+ (2.9.6)
+
=====================================
release-notes/VERSION → release-notes/VERSION-2.x
=====================================
@@ -14,7 +14,34 @@ JSON library.
=== Releases ===
------------------------------------------------------------------------
-2.9.4 (24-Jan-2017)
+2.9.8 (15-Dec-2018)
+
+#488: Fail earlier on coercions from "too big" `BigInteger` into
+ fixed-size types (`int`, `long`, `short`)
+- Improve exception message for missing Base64 padding (see databind#2183)
+
+2.9.7 (19-Sep-2018)
+
+#476: Problem with `BufferRecycler` via async parser (or when sharing parser
+ across threads)
+#477: Exception while decoding Base64 value with escaped `=` character
+#488: Fail earlier on coercions from "too big" `BigInteger` into
+ fixed-size types (`int`, `long`, `short`)
+
+2.9.6 (12-Jun-2018)
+
+#400: Add mechanism for forcing `BufferRecycler` released (to call on shutdown)
+ (contributed by Jeroen B)
+#460: Failing to link `ObjectCodec` with `JsonFactory` copy constructor
+#463: Ensure that `skipChildren()` of non-blocking `JsonParser` will throw
+ exception if not enough input
+ (requested by Doug R)
+
+2.9.5 (26-Mar-2018)
+
+No changes since 2.9.4
+
+2.9.4 (24-Jan-2018)
#414: Base64 MIME variant does not ignore white space chars as per RFC2045
(reported by tmoschou at github)
@@ -62,7 +89,7 @@ JSON library.
#374: Minimal and DefaultPrettyPrinter with configurable separators
(contributed by Rafal F)
-2.8.11 (not yet released)
+2.8.11 (23-Dec-2017)
#418: ArrayIndexOutOfBoundsException from UTF32Reader.read on invalid input
(reported, contributed fix for by pfitzsimons-r7 at github)
@@ -142,7 +169,23 @@ No changes since 2.8.0
for `getCurrentToken()` and `getCurrentTokenId()`, respectively. Existing methods
will likely be deprecated in 2.9.
-2.7.10 (not yet released)
+2.7.9.3:
+
+#1872: NullPointerException in SubTypeValidator.validateSubType when
+ validating Spring interface
+#1931: Two more c3p0 gadgets to exploit default typing issue
+
+2.7.9.2 (20-Dec-2017)
+
+#1607: `@JsonIdentityReference` not used when setup on class only
+#1628: Don't print to error stream about failure to load JDK 7 types
+#1680: Blacklist couple more types for deserialization
+#1737: Block more JDK types from polymorphic deserialization
+#1855: Blacklist for more serialization gadgets (dbcp/tomcat, spring)
+
+2.7.9.1 (18-Apr-2017)
+
+#1599: Jackson Deserializer security vulnerability
2.7.9 (04-Feb-2017)
=====================================
src/main/java/com/fasterxml/jackson/core/Base64Variant.java
=====================================
@@ -593,7 +593,19 @@ public final class Base64Variant
}
protected void _reportBase64EOF() throws IllegalArgumentException {
- throw new IllegalArgumentException("Unexpected end-of-String in base64 content");
+ throw new IllegalArgumentException(missingPaddingMessage());
}
+
+ /**
+ * Helper method that will construct a message to use in exceptions for cases where input ends
+ * prematurely in place where padding would be expected.
+ *
+ * @since 2.10
+ */
+ public String missingPaddingMessage() {
+ return String.format("Unexpected end of base64-encoded String: base64 variant '%s' expects padding (one or more '%c' characters) at the end",
+ getName(), getPaddingChar());
+ }
+
}
=====================================
src/main/java/com/fasterxml/jackson/core/Base64Variants.java
=====================================
@@ -68,9 +68,7 @@ public final class Base64Variants
// Replace plus with hyphen, slash with underscore (and no padding)
sb.setCharAt(sb.indexOf("+"), '-');
sb.setCharAt(sb.indexOf("/"), '_');
- /* And finally, let's not split lines either, wouldn't work too
- * well with URLs
- */
+ // And finally, let's not split lines either, wouldn't work too well with URLs
MODIFIED_FOR_URL = new Base64Variant("MODIFIED-FOR-URL", sb.toString(), false, Base64Variant.PADDING_CHAR_NONE, Integer.MAX_VALUE);
}
=====================================
src/main/java/com/fasterxml/jackson/core/JsonFactory.java
=====================================
@@ -286,7 +286,7 @@ public class JsonFactory
*/
protected JsonFactory(JsonFactory src, ObjectCodec codec)
{
- _objectCodec = null;
+ _objectCodec = codec;
_factoryFeatures = src._factoryFeatures;
_parserFeatures = src._parserFeatures;
_generatorFeatures = src._generatorFeatures;
@@ -960,7 +960,7 @@ public class JsonFactory
// 17-May-2017, tatu: Need to take care not to accidentally create JSON parser
// for non-JSON input:
_requireJSONFactory("Non-blocking source not (yet?) support for this format (%s)");
- IOContext ctxt = _createContext(null, false);
+ IOContext ctxt = _createNonBlockingContext(null);
ByteQuadsCanonicalizer can = _byteSymbolCanonicalizer.makeChild(_factoryFeatures);
return new NonBlockingJsonParser(ctxt, _parserFeatures, can);
}
@@ -1548,6 +1548,19 @@ public class JsonFactory
return new IOContext(_getBufferRecycler(), srcRef, resourceManaged);
}
+ /**
+ * Overridable factory method that actually instantiates desired
+ * context object for async (non-blocking) parsing
+ *
+ * @since 2.9.7
+ */
+ protected IOContext _createNonBlockingContext(Object srcRef) {
+ // [jackson-core#476]: disable buffer recycling for 2.9 to avoid concurrency issues;
+ // easiest done by just constructing private "recycler":
+ BufferRecycler recycler = new BufferRecycler();
+ return new IOContext(recycler, srcRef, false);
+ }
+
/**
* @since 2.8
*/
=====================================
src/main/java/com/fasterxml/jackson/core/base/ParserBase.java
=====================================
@@ -823,10 +823,10 @@ public abstract class ParserBase extends ParserMinimalBase
}
} catch (NumberFormatException nex) {
// Can this ever occur? Due to overflow, maybe?
- _wrapError("Malformed numeric value '"+_textBuffer.contentsAsString()+"'", nex);
+ _wrapError("Malformed numeric value ("+_longNumberDesc(_textBuffer.contentsAsString())+")", nex);
}
}
-
+
private void _parseSlowInt(int expType) throws IOException
{
String numStr = _textBuffer.contentsAsString();
@@ -843,16 +843,33 @@ public abstract class ParserBase extends ParserMinimalBase
_numberLong = Long.parseLong(numStr);
_numTypesValid = NR_LONG;
} else {
- // nope, need the heavy guns... (rare case)
- _numberBigInt = new BigInteger(numStr);
- _numTypesValid = NR_BIGINT;
+ // 16-Oct-2018, tatu: Need to catch "too big" early due to [jackson-core#488]
+ if ((expType == NR_INT) || (expType == NR_LONG)) {
+ _reportTooLongInt(expType, numStr);
+ }
+ if ((expType == NR_DOUBLE) || (expType == NR_FLOAT)) {
+ _numberDouble = NumberInput.parseDouble(numStr);
+ _numTypesValid = NR_DOUBLE;
+ } else {
+ // nope, need the heavy guns... (rare case)
+ _numberBigInt = new BigInteger(numStr);
+ _numTypesValid = NR_BIGINT;
+ }
}
} catch (NumberFormatException nex) {
// Can this ever occur? Due to overflow, maybe?
- _wrapError("Malformed numeric value '"+numStr+"'", nex);
+ _wrapError("Malformed numeric value ("+_longNumberDesc(numStr)+")", nex);
}
}
-
+
+ // @since 2.9.8
+ protected void _reportTooLongInt(int expType, String rawNum) throws IOException
+ {
+ final String numDesc = _longIntegerDesc(rawNum);
+ _reportError("Numeric value (%s) out of range of %s", numDesc,
+ (expType == NR_LONG) ? "long" : "int");
+ }
+
/*
/**********************************************************
/* Numeric conversions
@@ -1029,7 +1046,9 @@ public abstract class ParserBase extends ParserMinimalBase
// otherwise try to find actual triplet value
int bits = b64variant.decodeBase64Char(unescaped);
if (bits < 0) {
- throw reportInvalidBase64Char(b64variant, unescaped, index);
+ if (bits != Base64Variant.BASE64_VALUE_PADDING) {
+ throw reportInvalidBase64Char(b64variant, unescaped, index);
+ }
}
return bits;
}
@@ -1049,7 +1068,10 @@ public abstract class ParserBase extends ParserMinimalBase
// otherwise try to find actual triplet value
int bits = b64variant.decodeBase64Char(unescaped);
if (bits < 0) {
- throw reportInvalidBase64Char(b64variant, unescaped, index);
+ // second check since padding can only be 3rd or 4th byte (index #2 or #3)
+ if ((bits != Base64Variant.BASE64_VALUE_PADDING) || (index < 2)) {
+ throw reportInvalidBase64Char(b64variant, unescaped, index);
+ }
}
return bits;
}
@@ -1081,6 +1103,12 @@ public abstract class ParserBase extends ParserMinimalBase
return new IllegalArgumentException(base);
}
+ // since 2.9.8
+ protected void _handleBase64MissingPadding(Base64Variant b64variant) throws IOException
+ {
+ _reportError(b64variant.missingPaddingMessage());
+ }
+
/*
/**********************************************************
/* Internal/package methods: other
=====================================
src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java
=====================================
@@ -249,6 +249,12 @@ public abstract class ParserMinimalBase extends JsonParser
if (--open == 0) {
return this;
}
+ // 23-May-2018, tatu: [core#463] Need to consider non-blocking case...
+ } else if (t == JsonToken.NOT_AVAILABLE) {
+ // Nothing much we can do except to either return `null` (which seems wrong),
+ // or, what we actually do, signal error
+ _reportError("Not enough content available for `skipChildren()`: non-blocking parser? (%s)",
+ getClass().getName());
}
}
}
@@ -541,12 +547,36 @@ public abstract class ParserMinimalBase extends JsonParser
protected void reportOverflowInt() throws IOException {
_reportError(String.format("Numeric value (%s) out of range of int (%d - %s)",
- getText(), Integer.MIN_VALUE, Integer.MAX_VALUE));
+ _longIntegerDesc(getText()), Integer.MIN_VALUE, Integer.MAX_VALUE));
}
-
+
protected void reportOverflowLong() throws IOException {
_reportError(String.format("Numeric value (%s) out of range of long (%d - %s)",
- getText(), Long.MIN_VALUE, Long.MAX_VALUE));
+ _longIntegerDesc(getText()), Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+
+ // @since 2.9.8
+ protected String _longIntegerDesc(String rawNum) {
+ int rawLen = rawNum.length();
+ if (rawLen < 1000) {
+ return rawNum;
+ }
+ if (rawNum.startsWith("-")) {
+ rawLen -= 1;
+ }
+ return String.format("[Integer with %d digits]", rawLen);
+ }
+
+ // @since 2.9.8
+ protected String _longNumberDesc(String rawNum) {
+ int rawLen = rawNum.length();
+ if (rawLen < 1000) {
+ return rawNum;
+ }
+ if (rawNum.startsWith("-")) {
+ rawLen -= 1;
+ }
+ return String.format("[number with %d characters]", rawLen);
}
protected void _reportUnexpectedChar(int ch, String comment) throws JsonParseException
=====================================
src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java
=====================================
@@ -549,9 +549,13 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
- if (ch == '"' && !b64variant.usesPadding()) {
+ if (ch == '"') {
decodedData >>= 4;
buffer[outputPtr++] = (byte) decodedData;
+ if (b64variant.usesPadding()) {
+ --_inputPtr; // to keep parser state bit more consistent
+ _handleBase64MissingPadding(b64variant);
+ }
break;
}
bits = _decodeBase64Escape(b64variant, ch, 2);
@@ -563,7 +567,9 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
}
ch = _inputBuffer[_inputPtr++];
if (!b64variant.usesPaddingChar(ch)) {
- throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
+ if (_decodeBase64Escape(b64variant, ch, 3) != Base64Variant.BASE64_VALUE_PADDING) {
+ throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
+ }
}
// Got 12 bits, only need 8, need to shift
decodedData >>= 4;
@@ -582,10 +588,14 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
- if (ch == '"' && !b64variant.usesPadding()) {
+ if (ch == '"') {
decodedData >>= 2;
buffer[outputPtr++] = (byte) (decodedData >> 8);
buffer[outputPtr++] = (byte) decodedData;
+ if (b64variant.usesPadding()) {
+ --_inputPtr; // to keep parser state bit more consistent
+ _handleBase64MissingPadding(b64variant);
+ }
break;
}
bits = _decodeBase64Escape(b64variant, ch, 3);
@@ -2008,9 +2018,7 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
} while (ptr < inputLen);
}
- /* Either ran out of input, or bumped into an escape
- * sequence...
- */
+ // Either ran out of input, or bumped into an escape sequence...
_textBuffer.resetWithCopy(_inputBuffer, _inputPtr, (ptr-_inputPtr));
_inputPtr = ptr;
_finishString2();
@@ -2700,9 +2708,13 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
- if (ch == '"' && !b64variant.usesPadding()) {
+ if (ch == '"') {
decodedData >>= 4;
builder.append(decodedData);
+ if (b64variant.usesPadding()) {
+ --_inputPtr; // to keep parser state bit more consistent
+ _handleBase64MissingPadding(b64variant);
+ }
return builder.toByteArray();
}
bits = _decodeBase64Escape(b64variant, ch, 2);
@@ -2714,7 +2726,9 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
}
ch = _inputBuffer[_inputPtr++];
if (!b64variant.usesPaddingChar(ch)) {
- throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
+ if (_decodeBase64Escape(b64variant, ch, 3) != Base64Variant.BASE64_VALUE_PADDING) {
+ throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
+ }
}
// Got 12 bits, only need 8, need to shift
decodedData >>= 4;
@@ -2734,9 +2748,13 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
- if (ch == '"' && !b64variant.usesPadding()) {
+ if (ch == '"') {
decodedData >>= 2;
builder.appendTwoBytes(decodedData);
+ if (b64variant.usesPadding()) {
+ --_inputPtr; // to keep parser state bit more consistent
+ _handleBase64MissingPadding(b64variant);
+ }
return builder.toByteArray();
}
bits = _decodeBase64Escape(b64variant, ch, 3);
=====================================
src/main/java/com/fasterxml/jackson/core/json/UTF8DataInputJsonParser.java
=====================================
@@ -481,9 +481,12 @@ public class UTF8DataInputJsonParser
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// could also just be 'missing' padding
- if (ch == '"' && !b64variant.usesPadding()) {
+ if (ch == INT_QUOTE) {
decodedData >>= 4;
buffer[outputPtr++] = (byte) decodedData;
+ if (b64variant.usesPadding()) {
+ _handleBase64MissingPadding(b64variant);
+ }
break;
}
bits = _decodeBase64Escape(b64variant, ch, 2);
@@ -492,7 +495,10 @@ public class UTF8DataInputJsonParser
// Ok, must get padding
ch = _inputData.readUnsignedByte();
if (!b64variant.usesPaddingChar(ch)) {
- throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
+ if ((ch != INT_BACKSLASH)
+ || _decodeBase64Escape(b64variant, ch, 3) != Base64Variant.BASE64_VALUE_PADDING) {
+ throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
+ }
}
// Got 12 bits, only need 8, need to shift
decodedData >>= 4;
@@ -508,10 +514,13 @@ public class UTF8DataInputJsonParser
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// could also just be 'missing' padding
- if (ch == '"' && !b64variant.usesPadding()) {
+ if (ch == INT_QUOTE) {
decodedData >>= 2;
buffer[outputPtr++] = (byte) (decodedData >> 8);
buffer[outputPtr++] = (byte) decodedData;
+ if (b64variant.usesPadding()) {
+ _handleBase64MissingPadding(b64variant);
+ }
break;
}
bits = _decodeBase64Escape(b64variant, ch, 3);
@@ -2753,9 +2762,12 @@ public class UTF8DataInputJsonParser
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// could also just be 'missing' padding
- if (ch == '"' && !b64variant.usesPadding()) {
+ if (ch == INT_QUOTE) {
decodedData >>= 4;
builder.append(decodedData);
+ if (b64variant.usesPadding()) {
+ _handleBase64MissingPadding(b64variant);
+ }
return builder.toByteArray();
}
bits = _decodeBase64Escape(b64variant, ch, 2);
@@ -2763,7 +2775,10 @@ public class UTF8DataInputJsonParser
if (bits == Base64Variant.BASE64_VALUE_PADDING) {
ch = _inputData.readUnsignedByte();
if (!b64variant.usesPaddingChar(ch)) {
- throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
+ if ((ch != INT_BACKSLASH)
+ || _decodeBase64Escape(b64variant, ch, 3) != Base64Variant.BASE64_VALUE_PADDING) {
+ throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
+ }
}
// Got 12 bits, only need 8, need to shift
decodedData >>= 4;
@@ -2779,9 +2794,12 @@ public class UTF8DataInputJsonParser
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// could also just be 'missing' padding
- if (ch == '"' && !b64variant.usesPadding()) {
+ if (ch == INT_QUOTE) {
decodedData >>= 2;
builder.appendTwoBytes(decodedData);
+ if (b64variant.usesPadding()) {
+ _handleBase64MissingPadding(b64variant);
+ }
return builder.toByteArray();
}
bits = _decodeBase64Escape(b64variant, ch, 3);
=====================================
src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java
=====================================
@@ -487,18 +487,14 @@ public class UTF8StreamJsonParser
(_currToken != JsonToken.VALUE_EMBEDDED_OBJECT || _binaryValue == null)) {
_reportError("Current token ("+_currToken+") not VALUE_STRING or VALUE_EMBEDDED_OBJECT, can not access as binary");
}
- /* To ensure that we won't see inconsistent data, better clear up
- * state...
- */
+ // To ensure that we won't see inconsistent data, better clear up state...
if (_tokenIncomplete) {
try {
_binaryValue = _decodeBase64(b64variant);
} catch (IllegalArgumentException iae) {
throw _constructError("Failed to decode VALUE_STRING as base64 ("+b64variant+"): "+iae.getMessage());
}
- /* let's clear incomplete only now; allows for accessing other
- * textual content in error cases
- */
+ // let's clear incomplete only now; allows for accessing other textual content in error cases
_tokenIncomplete = false;
} else { // may actually require conversion...
if (_binaryValue == null) {
@@ -588,9 +584,13 @@ public class UTF8StreamJsonParser
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
- if (ch == '"' && !b64variant.usesPadding()) {
+ if (ch == INT_QUOTE) {
decodedData >>= 4;
buffer[outputPtr++] = (byte) decodedData;
+ if (b64variant.usesPadding()) {
+ --_inputPtr; // to keep parser state bit more consistent
+ _handleBase64MissingPadding(b64variant);
+ }
break;
}
bits = _decodeBase64Escape(b64variant, ch, 2);
@@ -602,7 +602,9 @@ public class UTF8StreamJsonParser
}
ch = _inputBuffer[_inputPtr++] & 0xFF;
if (!b64variant.usesPaddingChar(ch)) {
- throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
+ if (_decodeBase64Escape(b64variant, ch, 3) != Base64Variant.BASE64_VALUE_PADDING) {
+ throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
+ }
}
// Got 12 bits, only need 8, need to shift
decodedData >>= 4;
@@ -621,10 +623,14 @@ public class UTF8StreamJsonParser
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
- if (ch == '"' && !b64variant.usesPadding()) {
+ if (ch == INT_QUOTE) {
decodedData >>= 2;
buffer[outputPtr++] = (byte) (decodedData >> 8);
buffer[outputPtr++] = (byte) decodedData;
+ if (b64variant.usesPadding()) {
+ --_inputPtr; // to keep parser state bit more consistent
+ _handleBase64MissingPadding(b64variant);
+ }
break;
}
bits = _decodeBase64Escape(b64variant, ch, 3);
@@ -3608,10 +3614,14 @@ public class UTF8StreamJsonParser
// First branch: can get padding (-> 1 byte)
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
- // as per [JACKSON-631], could also just be 'missing' padding
- if (ch == '"' && !b64variant.usesPadding()) {
+ // could also just be 'missing' padding
+ if (ch == INT_QUOTE) {
decodedData >>= 4;
builder.append(decodedData);
+ if (b64variant.usesPadding()) {
+ --_inputPtr; // to keep parser state bit more consistent
+ _handleBase64MissingPadding(b64variant);
+ }
return builder.toByteArray();
}
bits = _decodeBase64Escape(b64variant, ch, 2);
@@ -3623,7 +3633,9 @@ public class UTF8StreamJsonParser
}
ch = _inputBuffer[_inputPtr++] & 0xFF;
if (!b64variant.usesPaddingChar(ch)) {
- throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
+ if (_decodeBase64Escape(b64variant, ch, 3) != Base64Variant.BASE64_VALUE_PADDING) {
+ throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
+ }
}
// Got 12 bits, only need 8, need to shift
decodedData >>= 4;
@@ -3641,21 +3653,22 @@ public class UTF8StreamJsonParser
bits = b64variant.decodeBase64Char(ch);
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
- // as per [JACKSON-631], could also just be 'missing' padding
- if (ch == '"' && !b64variant.usesPadding()) {
+ // could also just be 'missing' padding
+ if (ch == INT_QUOTE) {
decodedData >>= 2;
builder.appendTwoBytes(decodedData);
+ if (b64variant.usesPadding()) {
+ --_inputPtr; // to keep parser state bit more consistent
+ _handleBase64MissingPadding(b64variant);
+ }
return builder.toByteArray();
}
bits = _decodeBase64Escape(b64variant, ch, 3);
}
if (bits == Base64Variant.BASE64_VALUE_PADDING) {
- /* With padding we only get 2 bytes; but we have
- * to shift it a bit so it is identical to triplet
- * case with partial output.
- * 3 chars gives 3x6 == 18 bits, of which 2 are
- * dummies, need to discard:
- */
+ // With padding we only get 2 bytes; but we have to shift it
+ // a bit so it is identical to triplet case with partial output.
+ // 3 chars gives 3x6 == 18 bits, of which 2 are dummies, need to discard:
decodedData >>= 2;
builder.appendTwoBytes(decodedData);
continue;
=====================================
src/main/java/com/fasterxml/jackson/core/util/BufferRecyclers.java
=====================================
@@ -15,6 +15,32 @@ import com.fasterxml.jackson.core.io.JsonStringEncoder;
*/
public class BufferRecyclers
{
+ /**
+ * System property that is checked to see if recycled buffers (see {@link BufferRecycler})
+ * should be tracked, for purpose of forcing release of all such buffers, typically
+ * during major classloading.
+ *
+ * @since 2.9.6
+ */
+ public final static String SYSTEM_PROPERTY_TRACK_REUSABLE_BUFFERS
+ = "com.fasterxml.jackson.core.util.BufferRecyclers.trackReusableBuffers";
+
+ /*
+ /**********************************************************
+ /* Life-cycle
+ /**********************************************************
+ */
+
+ /**
+ * Flag that indicates whether {@link BufferRecycler} instances should be tracked.
+ */
+ private final static ThreadLocalBufferManager _bufferRecyclerTracker;
+ static {
+ _bufferRecyclerTracker = "true".equals(System.getProperty(SYSTEM_PROPERTY_TRACK_REUSABLE_BUFFERS))
+ ? ThreadLocalBufferManager.instance()
+ : null;
+ }
+
/*
/**********************************************************
/* BufferRecyclers for parsers, generators
@@ -29,6 +55,9 @@ public class BufferRecyclers
final protected static ThreadLocal<SoftReference<BufferRecycler>> _recyclerRef
= new ThreadLocal<SoftReference<BufferRecycler>>();
+ /**
+ * Main accessor to call for accessing possibly recycled {@link BufferRecycler} instance.
+ */
public static BufferRecycler getBufferRecycler()
{
SoftReference<BufferRecycler> ref = _recyclerRef.get();
@@ -36,11 +65,36 @@ public class BufferRecyclers
if (br == null) {
br = new BufferRecycler();
- _recyclerRef.set(new SoftReference<BufferRecycler>(br));
+ if (_bufferRecyclerTracker != null) {
+ ref = _bufferRecyclerTracker.wrapAndTrack(br);
+ } else {
+ ref = new SoftReference<BufferRecycler>(br);
+ }
+ _recyclerRef.set(ref);
}
return br;
}
+ /**
+ * Specialized method that will release all recycled {@link BufferRecycler} if
+ * (and only if) recycler tracking has been enabled
+ * (see {@link #SYSTEM_PROPERTY_TRACK_REUSABLE_BUFFERS}).
+ * This method is usually called on shutdown of the container like Application Server
+ * to ensure that no references are reachable via {@link ThreadLocal}s as this may cause
+ * unintentional retention of sizable amounts of memory. It may also be called regularly
+ * if GC for some reason does not clear up {@link SoftReference}s aggressively enough.
+ *
+ * @return Number of buffers released, if tracking enabled (zero or more); -1 if tracking not enabled.
+ *
+ * @since 2.9.6
+ */
+ public static int releaseBuffers() {
+ if (_bufferRecyclerTracker != null) {
+ return _bufferRecyclerTracker.releaseBuffers();
+ }
+ return -1;
+ }
+
/*
/**********************************************************
/* JsonStringEncoder
=====================================
src/main/java/com/fasterxml/jackson/core/util/ThreadLocalBufferManager.java
=====================================
@@ -0,0 +1,116 @@
+package com.fasterxml.jackson.core.util;
+
+import java.lang.ref.ReferenceQueue;
+import java.lang.ref.SoftReference;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * For issue [jackson-core#400] We keep a separate Set of all SoftReferences to BufferRecyclers
+ * which are (also) referenced using `ThreadLocals`.
+ * We do this to be able to release them (dereference) in `releaseBuffers()` and `shutdown()`
+ * method to reduce heap consumption during hot reloading of services where otherwise
+ * {@link ClassLoader} would have dangling reference via {@link ThreadLocal}s.
+ * When gc clears a SoftReference, it puts it on a newly introduced referenceQueue.
+ * We use this queue to release the inactive SoftReferences from the Set.
+ *
+ * @since 2.9.6
+ */
+class ThreadLocalBufferManager
+{
+ /**
+ * A lock to make sure releaseBuffers is only executed by one thread at a time
+ * since it iterates over and modifies the allSoftBufRecyclers.
+ */
+ private final Object RELEASE_LOCK = new Object();
+
+ /**
+ * A set of all SoftReferences to all BufferRecyclers to be able to release them on shutdown.
+ * 'All' means the ones created by this class, in this classloader.
+ * There may be more from other classloaders.
+ * We use a HashSet to have quick O(1) add and remove operations.
+ *<p>
+ * NOTE: assumption is that {@link SoftReference} has its {@code equals()} and
+ * {@code hashCode()} implementations defined so that they use object identity, so
+ * we do not need to use something like {@link IdentityHashMap}
+ */
+ private final Map<SoftReference<BufferRecycler>,Boolean> _trackedRecyclers
+ = new ConcurrentHashMap<SoftReference<BufferRecycler>, Boolean>();
+
+ /**
+ * Queue where gc will put just-cleared SoftReferences, previously referencing BufferRecyclers.
+ * We use it to remove the cleared softRefs from the above set.
+ */
+ private final ReferenceQueue<BufferRecycler> _refQueue = new ReferenceQueue<BufferRecycler>();
+
+ /*
+ /**********************************************************
+ /* Public API
+ /**********************************************************
+ */
+
+ /**
+ * Returns the lazily initialized singleton instance
+ */
+ public static ThreadLocalBufferManager instance() {
+ return ThreadLocalBufferManagerHolder.manager;
+ }
+
+ /**
+ * Releases the buffers retained in ThreadLocals. To be called for instance on shutdown event of applications which make use of
+ * an environment like an appserver which stays alive and uses a thread pool that causes ThreadLocals created by the
+ * application to survive much longer than the application itself.
+ * It will clear all bufRecyclers from the SoftRefs and release all SoftRefs itself from our set.
+ */
+ public int releaseBuffers() {
+ synchronized (RELEASE_LOCK) {
+ int count = 0;
+ // does this need to be in sync block too? Looping over Map definitely has to but...
+ removeSoftRefsClearedByGc(); // make sure the refQueue is empty
+ for (SoftReference<BufferRecycler> ref : _trackedRecyclers.keySet()) {
+ ref.clear(); // possibly already cleared by gc, nothing happens in that case
+ ++count;
+ }
+ _trackedRecyclers.clear(); //release cleared SoftRefs
+ return count;
+ }
+ }
+
+ public SoftReference<BufferRecycler> wrapAndTrack(BufferRecycler br) {
+ SoftReference<BufferRecycler> newRef;
+ newRef = new SoftReference<BufferRecycler>(br, _refQueue);
+ // also retain softRef to br in a set to be able to release it on shutdown
+ _trackedRecyclers.put(newRef, true);
+ // gc may have cleared one or more SoftRefs, clean them up to avoid a memleak
+ removeSoftRefsClearedByGc();
+ return newRef;
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods
+ /**********************************************************
+ */
+
+ /**
+ * Remove cleared (inactive) SoftRefs from our set. Gc may have cleared one or more,
+ * and made them inactive. We minimize contention by keeping synchronized sections short:
+ * the poll/remove methods
+ */
+ private void removeSoftRefsClearedByGc() {
+ SoftReference<?> clearedSoftRef;
+ while ((clearedSoftRef = (SoftReference<?>) _refQueue.poll()) != null) {
+ // uses reference-equality, quick, and O(1) removal by HashSet
+ _trackedRecyclers.remove(clearedSoftRef);
+ }
+ }
+
+ /**
+ * ThreadLocalBufferManagerHolder uses the thread-safe initialize-on-demand, holder class idiom that implicitly
+ * incorporates lazy initialization by declaring a static variable within a static Holder inner class
+ */
+ private static final class ThreadLocalBufferManagerHolder {
+ static final ThreadLocalBufferManager manager = new ThreadLocalBufferManager();
+ }
+}
=====================================
src/test/java/com/fasterxml/jackson/core/BaseTest.java
=====================================
@@ -505,6 +505,14 @@ public abstract class BaseTest
/**********************************************************
*/
+ protected static String quote(String str) {
+ return '"'+str+'"';
+ }
+
+ protected static String aposToQuotes(String json) {
+ return json.replace("'", "\"");
+ }
+
protected byte[] encodeInUTF32BE(String input)
{
int len = input.length();
@@ -519,12 +527,13 @@ public abstract class BaseTest
return result;
}
- protected String quote(String str) {
- return '"'+str+'"';
- }
-
- protected String aposToQuotes(String json) {
- return json.replace("'", "\"");
+ // @since 2.9.7
+ protected static byte[] utf8Bytes(String str) {
+ try {
+ return str.getBytes("UTF-8");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
protected void fieldNameFor(StringBuilder sb, int index)
@@ -548,6 +557,16 @@ public abstract class BaseTest
}
}
+ // @since 2.9.7
+ protected JsonFactory sharedStreamFactory() {
+ return JSON_FACTORY;
+ }
+
+ // @since 2.9.7
+ protected JsonFactory newStreamFactory() {
+ return new JsonFactory();
+ }
+
protected String fieldNameFor(int index)
{
StringBuilder sb = new StringBuilder(16);
=====================================
src/test/java/com/fasterxml/jackson/core/VersionTest.java
=====================================
@@ -6,16 +6,14 @@ import static org.junit.Assert.*;
/**
* Unit tests for class {@link Version}.
*
- * @date 2017-08-01
- * @see Version
- *
**/
public class VersionTest{
@Test
public void testCompareToOne() {
Version version = Version.unknownVersion();
- Version versionTwo = new Version(0, (-263), (-1820), "");
+ Version versionTwo = new Version(0, (-263), (-1820), "",
+ "", "");
assertEquals(263, version.compareTo(versionTwo));
}
@@ -23,7 +21,8 @@ public class VersionTest{
@Test
public void testCompareToReturningZero() {
Version version = Version.unknownVersion();
- Version versionTwo = new Version(0, 0, 0, "");
+ Version versionTwo = new Version(0, 0, 0, "",
+ "", "");
assertEquals(0, version.compareTo(versionTwo));
}
@@ -39,7 +38,8 @@ public class VersionTest{
@Test
public void testCompareToTwo() {
Version version = Version.unknownVersion();
- Version versionTwo = new Version((-1), 0, 0, "0.0.0");
+ Version versionTwo = new Version((-1), 0, 0, "0.0.0",
+ "", "");
assertTrue(version.compareTo(versionTwo) > 0);
}
@@ -51,5 +51,4 @@ public class VersionTest{
assertTrue(version.compareTo(versionTwo) < 0);
}
-
-}
\ No newline at end of file
+}
=====================================
src/test/java/com/fasterxml/jackson/core/base64/Base64BinaryParsingTest.java
=====================================
@@ -4,6 +4,8 @@ import static org.junit.Assert.assertArrayEquals;
import java.io.*;
+import org.junit.Assert;
+
import com.fasterxml.jackson.core.*;
public class Base64BinaryParsingTest
@@ -53,6 +55,12 @@ public class Base64BinaryParsingTest
}
}
+ public void testWithEscapedPadding() throws IOException {
+ for (int mode : ALL_MODES) {
+ _testEscapedPadding(mode);
+ }
+ }
+
public void testInvalidTokenForBase64() throws IOException
{
for (int mode : ALL_MODES) {
@@ -109,7 +117,75 @@ public class Base64BinaryParsingTest
p.close();
}
}
-
+
+ public void testOkMissingPadding() throws IOException {
+ final byte[] DOC1 = new byte[] { (byte) 0xAD };
+ _testOkMissingPadding(DOC1, MODE_INPUT_STREAM);
+ _testOkMissingPadding(DOC1, MODE_INPUT_STREAM_THROTTLED);
+ _testOkMissingPadding(DOC1, MODE_READER);
+ _testOkMissingPadding(DOC1, MODE_DATA_INPUT);
+
+ final byte[] DOC2 = new byte[] { (byte) 0xAC, (byte) 0xDC };
+ _testOkMissingPadding(DOC2, MODE_INPUT_STREAM);
+ _testOkMissingPadding(DOC2, MODE_INPUT_STREAM_THROTTLED);
+ _testOkMissingPadding(DOC2, MODE_READER);
+ _testOkMissingPadding(DOC2, MODE_DATA_INPUT);
+ }
+
+ private void _testOkMissingPadding(byte[] input, int mode) throws IOException
+ {
+ final Base64Variant b64 = Base64Variants.MODIFIED_FOR_URL;
+ final String encoded = b64.encode(input, false);
+ JsonParser p = createParser(mode, quote(encoded));
+ // 1 byte -> 2 encoded chars; 2 bytes -> 3 encoded chars
+ assertEquals(input.length+1, encoded.length());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ byte[] actual = p.getBinaryValue(b64);
+ Assert.assertArrayEquals(input, actual);
+ p.close();
+ }
+
+ public void testFailDueToMissingPadding() throws IOException {
+ final String DOC1 = quote("fQ"); // 1 bytes, no padding
+ _testFailDueToMissingPadding(DOC1, MODE_INPUT_STREAM);
+ _testFailDueToMissingPadding(DOC1, MODE_INPUT_STREAM_THROTTLED);
+ _testFailDueToMissingPadding(DOC1, MODE_READER);
+ _testFailDueToMissingPadding(DOC1, MODE_DATA_INPUT);
+
+ final String DOC2 = quote("A/A"); // 2 bytes, no padding
+ _testFailDueToMissingPadding(DOC2, MODE_INPUT_STREAM);
+ _testFailDueToMissingPadding(DOC2, MODE_INPUT_STREAM_THROTTLED);
+ _testFailDueToMissingPadding(DOC2, MODE_READER);
+ _testFailDueToMissingPadding(DOC2, MODE_DATA_INPUT);
+ }
+
+ private void _testFailDueToMissingPadding(String doc, int mode) throws IOException {
+ final String EXP_EXCEPTION_MATCH = "Unexpected end of base64-encoded String: base64 variant 'MIME' expects padding";
+
+ // First, without getting text value first:
+ JsonParser p = createParser(mode, doc);
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ try {
+ /*byte[] b =*/ p.getBinaryValue(Base64Variants.MIME);
+ fail("Should not pass");
+ } catch (JsonParseException e) {
+ verifyException(e, EXP_EXCEPTION_MATCH);
+ }
+ p.close();
+
+ // second, access String first
+ p = createParser(mode, doc);
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ /*String str =*/ p.getText();
+ try {
+ /*byte[] b =*/ p.getBinaryValue(Base64Variants.MIME);
+ fail("Should not pass");
+ } catch (JsonParseException e) {
+ verifyException(e, EXP_EXCEPTION_MATCH);
+ }
+ p.close();
+ }
+
/*
/**********************************************************
/* Test helper methods
@@ -129,7 +205,7 @@ public class Base64BinaryParsingTest
Base64Variants.PEM
};
- JsonFactory jsonFactory = new JsonFactory();
+ JsonFactory jsonFactory = sharedStreamFactory();
final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
StringWriter chars = null;
for (int len : LENS) {
@@ -192,7 +268,7 @@ public class Base64BinaryParsingTest
139000
};
- JsonFactory jsonFactory = new JsonFactory();
+ JsonFactory jsonFactory = sharedStreamFactory();
final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
StringWriter chars = null;
@@ -271,7 +347,7 @@ public class Base64BinaryParsingTest
private void _testInArray(int mode) throws IOException
{
- JsonFactory f = new JsonFactory();
+ JsonFactory f = sharedStreamFactory();
final int entryCount = 7;
@@ -333,4 +409,53 @@ public class Base64BinaryParsingTest
}
p.close();
}
+
+ private void _testEscapedPadding(int mode) throws IOException
+ {
+ // Input: "Test!" -> "VGVzdCE="
+ final String DOC = quote("VGVzdCE\\u003d");
+
+ // 06-Sep-2018, tatu: actually one more, test escaping of padding
+ JsonParser p = createParser(mode, DOC);
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("Test!", new String(p.getBinaryValue(), "US-ASCII"));
+ if (mode != MODE_DATA_INPUT) {
+ assertNull(p.nextToken());
+ }
+ p.close();
+
+ // also, try out alternate access method
+ p = createParser(mode, DOC);
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("Test!", new String(_readBinary(p), "US-ASCII"));
+ if (mode != MODE_DATA_INPUT) {
+ assertNull(p.nextToken());
+ }
+ p.close();
+
+ // and then different padding; "X" -> "WA=="
+ final String DOC2 = quote("WA\\u003D\\u003D");
+ p = createParser(mode, DOC2);
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("X", new String(p.getBinaryValue(), "US-ASCII"));
+ if (mode != MODE_DATA_INPUT) {
+ assertNull(p.nextToken());
+ }
+ p.close();
+
+ p = createParser(mode, DOC2);
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("X", new String(_readBinary(p), "US-ASCII"));
+ if (mode != MODE_DATA_INPUT) {
+ assertNull(p.nextToken());
+ }
+ p.close();
+ }
+
+ private byte[] _readBinary(JsonParser p) throws IOException
+ {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ p.readBinaryValue(bytes);
+ return bytes.toByteArray();
+ }
}
=====================================
src/test/java/com/fasterxml/jackson/core/json/JsonFactoryTest.java
=====================================
@@ -1,12 +1,97 @@
package com.fasterxml.jackson.core.json;
import java.io.*;
+import java.util.Iterator;
import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.type.ResolvedType;
+import com.fasterxml.jackson.core.type.TypeReference;
public class JsonFactoryTest
extends com.fasterxml.jackson.core.BaseTest
{
+ private static class BogusCodec extends ObjectCodec {
+ @Override
+ public Version version() { return null; }
+
+ @Override
+ public <T> T readValue(JsonParser p, Class<T> valueType) throws IOException {
+ return null;
+ }
+
+ @Override
+ public <T> T readValue(JsonParser p, TypeReference<?> valueTypeRef) throws IOException {
+ return null;
+ }
+
+ @Override
+ public <T> T readValue(JsonParser p, ResolvedType valueType) throws IOException {
+ return null;
+ }
+
+ @Override
+ public <T> Iterator<T> readValues(JsonParser p, Class<T> valueType) throws IOException {
+ return null;
+ }
+
+ @Override
+ public <T> Iterator<T> readValues(JsonParser p, TypeReference<?> valueTypeRef) throws IOException {
+ return null;
+ }
+
+ @Override
+ public <T> Iterator<T> readValues(JsonParser p, ResolvedType valueType) throws IOException {
+ return null;
+ }
+
+ @Override
+ public void writeValue(JsonGenerator gen, Object value) throws IOException {
+ }
+
+ @Override
+ public <T extends TreeNode> T readTree(JsonParser p) throws IOException {
+ return null;
+ }
+
+ @Override
+ public void writeTree(JsonGenerator gen, TreeNode tree) throws IOException {
+ }
+
+ @Override
+ public TreeNode createObjectNode() {
+ return null;
+ }
+
+ @Override
+ public TreeNode createArrayNode() {
+ return null;
+ }
+
+ @Override
+ public JsonParser treeAsTokens(TreeNode n) {
+ return null;
+ }
+
+ @Override
+ public <T> T treeToValue(TreeNode n, Class<T> valueType) throws JsonProcessingException {
+ return null;
+ }
+ }
+
+ // for testing [core#460]
+ @SuppressWarnings("serial")
+ static class CustomFactory extends JsonFactory {
+ public CustomFactory(JsonFactory f, ObjectCodec codec) {
+ super(f, codec);
+ }
+ }
+
+ /*
+ /**********************************************************************
+ /* Test methods
+ /**********************************************************************
+ */
+
public void testGeneratorFeatures() throws Exception
{
JsonFactory f = new JsonFactory();
@@ -115,21 +200,32 @@ public class JsonFactoryTest
{
JsonFactory jf = new JsonFactory();
// first, verify defaults
+ assertNull(jf.getCodec());
assertTrue(jf.isEnabled(JsonFactory.Feature.INTERN_FIELD_NAMES));
assertFalse(jf.isEnabled(JsonParser.Feature.ALLOW_COMMENTS));
assertFalse(jf.isEnabled(JsonGenerator.Feature.ESCAPE_NON_ASCII));
+
+ // then change, verify that changes "stick"
jf.disable(JsonFactory.Feature.INTERN_FIELD_NAMES);
jf.enable(JsonParser.Feature.ALLOW_COMMENTS);
jf.enable(JsonGenerator.Feature.ESCAPE_NON_ASCII);
- // then change, verify that changes "stick"
+ ObjectCodec codec = new BogusCodec();
+ jf.setCodec(codec);
+
assertFalse(jf.isEnabled(JsonFactory.Feature.INTERN_FIELD_NAMES));
assertTrue(jf.isEnabled(JsonParser.Feature.ALLOW_COMMENTS));
assertTrue(jf.isEnabled(JsonGenerator.Feature.ESCAPE_NON_ASCII));
+ assertSame(codec, jf.getCodec());
JsonFactory jf2 = jf.copy();
assertFalse(jf2.isEnabled(JsonFactory.Feature.INTERN_FIELD_NAMES));
- assertTrue(jf.isEnabled(JsonParser.Feature.ALLOW_COMMENTS));
- assertTrue(jf.isEnabled(JsonGenerator.Feature.ESCAPE_NON_ASCII));
+ assertTrue(jf2.isEnabled(JsonParser.Feature.ALLOW_COMMENTS));
+ assertTrue(jf2.isEnabled(JsonGenerator.Feature.ESCAPE_NON_ASCII));
+ // 16-May-2018, tatu: But! Note that despited [core#460], this should NOT copy it back
+ assertNull(jf2.getCodec());
+
+ // However: real copy constructor SHOULD copy it
+ JsonFactory jf3 = new CustomFactory(jf, codec);
+ assertSame(codec, jf3.getCodec());
}
}
-
=====================================
src/test/java/com/fasterxml/jackson/core/json/async/AsyncConcurrencyTest.java
=====================================
@@ -0,0 +1,163 @@
+package com.fasterxml.jackson.core.json.async;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.async.AsyncTestBase;
+import com.fasterxml.jackson.core.testsupport.AsyncReaderWrapper;
+
+public class AsyncConcurrencyTest extends AsyncTestBase
+{
+ private final static JsonFactory JSON_F = new JsonFactory();
+ static {
+ // To make it pass, try:
+// JSON_F.disable(JsonFactory.Feature.USE_THREAD_LOCAL_FOR_BUFFER_RECYCLING);
+ }
+
+ private final static String TEXT1 = "Short";
+ private final static String TEXT2 = "Some longer text";
+ private final static String TEXT3 = "and yet more";
+ private final static String TEXT4 = "... Longest yet although not superbly long still (see 'apos'?)";
+
+ private final static byte[] JSON_DOC = utf8Bytes(String.format(
+ "[\"%s\", \"%s\",\n\"%s\",\"%s\" ]", TEXT1, TEXT2, TEXT3, TEXT4));
+
+ private class WorkUnit {
+
+ private int stage = 0;
+
+ private AsyncReaderWrapper parser;
+
+ private boolean errored = false;
+
+ public boolean process() throws Exception {
+ // short-cut through if this instance has already failed
+ if (errored) {
+ return false;
+ }
+ try {
+ switch (stage++) {
+ case 0:
+ parser = asyncForBytes(JSON_F, 100, JSON_DOC, 0);
+ break;
+ case 1:
+ _assert(JsonToken.START_ARRAY);
+ break;
+ case 2:
+ _assert(TEXT1);
+ break;
+ case 3:
+ _assert(TEXT2);
+ break;
+ case 4:
+ _assert(TEXT3);
+ break;
+ case 5:
+ _assert(TEXT4);
+ break;
+ case 6:
+ _assert(JsonToken.END_ARRAY);
+ break;
+ default:
+ /*
+ if (parser.nextToken() != null) {
+ throw new IOException("Unexpected token at "+stage+"; expected `null`, got "+parser.currentToken());
+ }
+ */
+ parser.close();
+ parser = null;
+ stage = 0;
+ return true;
+ }
+ } catch (Exception e) {
+ errored = true;
+ throw e;
+ }
+ return false;
+ }
+
+ private void _assert(String exp) throws IOException {
+ _assert(JsonToken.VALUE_STRING);
+ String str = parser.currentText();
+ if (!exp.equals(str)) {
+ throw new IOException("Unexpected VALUE_STRING: expected '"+exp+"', got '"+str+"'");
+ }
+ }
+
+ private void _assert(JsonToken exp) throws IOException {
+ JsonToken t = parser.nextToken();
+ if (t != exp) {
+ throw new IOException("Unexpected token at "+stage+"; expected "+exp+", got "+t);
+ }
+ }
+ }
+
+ // [jackson-core#476]
+ public void testConcurrentAsync() throws Exception
+ {
+ final int MAX_ROUNDS = 30;
+ for (int i = 0; i < MAX_ROUNDS; ++i) {
+ _testConcurrentAsyncOnce(i, MAX_ROUNDS);
+ }
+ }
+
+ private void _testConcurrentAsyncOnce(final int round, final int maxRounds) throws Exception
+ {
+ final int numThreads = 3;
+ final ExecutorService executor = Executors.newFixedThreadPool(numThreads);
+ final AtomicInteger errorCount = new AtomicInteger(0);
+ final AtomicInteger completedCount = new AtomicInteger(0);
+ final AtomicReference<String> errorRef = new AtomicReference<String>();
+
+ // First, add a few shared work units
+ final ArrayBlockingQueue<WorkUnit> q = new ArrayBlockingQueue<WorkUnit>(20);
+ for (int i = 0; i < 7; ++i) {
+ q.add(new WorkUnit());
+ }
+
+ // then invoke swarm of workers on it...
+
+ final int REP_COUNT = 99000;
+ ArrayList<Future<?>> futures = new ArrayList<Future<?>>();
+ for (int i = 0; i < REP_COUNT; i++) {
+ Callable<Void> c = new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ WorkUnit w = q.take();
+ try {
+ if (w.process()) {
+ completedCount.incrementAndGet();
+ }
+ } catch (Throwable t) {
+ if (errorCount.getAndIncrement() == 0) {
+ errorRef.set(t.toString());
+ }
+ } finally {
+ q.add(w);
+ }
+ return null;
+ }
+
+ };
+ futures.add(executor.submit(c));
+ }
+ executor.shutdown();
+ executor.awaitTermination(5, TimeUnit.SECONDS);
+ int count = errorCount.get();
+
+ if (count > 0) {
+ fail("Expected no problems (round "+round+"/"+maxRounds
+ +"); got "+count+", first with: "+errorRef.get());
+ }
+ final int EXP_COMPL = ((REP_COUNT + 7) / 8);
+ int compl = completedCount.get();
+
+ if (compl < (EXP_COMPL-10) || compl > EXP_COMPL) {
+ fail("Expected about "+EXP_COMPL+" completed rounds, got: "+compl);
+ }
+ }
+}
=====================================
src/test/java/com/fasterxml/jackson/core/json/async/AsyncTokenFilterTest.java
=====================================
@@ -0,0 +1,70 @@
+package com.fasterxml.jackson.core.json.async;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.async.AsyncTestBase;
+import com.fasterxml.jackson.core.filter.FilteringParserDelegate;
+import com.fasterxml.jackson.core.filter.TokenFilter;
+
+// [core#462], [core#463]
+public class AsyncTokenFilterTest extends AsyncTestBase
+{
+ private final JsonFactory JSON_F = new JsonFactory();
+
+ private final static String INPUT_STRING = aposToQuotes("{'a': 1, 'b': [2, {'c': 3}]}");
+ private final static byte[] INPUT_BYTES = utf8Bytes(INPUT_STRING);
+ private final static TokenFilter TOKEN_FILTER = new TokenFilter() {
+ @Override
+ public TokenFilter includeProperty(String name) {
+ return name == "a" ? TokenFilter.INCLUDE_ALL : null;
+ }
+ };
+
+ private final static JsonToken[] EXPECTED_TOKENS = new JsonToken[]{
+ JsonToken.START_OBJECT,
+ JsonToken.FIELD_NAME,
+ JsonToken.VALUE_NUMBER_INT,
+ JsonToken.END_OBJECT
+ };
+
+ // Passes if (but only if) all content is actually available
+ public void testFilteredNonBlockingParserAllContent() throws IOException
+ {
+ NonBlockingJsonParser nonBlockingParser = (NonBlockingJsonParser) JSON_F.createNonBlockingByteArrayParser();
+ FilteringParserDelegate filteredParser = new FilteringParserDelegate(nonBlockingParser,
+ TOKEN_FILTER, true, true);
+ nonBlockingParser.feedInput(INPUT_BYTES, 0, INPUT_BYTES.length);
+ int expectedIdx = 0;
+ while (expectedIdx < EXPECTED_TOKENS.length) {
+ // grab next token
+ JsonToken actual = filteredParser.nextToken();
+
+ // make sure it's the right one and mark it as seen.
+ assertToken(EXPECTED_TOKENS[expectedIdx], actual);
+ expectedIdx++;
+ }
+
+ filteredParser.close();
+ nonBlockingParser.close();
+ }
+
+ public void testSkipChildrenFailOnSplit() throws IOException
+ {
+ NonBlockingJsonParser nbParser = (NonBlockingJsonParser) JSON_F.createNonBlockingByteArrayParser();
+ FilteringParserDelegate filteredParser = new FilteringParserDelegate(nbParser,
+ TOKEN_FILTER, true, true);
+ nbParser.feedInput(INPUT_BYTES, 0, 5);
+
+ assertToken(JsonToken.START_OBJECT, nbParser.nextToken());
+ try {
+ nbParser.skipChildren();
+ fail("Should not pass!");
+ } catch (JsonParseException e) {
+ verifyException(e, "not enough content available");
+ verifyException(e, "skipChildren()");
+ }
+ nbParser.close();
+ filteredParser.close();
+ }
+}
=====================================
src/test/java/com/fasterxml/jackson/core/read/NumberOverflowTest.java
=====================================
@@ -0,0 +1,131 @@
+package com.fasterxml.jackson.core.read;
+
+import java.math.BigInteger;
+
+import com.fasterxml.jackson.core.*;
+
+public class NumberOverflowTest
+ extends com.fasterxml.jackson.core.BaseTest
+{
+ private final JsonFactory FACTORY = new JsonFactory();
+
+ // NOTE: this should be long enough to trigger perf problems
+ private final static int BIG_NUM_LEN = 199999;
+ private final static String BIG_POS_INTEGER;
+ static {
+ StringBuilder sb = new StringBuilder(BIG_NUM_LEN);
+ for (int i = 0; i < BIG_NUM_LEN; ++i) {
+ sb.append('9');
+ }
+ BIG_POS_INTEGER = sb.toString();
+ }
+
+ private final static String BIG_POS_DOC = "["+BIG_POS_INTEGER+"]";
+ private final static String BIG_NEG_DOC = "[ -"+BIG_POS_INTEGER+"]";
+
+ public void testSimpleLongOverflow() throws Exception
+ {
+ BigInteger below = BigInteger.valueOf(Long.MIN_VALUE);
+ below = below.subtract(BigInteger.ONE);
+ BigInteger above = BigInteger.valueOf(Long.MAX_VALUE);
+ above = above.add(BigInteger.ONE);
+
+ String DOC_BELOW = below.toString() + " ";
+ String DOC_ABOVE = below.toString() + " ";
+
+ for (int mode : ALL_MODES) {
+ JsonParser p = createParser(FACTORY, mode, DOC_BELOW);
+ p.nextToken();
+ try {
+ long x = p.getLongValue();
+ fail("Expected an exception for underflow (input "+p.getText()+"): instead, got long value: "+x);
+ } catch (JsonParseException e) {
+ verifyException(e, "out of range of long");
+ }
+ p.close();
+
+ p = createParser(mode, DOC_ABOVE);
+ p.nextToken();
+ try {
+ long x = p.getLongValue();
+ fail("Expected an exception for underflow (input "+p.getText()+"): instead, got long value: "+x);
+ } catch (JsonParseException e) {
+ verifyException(e, "out of range of long");
+ }
+ p.close();
+ }
+ }
+
+ // Note: only 4 cardinal types; `short`, `byte` and `char` use same code paths
+ // Note: due to [jackson-core#493], we'll skip DataInput-backed parser
+
+ // [jackson-core#488]
+ public void testMaliciousLongOverflow() throws Exception
+ {
+ for (int mode : ALL_STREAMING_MODES) {
+ for (String doc : new String[] { BIG_POS_DOC, BIG_NEG_DOC }) {
+ JsonParser p = createParser(mode, doc);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ try {
+ p.getLongValue();
+ fail("Should not pass");
+ } catch (JsonParseException e) {
+ verifyException(e, "out of range of long");
+ verifyException(e, "Integer with "+BIG_NUM_LEN+" digits");
+ }
+ p.close();
+ }
+ }
+ }
+
+ // [jackson-core#488]
+ public void testMaliciousIntOverflow() throws Exception
+ {
+ for (int mode : ALL_STREAMING_MODES) {
+ for (String doc : new String[] { BIG_POS_DOC, BIG_NEG_DOC }) {
+ JsonParser p = createParser(mode, doc);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ try {
+ p.getIntValue();
+ fail("Should not pass");
+ } catch (JsonParseException e) {
+ verifyException(e, "out of range of int");
+ verifyException(e, "Integer with "+BIG_NUM_LEN+" digits");
+ }
+ p.close();
+ }
+ }
+ }
+
+ // [jackson-core#488]
+ public void testMaliciousBigIntToDouble() throws Exception
+ {
+ for (int mode : ALL_STREAMING_MODES) {
+ final String doc = BIG_POS_DOC;
+ JsonParser p = createParser(mode, doc);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ double d = p.getDoubleValue();
+ assertEquals(Double.valueOf(BIG_POS_INTEGER), d);
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ p.close();
+ }
+ }
+
+ // [jackson-core#488]
+ public void testMaliciousBigIntToFloat() throws Exception
+ {
+ for (int mode : ALL_STREAMING_MODES) {
+ final String doc = BIG_POS_DOC;
+ JsonParser p = createParser(mode, doc);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ float f = p.getFloatValue();
+ assertEquals(Float.valueOf(BIG_POS_INTEGER), f);
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ p.close();
+ }
+ }
+}
=====================================
src/test/java/com/fasterxml/jackson/core/read/NumberParsingTest.java
=====================================
@@ -108,7 +108,7 @@ public class NumberParsingTest
_testSimpleLong(MODE_READER);
_testSimpleLong(MODE_DATA_INPUT);
}
-
+
private void _testSimpleLong(int mode) throws Exception
{
long EXP_L = 12345678907L;
@@ -309,39 +309,6 @@ public class NumberParsingTest
p.close();
}
- public void testLongOverflow() throws Exception
- {
- BigInteger below = BigInteger.valueOf(Long.MIN_VALUE);
- below = below.subtract(BigInteger.ONE);
- BigInteger above = BigInteger.valueOf(Long.MAX_VALUE);
- above = above.add(BigInteger.ONE);
-
- String DOC_BELOW = below.toString() + " ";
- String DOC_ABOVE = below.toString() + " ";
-
- for (int mode : ALL_MODES) {
- JsonParser p = createParser(mode, DOC_BELOW);
- p.nextToken();
- try {
- long x = p.getLongValue();
- fail("Expected an exception for underflow (input "+p.getText()+"): instead, got long value: "+x);
- } catch (JsonParseException e) {
- verifyException(e, "out of range of long");
- }
- p.close();
-
- p = createParser(mode, DOC_ABOVE);
- p.nextToken();
- try {
- long x = p.getLongValue();
- fail("Expected an exception for underflow (input "+p.getText()+"): instead, got long value: "+x);
- } catch (JsonParseException e) {
- verifyException(e, "out of range of long");
- }
- p.close();
-
- }
- }
/**
* Method that tries to test that number parsing works in cases where
View it on GitLab: https://salsa.debian.org/java-team/jackson-core/compare/6969329227fdea42a8772a8c5a6e8937848cf274...9156302804250bdda1322629f87827ad35ddeceb
--
View it on GitLab: https://salsa.debian.org/java-team/jackson-core/compare/6969329227fdea42a8772a8c5a6e8937848cf274...9156302804250bdda1322629f87827ad35ddeceb
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-java-commits/attachments/20181228/2c950971/attachment.html>
More information about the pkg-java-commits
mailing list