This commit is contained in:
Looly 2024-12-08 18:17:41 +08:00
parent 15c1237ecc
commit 45df318b56
12 changed files with 149 additions and 51 deletions

View File

@ -312,6 +312,28 @@ public class CharUtil implements CharPool {
return SLASH == c || BACKSLASH == c; return SLASH == c || BACKSLASH == c;
} }
/**
* 是否为零宽字符
*
* @param c 字符
* @return 是否为零宽字符
*/
public static boolean isZeroWidthChar(final char c) {
switch (c) {
case '\u200B': // 零宽空格
case '\u200C': // 零宽非换行空格
case '\u200D': // 零宽连接符
case '\uFEFF': // 零宽无断空格
case '\u2060': // 零宽连字符
case '\u2063': // 零宽不连字符
case '\u2064': // 零宽连字符
case '\u2065': // 零宽不连字符
return true;
default:
return false;
}
}
/** /**
* 比较两个字符是否相同 * 比较两个字符是否相同
* *

View File

@ -73,6 +73,18 @@ public class JSONConfig implements Serializable {
*/ */
private NumberWriteMode numberWriteMode = NumberWriteMode.NORMAL; private NumberWriteMode numberWriteMode = NumberWriteMode.NORMAL;
/**
* 是否忽略零宽字符这些字符可能会导致解析安全问题这些字符包括
* <ul>
* <li>零宽空格{@code \u200B}</li>
* <li>零宽非换行空{@code \u200C}</li>
* <li>零宽连接符{@code \u200D}</li>
* <li>零宽无断空格{@code \uFEFF}</li>
* </ul>
* 如果此值为{@code false}则转义否则去除
*/
private boolean ignoreZeroWithChar = true;
/** /**
* 创建默认的配置项 * 创建默认的配置项
* *
@ -289,6 +301,36 @@ public class JSONConfig implements Serializable {
return this; return this;
} }
/**
* 是否忽略零宽字符这些字符可能会导致解析安全问题这些字符包括
* <ul>
* <li>零宽空格{@code \u200B}</li>
* <li>零宽非换行空{@code \u200C}</li>
* <li>零宽连接符{@code \u200D}</li>
* <li>零宽无断空格{@code \uFEFF}</li>
* </ul>
* @return 此值为{@code false}则转义否则去除
*/
public boolean isIgnoreZeroWithChar() {
return ignoreZeroWithChar;
}
/**
* 设置是否忽略零宽字符这些字符可能会导致解析安全问题这些字符包括
* <ul>
* <li>零宽空格{@code \u200B}</li>
* <li>零宽非换行空{@code \u200C}</li>
* <li>零宽连接符{@code \u200D}</li>
* <li>零宽无断空格{@code \uFEFF}</li>
* </ul>
* @param ignoreZeroWithChar 此值为{@code false}则转义否则去除
* @return this
*/
public JSONConfig setIgnoreZeroWithChar(final boolean ignoreZeroWithChar) {
this.ignoreZeroWithChar = ignoreZeroWithChar;
return this;
}
/** /**
* 重复key或重复对象处理方式<br> * 重复key或重复对象处理方式<br>
* 只针对{@link JSONObject}检查在put时key的重复情况 * 只针对{@link JSONObject}检查在put时key的重复情况

View File

@ -47,6 +47,9 @@ public class JSONTokener extends ReaderWrapper {
*/ */
public static final int EOF = 0; public static final int EOF = 0;
/**
* 当前字符
*/
private long character; private long character;
/** /**
* 是否结尾 End of stream * 是否结尾 End of stream
@ -68,6 +71,7 @@ public class JSONTokener extends ReaderWrapper {
* 是否使用前一个字符 * 是否使用前一个字符
*/ */
private boolean usePrevious; private boolean usePrevious;
private boolean ignoreZeroWithChar;
// ------------------------------------------------------------------------------------ Constructor start // ------------------------------------------------------------------------------------ Constructor start
@ -75,27 +79,30 @@ public class JSONTokener extends ReaderWrapper {
* 从InputStream中构建使用UTF-8编码 * 从InputStream中构建使用UTF-8编码
* *
* @param inputStream InputStream * @param inputStream InputStream
* @param ignoreZeroWithChar 是否忽略零宽字符
* @throws JSONException JSON异常包装IO异常 * @throws JSONException JSON异常包装IO异常
*/ */
public JSONTokener(final InputStream inputStream) throws JSONException { public JSONTokener(final InputStream inputStream, final boolean ignoreZeroWithChar) throws JSONException {
this(IoUtil.toUtf8Reader(inputStream)); this(IoUtil.toUtf8Reader(inputStream), ignoreZeroWithChar);
} }
/** /**
* 从字符串中构建 * 从字符串中构建
* *
* @param s JSON字符串 * @param s JSON字符串
* @param ignoreZeroWithChar 是否忽略零宽字符
*/ */
public JSONTokener(final CharSequence s) { public JSONTokener(final CharSequence s, final boolean ignoreZeroWithChar) {
this(new StringReader(Assert.notBlank(s).toString())); this(new StringReader(Assert.notBlank(s).toString()), ignoreZeroWithChar);
} }
/** /**
* 从Reader中构建 * 从Reader中构建
* *
* @param reader Reader * @param reader Reader
* @param ignoreZeroWithChar 是否忽略零宽字符
*/ */
public JSONTokener(final Reader reader) { public JSONTokener(final Reader reader, final boolean ignoreZeroWithChar) {
super(IoUtil.toMarkSupport(Assert.notNull(reader))); super(IoUtil.toMarkSupport(Assert.notNull(reader)));
this.eof = false; this.eof = false;
this.usePrevious = false; this.usePrevious = false;
@ -103,6 +110,7 @@ public class JSONTokener extends ReaderWrapper {
this.index = 0; this.index = 0;
this.character = 1; this.character = 1;
this.line = 1; this.line = 1;
this.ignoreZeroWithChar = ignoreZeroWithChar;
} }
// ------------------------------------------------------------------------------------ Constructor end // ------------------------------------------------------------------------------------ Constructor end
@ -160,35 +168,15 @@ public class JSONTokener extends ReaderWrapper {
* @throws JSONException JSON异常包装IO异常 * @throws JSONException JSON异常包装IO异常
*/ */
public char next() throws JSONException { public char next() throws JSONException {
int c; char c;
if (this.usePrevious) { while(true){
this.usePrevious = false; c = _next();
c = this.previous; if(this.ignoreZeroWithChar && CharUtil.isZeroWidthChar(c)){
} else { continue;
try {
c = read();
} catch (final IOException exception) {
throw new JSONException(exception);
} }
return c;
if (c <= EOF) { // End of stream
this.eof = true;
c = EOF;
} }
} }
this.index += 1;
if (this.previous == '\r') {
this.line += 1;
this.character = c == '\n' ? 0 : 1;
} else if (c == '\n') {
this.line += 1;
this.character = 0;
} else {
this.character += 1;
}
this.previous = (char) c;
return this.previous;
}
/** /**
* 获取上一个读取的字符如果没有读取过则返回'\0' * 获取上一个读取的字符如果没有读取过则返回'\0'
@ -291,8 +279,8 @@ public class JSONTokener extends ReaderWrapper {
/** /**
* 获取下一个冒号非冒号则抛出异常 * 获取下一个冒号非冒号则抛出异常
* *
* @throws JSONException 非冒号字符
* @return 冒号字符 * @return 冒号字符
* @throws JSONException 非冒号字符
*/ */
public char nextColon() throws JSONException { public char nextColon() throws JSONException {
final char c = nextClean(); final char c = nextClean();
@ -413,6 +401,43 @@ public class JSONTokener extends ReaderWrapper {
return " at " + this.index + " [character " + this.character + " line " + this.line + "]"; return " at " + this.index + " [character " + this.character + " line " + this.line + "]";
} }
/**
* 获得源字符串中的下一个字符
*
* @return 下一个字符, or 0 if past the end of the source string.
* @throws JSONException JSON异常包装IO异常
*/
private char _next() throws JSONException {
int c;
if (this.usePrevious) {
this.usePrevious = false;
c = this.previous;
} else {
try {
c = read();
} catch (final IOException exception) {
throw new JSONException(exception);
}
if (c <= EOF) { // End of stream
this.eof = true;
c = EOF;
}
}
this.index += 1;
if (this.previous == '\r') {
this.line += 1;
this.character = c == '\n' ? 0 : 1;
} else if (c == '\n') {
this.line += 1;
this.character = 0;
} else {
this.character += 1;
}
this.previous = (char) c;
return this.previous;
}
/** /**
* 获取反转义的字符 * 获取反转义的字符
* *

View File

@ -101,7 +101,8 @@ public class ArrayTypeAdapter implements MatcherJSONSerializer<Object>, MatcherJ
switch (bytes[0]) { switch (bytes[0]) {
case '{': case '{':
case '[': case '[':
return context.getFactory().ofParser(new JSONTokener(IoUtil.toStream(bytes))).parse(); return context.getFactory().ofParser(
new JSONTokener(IoUtil.toStream(bytes), context.config().isIgnoreZeroWithChar())).parse();
} }
} }

View File

@ -85,7 +85,7 @@ public class CharSequenceTypeAdapter implements MatcherJSONSerializer<CharSequen
} }
// 按照JSON字符串解析 // 按照JSON字符串解析
return context.getFactory().ofParser(new JSONTokener(jsonStr)).parse(); return context.getFactory().ofParser(new JSONTokener(jsonStr, context.config().isIgnoreZeroWithChar())).parse();
} }
@Override @Override

View File

@ -44,7 +44,7 @@ public class ResourceSerializer implements MatcherJSONSerializer<Resource> {
@Override @Override
public JSON serialize(final Resource bean, final JSONContext context) { public JSON serialize(final Resource bean, final JSONContext context) {
return context.getFactory().ofParser(new JSONTokener(bean.getStream())).parse(); return context.getFactory().ofParser(new JSONTokener(bean.getStream(), context.config().isIgnoreZeroWithChar())).parse();
} }
/** /**

View File

@ -53,9 +53,9 @@ public class TokenerSerializer implements MatcherJSONSerializer<Object> {
} else if (bean instanceof JSONParser) { } else if (bean instanceof JSONParser) {
return ((JSONParser) bean).parse(); return ((JSONParser) bean).parse();
} else if (bean instanceof Reader) { } else if (bean instanceof Reader) {
return mapFromTokener(new JSONTokener((Reader) bean), context.getFactory()); return mapFromTokener(new JSONTokener((Reader) bean, context.config().isIgnoreZeroWithChar()), context.getFactory());
} else if (bean instanceof InputStream) { } else if (bean instanceof InputStream) {
return mapFromTokener(new JSONTokener((InputStream) bean), context.getFactory()); return mapFromTokener(new JSONTokener((InputStream) bean, context.config().isIgnoreZeroWithChar()), context.getFactory());
} }
throw new IllegalArgumentException("Unsupported source: " + bean); throw new IllegalArgumentException("Unsupported source: " + bean);

View File

@ -49,7 +49,7 @@ public class XMLTokener extends JSONTokener {
* @param s A source string. * @param s A source string.
*/ */
public XMLTokener(final CharSequence s) { public XMLTokener(final CharSequence s) {
super(s); super(s, true);
} }
/** /**

View File

@ -34,7 +34,7 @@ public class JSONTokenerTest {
@Test @Test
void nextTest() { void nextTest() {
final JSONTokener jsonTokener = new JSONTokener("{\"ab\": \"abc\"}"); final JSONTokener jsonTokener = new JSONTokener("{\"ab\": \"abc\"}", true);
final char c = jsonTokener.nextTokenChar(); final char c = jsonTokener.nextTokenChar();
assertEquals('{', c); assertEquals('{', c);
assertEquals("ab", jsonTokener.nextString()); assertEquals("ab", jsonTokener.nextString());
@ -51,7 +51,7 @@ public class JSONTokenerTest {
*/ */
@Test @Test
void nextWithoutWrapperTest() { void nextWithoutWrapperTest() {
final JSONTokener jsonTokener = new JSONTokener("{ab: abc}"); final JSONTokener jsonTokener = new JSONTokener("{ab: abc}", true);
final char c = jsonTokener.nextTokenChar(); final char c = jsonTokener.nextTokenChar();
assertEquals('{', c); assertEquals('{', c);
assertEquals("ab", jsonTokener.nextString()); assertEquals("ab", jsonTokener.nextString());

View File

@ -3,15 +3,23 @@ package org.dromara.hutool.json.reader;
import org.dromara.hutool.core.io.resource.ResourceUtil; import org.dromara.hutool.core.io.resource.ResourceUtil;
import org.dromara.hutool.core.util.CharsetUtil; import org.dromara.hutool.core.util.CharsetUtil;
import org.dromara.hutool.json.JSON; import org.dromara.hutool.json.JSON;
import org.dromara.hutool.json.JSONConfig;
import org.dromara.hutool.json.JSONUtil; import org.dromara.hutool.json.JSONUtil;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
public class Issue3808Test { public class Issue3808Test {
@Test @Test
void parseTest() { void parseEscapeZeroWithCharTest() {
final String str = ResourceUtil.readStr("issue3808.json", CharsetUtil.UTF_8); final String str = ResourceUtil.readStr("issue3808.json", CharsetUtil.UTF_8);
final JSON parse = JSONUtil.parse(str); final JSON parse = JSONUtil.parse(str, JSONConfig.of().setIgnoreZeroWithChar(false));
Assertions.assertNotNull(parse); Assertions.assertEquals("{\"recommend_text\":\"✅宁波,\\u200c一座历史悠久的文化名城\\n你好\",\"\\u200c一\":\"aaa\"}", parse.toString());
}
@Test
void parseIgnoreZeroWithCharTest() {
final String str = ResourceUtil.readStr("issue3808.json", CharsetUtil.UTF_8);
final JSON parse = JSONUtil.parse(str, JSONConfig.of().setIgnoreZeroWithChar(true));
Assertions.assertEquals("{\"recommend_text\":\"✅宁波,一座历史悠久的文化名城\\n你好\",\",一\":\"aaa\"}", parse.toString());
} }
} }

View File

@ -24,7 +24,7 @@ public class JSONParserTest {
@Test @Test
void parseTest() { void parseTest() {
final String jsonStr = " {\"a\": 1} "; final String jsonStr = " {\"a\": 1} ";
final JSONParser jsonParser = JSONParser.of(new JSONTokener(jsonStr), JSONFactory.getInstance()); final JSONParser jsonParser = JSONParser.of(new JSONTokener(jsonStr, true), JSONFactory.getInstance());
final JSON parse = jsonParser.parse(); final JSON parse = jsonParser.parse();
Assertions.assertEquals("{\"a\":1}", parse.toString()); Assertions.assertEquals("{\"a\":1}", parse.toString());
} }
@ -34,14 +34,14 @@ public class JSONParserTest {
final String jsonStr = "{\"a\": 1}"; final String jsonStr = "{\"a\": 1}";
final JSONObject jsonObject = JSONUtil.ofObj(); final JSONObject jsonObject = JSONUtil.ofObj();
JSONParser.of(new JSONTokener(jsonStr), JSONFactory.getInstance()).parseTo(jsonObject); JSONParser.of(new JSONTokener(jsonStr, true), JSONFactory.getInstance()).parseTo(jsonObject);
Assertions.assertEquals("{\"a\":1}", jsonObject.toString()); Assertions.assertEquals("{\"a\":1}", jsonObject.toString());
} }
@Test @Test
void parseToArrayTest() { void parseToArrayTest() {
final String jsonStr = "[{},2,3]"; final String jsonStr = "[{},2,3]";
final JSONParser jsonParser = JSONParser.of(new JSONTokener(jsonStr), JSONFactory.getInstance()); final JSONParser jsonParser = JSONParser.of(new JSONTokener(jsonStr, true), JSONFactory.getInstance());
final JSONArray jsonArray = new JSONArray(); final JSONArray jsonArray = new JSONArray();
jsonParser.parseTo(jsonArray); jsonParser.parseTo(jsonArray);

View File

@ -1 +1 @@
{"recommend_text":"✅宁波,‌一座历史悠久的文化名城"} {"recommend_text":"✅宁波,‌一座历史悠久的文化名城\n你好", : "aaa"}