Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(java): fix long type name meta string encoding #1837

Merged
merged 2 commits into from
Sep 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/specification/java_serialization_spec.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,11 +155,11 @@ Meta header is a 64 bits number value encoded in little endian order.
fields info in meta for deserializing compatible fields.
- Package name encoding(omitted when class is registered):
- encoding algorithm: `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL`
- Header: `6 bits size | 2 bits encoding flags`. The `6 bits size: 0~63` will be used to indicate size `0~62`,
the value `63` the size need more byte to read, the encoding will encode `size - 62` as a varint next.
- Header: `6 bits size | 2 bits encoding flags`. The `6 bits size: 0~63` will be used to indicate size `0~63`,
the value `63` the size need more byte to read, the encoding will encode `size - 63` as a varint next.
- Class name encoding(omitted when class is registered):
- encoding algorithm: `UTF8/LOWER_UPPER_DIGIT_SPECIAL/FIRST_TO_LOWER_SPECIAL/ALL_TO_LOWER_SPECIAL`
- header: `6 bits size | 2 bits encoding flags`. The `6 bits size: 0~63` will be used to indicate size `1~64`,
- header: `6 bits size | 2 bits encoding flags`. The `6 bits size: 0~63` will be used to indicate size `0~63`,
the value `63` the size need more byte to read, the encoding will encode `size - 63` as a varint next.
- Field info:
- header(8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import static org.apache.fury.meta.ClassDef.COMPRESSION_FLAG;
import static org.apache.fury.meta.ClassDef.SIZE_TWO_BYTES_FLAG;
import static org.apache.fury.meta.ClassDefEncoder.BIG_NAME_THRESHOLD;
import static org.apache.fury.meta.Encoders.fieldNameEncodings;
import static org.apache.fury.meta.Encoders.pkgEncodings;
import static org.apache.fury.meta.Encoders.typeNameEncodings;
Expand Down Expand Up @@ -128,34 +129,31 @@ private static String readPkgName(MemoryBuffer buffer) {
// - Package name encoding(omitted when class is registered):
// - encoding algorithm: `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL`
// - Header: `6 bits size | 2 bits encoding flags`.
// The `6 bits size: 0~63` will be used to indicate size `0~62`,
// the value `63` the size need more byte to read, the encoding will encode `size - 62` as
// The `6 bits size: 0~63` will be used to indicate size `0~63`,
// the value `63` the size need more byte to read, the encoding will encode `size - 63` as
// a varint next.
int header = buffer.readByte() & 0xff;
int encodingFlags = header & 0b11;
Encoding encoding = pkgEncodings[encodingFlags];
return readName(Encoders.PACKAGE_DECODER, buffer, header, encoding, 62);
return readName(Encoders.PACKAGE_DECODER, buffer, pkgEncodings);
}

private static String readTypeName(MemoryBuffer buffer) {
// - Class name encoding(omitted when class is registered):
// - encoding algorithm:
// `UTF8/LOWER_UPPER_DIGIT_SPECIAL/FIRST_TO_LOWER_SPECIAL/ALL_TO_LOWER_SPECIAL`
// - header: `6 bits size | 2 bits encoding flags`.
// The `6 bits size: 0~63` will be used to indicate size `1~64`,
// The `6 bits size: 0~63` will be used to indicate size `0~63`,
// the value `63` the size need more byte to read, the encoding will encode `size - 63` as
// a varint next.
int header = buffer.readByte() & 0xff;
int encodingFlags = header & 0b11;
Encoding encoding = typeNameEncodings[encodingFlags];
return readName(Encoders.TYPE_NAME_DECODER, buffer, header, encoding, 63);
return readName(Encoders.TYPE_NAME_DECODER, buffer, typeNameEncodings);
}

private static String readName(
MetaStringDecoder decoder, MemoryBuffer buffer, int header, Encoding encoding, int max) {
MetaStringDecoder decoder, MemoryBuffer buffer, Encoding[] encodings) {
int header = buffer.readByte() & 0xff;
int encodingFlags = header & 0b11;
Encoding encoding = encodings[encodingFlags];
int size = header >> 2;
if (size == max) {
size = buffer.readVarUint32Small7() + max;
if (size == BIG_NAME_THRESHOLD) {
size = buffer.readVarUint32Small7() + BIG_NAME_THRESHOLD;
}
return decoder.decode(buffer.readBytes(size), encoding);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -277,8 +277,7 @@ private static void writePkgName(MemoryBuffer buffer, String pkg) {
// a varint next.
MetaString pkgMetaString = Encoders.encodePackage(pkg);
byte[] encoded = pkgMetaString.getBytes();
int pkgHeader = (encoded.length << 2) | pkgEncodingsList.indexOf(pkgMetaString.getEncoding());
writeName(buffer, encoded, pkgHeader, 62);
writeName(buffer, encoded, pkgEncodingsList.indexOf(pkgMetaString.getEncoding()));
}

private static void writeTypeName(MemoryBuffer buffer, String typeName) {
Expand All @@ -291,17 +290,19 @@ private static void writeTypeName(MemoryBuffer buffer, String typeName) {
// a varint next.
MetaString metaString = Encoders.encodeTypeName(typeName);
byte[] encoded = metaString.getBytes();
int header = (encoded.length << 2) | typeNameEncodingsList.indexOf(metaString.getEncoding());
writeName(buffer, encoded, header, 63);
writeName(buffer, encoded, typeNameEncodingsList.indexOf(metaString.getEncoding()));
}

private static void writeName(MemoryBuffer buffer, byte[] encoded, int header, int max) {
boolean bigSize = encoded.length > max;
static final int BIG_NAME_THRESHOLD = 0b111111;

private static void writeName(MemoryBuffer buffer, byte[] encoded, int encoding) {
boolean bigSize = encoded.length >= BIG_NAME_THRESHOLD;
if (bigSize) {
header |= 0b11111100;
buffer.writeVarUint32Small7(header);
buffer.writeVarUint32Small7(encoded.length - max);
int header = (BIG_NAME_THRESHOLD << 2) | encoding;
buffer.writeByte(header);
buffer.writeVarUint32Small7(encoded.length - BIG_NAME_THRESHOLD);
} else {
int header = (encoded.length << 2) | encoding;
buffer.writeByte(header);
}
buffer.writeBytes(encoded);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static org.apache.fury.meta.ClassDefEncoder.buildFieldsInfo;
import static org.apache.fury.meta.ClassDefEncoder.getClassFields;

import java.io.Serializable;
import java.util.List;
import lombok.Data;
import org.apache.fury.Fury;
Expand Down Expand Up @@ -78,7 +79,35 @@ public void testEmptySubClassSerializer() {
ClassDef classDef1 =
ClassDef.readClassDef(
fury.getClassResolver(), MemoryBuffer.fromByteArray(classDef.getEncoded()));

Assert.assertEquals(classDef, classDef1);
}

@Test
public void testBigClassNameObject() {
Fury fury = Fury.builder().withMetaShare(true).build();
ClassDef classDef =
ClassDef.buildClassDef(
fury,
TestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLength
.InnerClassTestLengthInnerClassTestLengthInnerClassTestLength.class);
ClassDef classDef1 =
ClassDef.readClassDef(
fury.getClassResolver(), MemoryBuffer.fromByteArray(classDef.getEncoded()));
Assert.assertEquals(classDef1, classDef);
}

@Data
public static
class TestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLength
implements Serializable {
private String name;
private InnerClassTestLengthInnerClassTestLengthInnerClassTestLength innerClassTestLength;

@Data
public static class InnerClassTestLengthInnerClassTestLengthInnerClassTestLength
implements Serializable {
private static final long serialVersionUID = -867612757789099089L;
private Long itemId;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.apache.fury.config.CompatibleMode;
import org.apache.fury.config.FuryBuilder;
import org.apache.fury.config.Language;
import org.apache.fury.meta.ClassDefEncoderTest;
import org.apache.fury.reflect.ReflectionUtils;
import org.apache.fury.resolver.MetaContext;
import org.apache.fury.serializer.collection.UnmodifiableSerializersTest;
Expand Down Expand Up @@ -657,4 +658,19 @@ void testEmptySubClass(boolean referenceTracking, boolean compressNumber, boolea
Assert.assertEquals(o.getClass(), o1.getClass());
Assert.assertTrue(ReflectionUtils.objectFieldsEquals(o, o1));
}

@Test
public void testBigClassNameObject() {
Fury fury =
builder()
.withRefTracking(true)
.withCompatibleMode(CompatibleMode.COMPATIBLE)
.withScopedMetaShare(false)
.build();
Object o =
new ClassDefEncoderTest
.TestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLengthTestClassLength
.InnerClassTestLengthInnerClassTestLengthInnerClassTestLength();
serDeCheck(fury, o);
}
}
Loading