refact(server): enlarge bytes write limit & remove param big when encode/decode string id length

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java

@@ -55,14 +55,18 @@
     private final AbstractSerializer serializer;
 
     public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes) {
+        this(graph, capacity, avgEntryBytes, Runtime.getRuntime().availableProcessors() * 2);
+    }
+
+    public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes, int segments) {
         // NOTE: capacity unit is bytes, the super capacity expect elements size
         super(capacity);
-        long capacityInBytes = capacity * (avgEntryBytes + 64L);
+        long capacityInBytes = Math.max(capacity, segments) * (avgEntryBytes + 64L);
         if (capacityInBytes <= 0L) {
             capacityInBytes = 1L;
         }
         this.graph = graph;
-        this.cache = this.builder().capacity(capacityInBytes).build();
+        this.cache = this.builder().capacity(capacityInBytes).segmentCount(segments).build();
         this.serializer = new BinarySerializer();
     }
 
@@ -162,19 +166,18 @@
 
         @Override
         public Id deserialize(ByteBuffer input) {
-            return BytesBuffer.wrap(input).readId(true);
+            return BytesBuffer.wrap(input).readId();
         }
 
         @Override
         public void serialize(Id id, ByteBuffer output) {
-            BytesBuffer.wrap(output).writeId(id, true);
+            BytesBuffer.wrap(output).writeId(id);
         }
 
         @Override
         public int serializedSize(Id id) {
             // NOTE: return size must be == actual bytes to write
-            return BytesBuffer.allocate(id.length() + 2)
-                              .writeId(id, true).position();
+            return BytesBuffer.allocate(id.length() + 2).writeId(id).position();
         }
     }
 

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java

@@ -1312,9 +1312,9 @@ private Id readId(byte[] value) {
         }
 
         private byte[] writeIds(Collection<Id> ids) {
-            E.checkState(ids.size() <= BytesBuffer.UINT16_MAX,
+            E.checkState(ids.size() <= BytesBuffer.MAX_PROPERTIES,
                          "The number of properties of vertex/edge label " +
-                         "can't exceed '%s'", BytesBuffer.UINT16_MAX);
+                         "can't exceed '%s'", BytesBuffer.MAX_PROPERTIES);
             int size = 2;
             for (Id id : ids) {
                 size += (1 + id.length());

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java

@@ -52,20 +52,25 @@ public final class BytesBuffer extends OutputStream {
     public static final int FLOAT_LEN = Float.BYTES;
     public static final int DOUBLE_LEN = Double.BYTES;
     public static final int BLOB_LEN = 4;
+    public static final int BYTES_LEN = 5;
 
     public static final int UINT8_MAX = ((byte) -1) & 0xff;
     public static final int UINT16_MAX = ((short) -1) & 0xffff;
     public static final long UINT32_MAX = (-1) & 0xffffffffL;
+    public static final int INT32_MAX = Integer.MAX_VALUE;
 
+    // TODO: support user-defined configuration
     // NOTE: +1 to let code 0 represent length 1
-    public static final int ID_LEN_MASK = 0x7f;
-    public static final int ID_LEN_MAX = 0x7f + 1; // 128
-    public static final int BIG_ID_LEN_MAX = 0x7fff + 1; // 32768
+    public static final int ID_LEN_MAX = 0x3fff + 1; // 16384
 
     public static final byte STRING_ENDING_BYTE = (byte) 0x00;
     public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff;
-    public static final int STRING_LEN_MAX = UINT16_MAX;
+
+    // TODO: support user-defined configuration
     public static final long BLOB_LEN_MAX = 1 * Bytes.GB;
+    public static final long BYTES_LEN_MAX = INT32_MAX;
+
+    public static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX;
 
     // The value must be in range [8, ID_LEN_MAX]
     public static final int INDEX_HASH_ID_THRESHOLD = 32;
@@ -288,10 +293,10 @@ public double readDouble() {
     }
 
     public BytesBuffer writeBytes(byte[] bytes) {
-        E.checkArgument(bytes.length <= UINT16_MAX,
+        E.checkArgument(bytes.length <= BYTES_LEN_MAX,
                         "The max length of bytes is %s, but got %s",
-                        UINT16_MAX, bytes.length);
-        require(SHORT_LEN + bytes.length);
+                        BYTES_LEN_MAX, bytes.length);
+        require(BYTES_LEN + bytes.length);
         this.writeVInt(bytes.length);
         this.write(bytes);
         return this;
@@ -629,10 +634,6 @@ public Object readProperty(DataType dataType) {
     }
 
     public BytesBuffer writeId(Id id) {
-        return this.writeId(id, false);
-    }
-
-    public BytesBuffer writeId(Id id, boolean big) {
         switch (id.type()) {
             case LONG:
                 // Number Id
@@ -656,20 +657,18 @@ public BytesBuffer writeId(Id id, boolean big) {
                 bytes = id.asBytes();
                 int len = bytes.length;
                 E.checkArgument(len > 0, "Can't write empty id");
-                if (!big) {
-                    E.checkArgument(len <= ID_LEN_MAX,
-                                    "Id max length is %s, but got %s {%s}",
-                                    ID_LEN_MAX, len, id);
-                    len -= 1; // mapping [1, 128] to [0, 127]
+                E.checkArgument(len <= ID_LEN_MAX,
+                                "Big id max length is %s, but got %s {%s}",
+                                ID_LEN_MAX, len, id);
+                len -= 1; // mapping [1, 16384] to [0, 16383]
+                if (len <= 0x3f) {
+                    // If length is <= 63, use a single byte with the highest bit set to 1
                     this.writeUInt8(len | 0x80);
                 } else {
-                    E.checkArgument(len <= BIG_ID_LEN_MAX,
-                                    "Big id max length is %s, but got %s {%s}",
-                                    BIG_ID_LEN_MAX, len, id);
-                    len -= 1;
                     int high = len >> 8;
                     int low = len & 0xff;
-                    this.writeUInt8(high | 0x80);
+                    // Write high 8 bits with highest two bits set to 11
+                    this.writeUInt8(high | 0xc0);
                     this.writeUInt8(low);
                 }
                 this.write(bytes);
@@ -679,10 +678,6 @@ public BytesBuffer writeId(Id id, boolean big) {
     }
 
     public Id readId() {
-        return this.readId(false);
-    }
-
-    public Id readId(boolean big) {
         byte b = this.read();
         boolean number = (b & 0x80) == 0;
         if (number) {
@@ -698,13 +693,13 @@ public Id readId(boolean big) {
             }
         } else {
             // String Id
-            int len = b & ID_LEN_MASK;
-            if (big) {
+            int len = b & 0x3f; // Take the lowest 6 bits as part of the length
+            if ((b & 0x40) != 0) { // If the 7th bit is set, length information spans 2 bytes
                 int high = len << 8;
                 int low = this.readUInt8();
                 len = high + low;
             }
-            len += 1; // restore [0, 127] to [1, 128]
+            len += 1; // restore [0, 16383] to [1, 16384]
             byte[] id = this.read(len);
             return IdGenerator.of(id, IdType.STRING);
         }

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java

@@ -147,9 +147,9 @@ public void assignId() {
 
         if (this.fresh()) {
             int len = this.id.length();
-            E.checkArgument(len <= BytesBuffer.BIG_ID_LEN_MAX,
+            E.checkArgument(len <= BytesBuffer.ID_LEN_MAX,
                             "The max length of edge id is %s, but got %s {%s}",
-                            BytesBuffer.BIG_ID_LEN_MAX, len, this.id);
+                            BytesBuffer.ID_LEN_MAX, len, this.id);
         }
     }
 

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java

@@ -55,7 +55,6 @@ public abstract class HugeElement implements Element, GraphType, Idfiable, Compa
 
     private static final MutableIntObjectMap<HugeProperty<?>> EMPTY_MAP =
             CollectionFactory.newIntObjectMap();
-    private static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX;
 
     private final HugeGraph graph;
     private MutableIntObjectMap<HugeProperty<?>> properties;
@@ -279,7 +278,7 @@ public <V> HugeProperty<?> setProperty(HugeProperty<V> prop) {
         PropertyKey pkey = prop.propertyKey();
 
         E.checkArgument(this.properties.containsKey(intFromId(pkey.id())) ||
-                        this.properties.size() < MAX_PROPERTIES,
+                        this.properties.size() < BytesBuffer.MAX_PROPERTIES,
                         "Exceeded the maximum number of properties");
         return this.properties.put(intFromId(pkey.id()), prop);
     }

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java

@@ -725,7 +725,7 @@ private void checkPropertySize(String property, String propertyName) {
     }
 
     private void checkPropertySize(int propertyLength, String propertyName) {
-        long propertyLimit = BytesBuffer.STRING_LEN_MAX;
+        long propertyLimit = BytesBuffer.MAX_PROPERTIES;
         HugeGraph graph = this.scheduler().graph();
         if (propertyName.equals(P.INPUT)) {
             propertyLimit = graph.option(CoreOptions.TASK_INPUT_SIZE_LIMIT);

hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java

@@ -75,7 +75,6 @@
 import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
 import org.junit.Assume;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import com.google.common.collect.ImmutableList;
@@ -470,12 +469,13 @@ public void testAddEdgeWithLargeSortkey() {
         Vertex book = graph.addVertex(T.label, "book", "name", "Test-Book-1");
 
         Assert.assertThrows(IllegalArgumentException.class, () -> {
-            final int LEN = BytesBuffer.BIG_ID_LEN_MAX;
+            final int LEN = BytesBuffer.ID_LEN_MAX;
             String largeTime = "{large-time}" + new String(new byte[LEN]);
             james.addEdge("write", book, "time", largeTime);
             graph.tx().commit();
         }, e -> {
-            Assert.assertContains("The max length of edge id is 32768",
+            Assert.assertContains(String.format("The max length of edge id is %s",
+                                                BytesBuffer.ID_LEN_MAX),
                                   e.getMessage());
         });
     }

hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java

@@ -44,6 +44,7 @@
 import org.apache.hugegraph.backend.query.Condition;
 import org.apache.hugegraph.backend.query.ConditionQuery;
 import org.apache.hugegraph.backend.query.Query;
+import org.apache.hugegraph.backend.serializer.BytesBuffer;
 import org.apache.hugegraph.backend.store.BackendTable;
 import org.apache.hugegraph.backend.store.Shard;
 import org.apache.hugegraph.backend.tx.GraphTransaction;
@@ -1039,10 +1040,10 @@ public void testAddVertexWithCustomizeStringIdStrategyWithoutValidId() {
                             "name", "marko", "age", 18, "city", "Beijing");
         });
 
-        // Expect id length <= 128
+        // Expect id length <= BytesBuffer.ID_LEN_MAX
         Assert.assertThrows(IllegalArgumentException.class, () -> {
-            String largeId = new String(new byte[128]) + ".";
-            assert largeId.length() == 129;
+            String largeId = new String(new byte[BytesBuffer.ID_LEN_MAX]) + ".";
+            assert largeId.length() == BytesBuffer.ID_LEN_MAX + 1;
             graph.addVertex(T.label, "programmer", T.id, largeId,
                             "name", "marko", "age", 18, "city", "Beijing");
         });

hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java

@@ -139,6 +139,7 @@ protected void checkNotInCache(Cache<Id, Object> cache, Id id) {
     public static class OffheapCacheTest extends CacheTest {
 
         private static final long ENTRY_SIZE = 40L;
+        private static final int SEGMENTS = 4;
         private final HugeGraph graph = Mockito.mock(HugeGraph.class);
 
         @Override
@@ -148,7 +149,7 @@ protected Cache<Id, Object> newCache() {
 
         @Override
         protected Cache<Id, Object> newCache(long capacity) {
-            return new OffheapCache(this.graph(), capacity, ENTRY_SIZE);
+            return new OffheapCache(this.graph(), capacity, ENTRY_SIZE, SEGMENTS);
         }
 
         @Override
@@ -325,7 +326,7 @@ public void testUpdateAndGetWithDataType() {
 
     @Test
     public void testUpdateAndGetWithSameSizeAndCapacity() {
-        int limit = 40;
+        int limit = 1000;
         Cache<Id, Object> cache = newCache(limit);
         Map<Id, Object> map = new LimitMap(limit);
 
@@ -451,7 +452,7 @@ public void testSize() {
 
     @Test
     public void testSizeWithReachCapacity() {
-        int limit = 20;
+        int limit = 1000;
         Cache<Id, Object> cache = newCache(limit);
         Map<Id, Object> map = new LimitMap(limit);
 

hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java

@@ -154,4 +154,22 @@ private byte[] genBytes(String string) {
         }
         return bytes;
     }
+
+    /**
+     * Converts a byte array to a hexadecimal string.
+     *
+     * @param bytes the byte array to convert
+     * @return the hexadecimal string representation of the byte array
+     */
+    private String bytesToHex(byte[] bytes) {
+        StringBuilder hexString = new StringBuilder();
+        for (byte b : bytes) {
+            String hex = Integer.toHexString(0xFF & b);
+            if (hex.length() == 1) {
+                hexString.append('0'); // pad with leading zero if needed
+            }
+            hexString.append(hex);
+        }
+        return hexString.toString();
+    }
 }