C轮融资|Fury：一个基于JIT动态编译的高性能多语言原生序列化框架( 四 ) 创投圈

package mainimport \"code.alipay.com/ray-project/fury/go/fury\"import \"fmt\"func main() { type SomeClass struct { F1 *SomeClass F2 map[string
string F3 map[string
stringfury_ := fury.NewFury(true) if err := fury_.RegisterTagType(\"example.SomeClass\" SomeClass{); err != nil { panic(err)value :=SomeClass{F2: map[string
string{\"k1\": \"v1\" \"k2\": \"v2\" value.F3 = value.F2 value.F1 = value bytes err := fury_.Marshal(value) if err != nil {var newValue interface{ // bytes can be data serialized by other languages. if err := fury_.Unmarshal(bytesnewValue); err != nil { panic(err)fmt.Println(newValue) Zero-Copy序列化
对于大规模数据传输场景，内存拷贝有时会成为整个系统的瓶颈。为此各种语言和框架做了大量优化，比如Java提供了NIO能力，避免了内存在用户态和内核态之间的来回拷贝；Kafka使用Java的NIO来实现零拷贝；Python Pickle5提供了Out-Of-Band Buffer[7
序列化能力来避免额外拷贝。
对于高性能跨语言数据传输，序列化框架也需要能够支持Zero-Copy ，避免数据Buffer的额外拷贝。下面是一个Fury序列化多个基本类型数组组成的对象树的示例，分别对应到Java基本类型数组、Python Numpy数组、Golang 基本类型slice 。对于ByteBuffer零拷贝，在本文的性能测试部分也给出了部分介绍。
Java序列化示例
Java序列化
import io.fury.*;import io.fury.serializers.BufferObject;import io.fury.memory.MemoryBuffer;import java.util.*;import java.util.stream.Collectors;public class ZeroCopyExample { // mvn exec:java -Dexec.mainClass=\"io.fury.examples.ZeroCopyExample\" public static void main(String[
args) { // Fury应该在多个对象序列化之间复用，不要每次创建新的Fury实例 Fury fury = Fury.builder() .withLanguage(Language.JAVA) .withClassRegistrationRequired(false) .build(); ListObjectlist = Arrays.asList(\"str\" new byte[1000
new int[100
new double[100
); CollectionBufferObjectbufferObjects = new ArrayList(); byte[
bytes = fury.serialize(list e -!bufferObjects.add(e)); ListMemoryBufferbuffers = bufferObjects.stream().map(BufferObject::toBuffer).collect(Collectors.toList()); System.out.println(fury.deserialize(bytes buffers));跨语言序列化：
import io.fury.*;import io.fury.serializers.BufferObject;import io.fury.memory.MemoryBuffer;import java.util.*;import java.util.stream.Collectors;public class ZeroCopyExample { // mvn exec:java -Dexec.mainClass=\"io.fury.examples.ZeroCopyExample\" public static void main(String[
args) { Fury fury = Fury.builder().withLanguage(Language.XLANG).build(); ListObjectlist = Arrays.asList(\"str\" new byte[1000
new int[100
new double[100
); CollectionBufferObjectbufferObjects = new ArrayList(); byte[
bytes = fury.serialize(list e -!bufferObjects.add(e)); // bytes can be data serialized by other languages. ListMemoryBufferbuffers = bufferObjects.stream().map(BufferObject::toBuffer).collect(Collectors.toList()); System.out.println(fury.deserialize(bytes buffers));Python序列化示例
import arrayimport pyfuryimport numpy as npif __name__ == \"__main__\": fury_ = pyfury.Fury() list_ = [\"str\" bytes(bytearray(1000)) array.array(\"i\" range(100)) np.full(100 0.0 dtype=np.double)
serialized_objects = [
data = https://mparticle.uc.cn/api/fury_.serialize(list_ buffer_callback=serialized_objects.append) buffers = [o.to_buffer() for o in serialized_objects
# bytes can be data serialized by other languages. print(fury_.deserialize(data buffers=buffers)) Golang序列化示例
package mainimport \"code.alipay.com/ray-project/fury/go/fury\"import \"fmt\"func main() { fury := fury.NewFury(true) // Golang版本暂不支持其他基本类型slice的zero-copy list := [
interface{{\"str\" make([
byte 1000) buf := fury.NewByteBuffer(nil) var serializedObjects [
fury.SerializedObject fury.Serialize(buf list func(o fury.SerializedObject) bool { serializedObjects = append(serializedObjects o) return false ) var newList [