From a3d4cbdd4c081582169fe0a2c59cf4c4f9635fa5 Mon Sep 17 00:00:00 2001
From: Bingyi Sun <sunbingyi1992@gmail.com>
Date: Mon, 22 Nov 2021 10:05:14 +0800
Subject: [PATCH] Convert c array to a golang slice (#12066)

issue: #11974
Signed-off-by: sunby <bingyi.sun@zilliz.com>

Co-authored-by: sunby <bingyi.sun@zilliz.com>
---
 internal/storage/payload.go      | 14 ++++++++++----
 internal/storage/payload_test.go | 23 +++++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/internal/storage/payload.go b/internal/storage/payload.go
index b60bc686b8..b7f6ef2910 100644
--- a/internal/storage/payload.go
+++ b/internal/storage/payload.go
@@ -22,6 +22,7 @@ import "C"
 import (
 	"errors"
 	"fmt"
+	"reflect"
 	"unsafe"
 
 	"github.com/milvus-io/milvus/internal/log"
@@ -307,14 +308,19 @@ func (w *PayloadWriter) FinishPayloadWriter() error {
 
 func (w *PayloadWriter) GetPayloadBufferFromWriter() ([]byte, error) {
 	cb := C.GetPayloadBufferFromWriter(w.payloadWriterPtr)
-	pointer := unsafe.Pointer(cb.data)
+	pointer := uintptr(unsafe.Pointer(cb.data))
 	length := int(cb.length)
 	if length <= 0 {
 		return nil, errors.New("empty buffer")
 	}
-	// refer to: https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
-	slice := (*[1 << 28]byte)(pointer)[:length:length]
-	return slice, nil
+
+	var data []byte
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&data))
+	sh.Data = pointer
+	sh.Len = length
+	sh.Cap = length
+
+	return data, nil
 }
 
 func (w *PayloadWriter) GetPayloadLengthFromWriter() (int, error) {
diff --git a/internal/storage/payload_test.go b/internal/storage/payload_test.go
index ce33162855..a9f50b2849 100644
--- a/internal/storage/payload_test.go
+++ b/internal/storage/payload_test.go
@@ -891,4 +891,27 @@ func TestPayload_ReaderandWriter(t *testing.T) {
 		_, _, err = r.GetFloatVectorFromPayload()
 		assert.NotNil(t, err)
 	})
+
+	t.Run("TestWriteLargeSizeData", func(t *testing.T) {
+		size := 1 << 29 // 512M
+		var vec []float32
+		for i := 0; i < size/4; i++ {
+			vec = append(vec, 1)
+		}
+
+		w, err := NewPayloadWriter(schemapb.DataType_FloatVector)
+		assert.Nil(t, err)
+
+		err = w.AddFloatVectorToPayload(vec, 128)
+		assert.Nil(t, err)
+
+		err = w.FinishPayloadWriter()
+		assert.Nil(t, err)
+
+		_, err = w.GetPayloadBufferFromWriter()
+		assert.Nil(t, err)
+
+		err = w.ReleasePayloadWriter()
+		assert.Nil(t, err)
+	})
 }