fix crash when GPU search (#3074)

* fix crash when GPU search

Signed-off-by: shengjun.li <shengjun.li@zilliz.com>

CHANGELOG.md

@@ -8,6 +8,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#2890 Fix the index size caculation in cache
 -   \#2952 Fix the result merging of IVF_PQ IP
 -   \#2975 Fix config UT failed
+-   \#3012 If the cache is too small, queries using multiple GPUs will cause to crash
 
 ## Feature
 

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -588,6 +588,11 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) {
             bool gpu_cache_enable = false;
             STATUS_CHECK(server::Config::GetInstance().GetGpuResourceConfigCacheEnable(gpu_cache_enable));
 
+            /* CopyCpuToGpu() is an asynchronous method.
+             * It should be make sure that the CPU index is always valid.
+             * Therefore, we reserve its shared pointer.
+             */
+            index_reserve_ = index_;
             if (gpu_cache_enable) {
                 gpu_cache_mgr->Reserve(index_->Size());
                 index_ = knowhere::cloner::CopyCpuToGpu(index_, device_id, knowhere::Config());

core/src/db/engine/ExecutionEngineImpl.h

@@ -123,6 +123,9 @@ class ExecutionEngineImpl : public ExecutionEngine {
 
  protected:
     knowhere::VecIndexPtr index_ = nullptr;
+#ifdef MILVUS_GPU_VERSION
+    knowhere::VecIndexPtr index_reserve_ = nullptr;  // reserve the cpu index before copying it to gpu
+#endif
     std::string location_;
     int64_t dim_;
     EngineType index_type_;