Disable knowhere logging for embedded Milvus and some other tweaks (#16496)

/kind enhancement

issue: #15711
Signed-off-by: Yuchen Gao <yuchen.gao@zilliz.com>

Makefile

@@ -129,7 +129,7 @@ milvus: build-cpp print-build-info
 embd-milvus: build-cpp-embd print-build-info
 	@echo "Building **Embedded** Milvus ..."
 	@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build \
-		-ldflags="-X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
+		-ldflags="-r /tmp/milvus/lib/ -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
 		-buildmode=c-shared -o $(INSTALL_PATH)/embd-milvus.so $(PWD)/pkg/embedded/embedded.go 1>/dev/null
 
 build-go: milvus

cmd/milvus/milvus.go

@@ -189,20 +189,26 @@ func printUsage(w io.Writer, f *flag.Flag) {
 }
 
 // create runtime folder
-func createRuntimeDir() string {
+func createRuntimeDir(sType string) string {
+	var writer io.Writer
+	if sType == typeutil.EmbeddedRole {
+		writer = io.Discard
+	} else {
+		writer = os.Stderr
+	}
 	runtimeDir := "/run/milvus"
 	if runtime.GOOS == "windows" {
 		runtimeDir = "run"
 		if err := makeRuntimeDir(runtimeDir); err != nil {
-			fmt.Fprintf(os.Stderr, "Create runtime directory at %s failed\n", runtimeDir)
+			fmt.Fprintf(writer, "Create runtime directory at %s failed\n", runtimeDir)
 			os.Exit(-1)
 		}
 	} else {
 		if err := makeRuntimeDir(runtimeDir); err != nil {
-			fmt.Fprintf(os.Stderr, "Set runtime dir at %s failed, set it to /tmp/milvus directory\n", runtimeDir)
+			fmt.Fprintf(writer, "Set runtime dir at %s failed, set it to /tmp/milvus directory\n", runtimeDir)
 			runtimeDir = "/tmp/milvus"
 			if err = makeRuntimeDir(runtimeDir); err != nil {
-				fmt.Fprintf(os.Stderr, "Create runtime directory at %s failed\n", runtimeDir)
+				fmt.Fprintf(writer, "Create runtime directory at %s failed\n", runtimeDir)
 				os.Exit(-1)
 			}
 		}
@@ -311,7 +317,7 @@ func RunMilvus(args []string) {
 		params.SetLogger(0)
 	}
 
-	runtimeDir := createRuntimeDir()
+	runtimeDir := createRuntimeDir(serverType)
 	filename := getPidFileName(serverType, svrAlias)
 	switch command {
 	case "run":

internal/core/src/config/CMakeLists.txt

@@ -14,6 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+option( EMBEDDED_MILVUS "Enable embedded Milvus" OFF )
+
+if ( EMBEDDED_MILVUS )
+    add_compile_definitions( EMBEDDED_MILVUS )
+endif()
+
 set(CONFIG_SRC
         ConfigKnowhere.cpp
         )

internal/core/src/config/ConfigKnowhere.cpp

@@ -35,6 +35,18 @@ KnowhereInitImpl() {
         knowhere::KnowhereConfig::SetStatisticsLevel(0);
         el::Configurations el_conf;
         el_conf.setGlobally(el::ConfigurationType::Enabled, std::to_string(false));
+#if defined(EMBEDDED_MILVUS)
+        el::Configurations defaultConf;
+        defaultConf.setToDefault();
+        // Disable all logs for embedded milvus.
+        defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "false");
+        defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "false");
+        defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "false");
+        defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "false");
+        defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "false");
+        defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false");
+        el::Loggers::reconfigureLogger("default", defaultConf);
+#endif
     };
 
     std::call_once(init_knowhere_once_, init);

internal/core/src/indexbuilder/VecIndexCreator.cpp

@@ -165,7 +165,8 @@ VecIndexCreator::BuildWithoutIds(const knowhere::DatasetPtr& dataset) {
         }
     }
     auto conf_adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type);
-    std::cout << "Konwhere BuildWithoutIds config_ is " << config_ << std::endl;
+    // TODO: Use easylogging instead, if you really need to keep this log.
+    // std::cout << "Konwhere BuildWithoutIds config_ is " << config_ << std::endl;
     AssertInfo(conf_adapter->CheckTrain(config_, index_mode), "something wrong in index parameters!");
 
     if (is_in_need_id_list(index_type)) {

internal/proxy/authentication_interceptor.go

@@ -63,7 +63,7 @@ func AuthenticationInterceptor(ctx context.Context) (context.Context, error) {
 		return nil, ErrMissingMetadata()
 	}
 	if globalMetaCache == nil {
-		return nil, ErrGlobalMetaCacheUninitialized()
+		return nil, ErrProxyNotReady()
 	}
 	// check:
 	//	1. if rpc call from a member (like index/query/data component)

internal/proxy/error.go

@@ -80,6 +80,6 @@ func ErrUnauthenticated() error {
 	return fmt.Errorf("unauthenticated: invalid credential")
 }
 
-func ErrGlobalMetaCacheUninitialized() error {
-	return fmt.Errorf("internal: globalMetaCache not initialized")
+func ErrProxyNotReady() error {
+	return fmt.Errorf("internal: Milvus Proxy is not ready yet. please wait")
 }

internal/util/etcd/etcd_server.go

@@ -45,6 +45,7 @@ func InitEtcdServer(etcdCfg *paramtable.EtcdConfig) error {
 			cfg.LogLevel = etcdCfg.EtcdLogLevel
 			e, err := embed.StartEtcd(cfg)
 			if err != nil {
+				log.Error("failed to init embedded Etcd server", zap.Error(err))
 				initError = err
 			}
 			etcdServer = e