Merge remote-tracking branch 'upstream/master' into branch-0.5.0

Former-commit-id: 65cb3f61f900a694157897b93a20bd80b3670ad8

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 Please mark all change in change log and use the ticket from JIRA.
 
+# Milvus 0.5.1 (TODO)
+
+## Bug
+## Improvement
+- \#64 - Improvement dump function in scheduler
+
+## Feature
+## Task
+
 # Milvus 0.5.0 (2019-10-21)
 
 ## Bug
@@ -57,8 +66,9 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-648 - Improve unittest
 - MS-655 - Upgrade SPTAG
 - \#42 - Put union of index_build_device and search resources to gpu_pool
+- \#67 - Avoid linking targets multiple times in cmake
 
-## New Feature
+## Feature
 - MS-614 - Preload table at startup
 - MS-627 - Integrate new index: IVFSQHybrid
 - MS-631 - IVFSQ8H Index support
@@ -202,7 +212,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-576 - Scheduler refactor
 - MS-592 - Change showtables stream transport to unary
 
-## New Feature
+## Feature
 - MS-343 - Implement ResourceMgr
 - MS-338 - NewAPI: refine code to support CreateIndex
 - MS-339 - NewAPI: refine code to support DropIndex
@@ -263,7 +273,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-332 - Set grpc and thrift server run concurrently
 - MS-352 - Add hybrid index
 
-## New Feature
+## Feature
 - MS-180 - Add new mem manager
 - MS-195 - Add nlist and use_blas_threshold conf
 - MS-137 - Integrate knowhere
@@ -297,7 +307,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-126 - Add more error code
 - MS-128 - Change default db path
 
-## New Feature
+## Feature
 
 - MS-57 - Implement index load/search pipeline
 - MS-56 - Add version information when server is started
@@ -342,7 +352,7 @@ Please mark all change in change log and use the ticket from JIRA.
 
 - MS-20 - Clean Code Part 1
 
-## New Feature
+## Feature
 
 - MS-5 - Implement Auto Archive Feature
 - MS-6 - Implement SDK interface part 1

NOTICE.md

@@ -26,4 +26,5 @@
 | gperftools    | [BSD 3-Clause](https://github.com/gperftools/gperftools/blob/master/COPYING)                                                 |
 | grpc          | [Apache 2.0](https://github.com/grpc/grpc/blob/master/LICENSE)                                                   |
 | EASYLOGGINGPP | [MIT](https://github.com/zuhd-org/easyloggingpp/blob/master/LICENSEhttps://github.com/zuhd-org/easyloggingpp/blob/master/LICENSE)                                                          |
+| Json          | [MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT)                                                          |
 

core/build.sh

@@ -93,7 +93,7 @@ cd ${BUILD_OUTPUT_DIR}
 
 # remove make cache since build.sh -l use default variables
 # force update the variables each time
-make rebuild_cache
+make rebuild_cache > /dev/null 2>&1
 
 CMAKE_CMD="cmake \
 -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
@@ -139,13 +139,12 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
 #    fi
 #    echo "clang-tidy check passed!"
 else
-    # compile and build
-    make -j 4 || exit 1
 
     # strip binary symbol
     if [[ ${BUILD_TYPE} != "Debug" ]]; then
         strip src/milvus_server
     fi
 
-    make install || exit 1
+    # compile and build
+    make -j 8 install || exit 1
 fi

core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -331,10 +331,9 @@ macro(build_arrow)
                         CMAKE_ARGS
                         ${ARROW_CMAKE_ARGS}
                         BUILD_COMMAND
-                        ${MAKE}
-                        ${MAKE_BUILD_ARGS}
+                        ""
                         INSTALL_COMMAND
-                        ${MAKE} install
+                        ${MAKE} ${MAKE_BUILD_ARGS} install
                         BUILD_BYPRODUCTS
                         "${ARROW_STATIC_LIB}"
                         )
@@ -365,10 +364,9 @@ macro(build_arrow)
                 CMAKE_ARGS
                 ${ARROW_CMAKE_ARGS}
                 BUILD_COMMAND
-                ${MAKE}
-                ${MAKE_BUILD_ARGS}
+                ""
                 INSTALL_COMMAND
-                ${MAKE} install
+                ${MAKE} ${MAKE_BUILD_ARGS} install
                 BUILD_BYPRODUCTS
                 "${ARROW_STATIC_LIB}"
                 )

core/src/scheduler/TaskTable.cpp

@@ -52,19 +52,13 @@ ToString(TaskTableItemState state) {
     }
 }
 
-std::string
-ToString(const TaskTimestamp& timestamp) {
-    std::stringstream ss;
-    ss << "<start=" << timestamp.start;
-    ss << ", load=" << timestamp.load;
-    ss << ", loaded=" << timestamp.loaded;
-    ss << ", execute=" << timestamp.execute;
-    ss << ", executed=" << timestamp.executed;
-    ss << ", move=" << timestamp.move;
-    ss << ", moved=" << timestamp.moved;
-    ss << ", finish=" << timestamp.finish;
-    ss << ">";
-    return ss.str();
+json
+TaskTimestamp::Dump() {
+    json ret{
+        {"start", start},       {"load", load}, {"loaded", loaded}, {"execute", execute},
+        {"executed", executed}, {"move", move}, {"moved", moved},   {"finish", finish},
+    };
+    return ret;
 }
 
 bool
@@ -146,15 +140,15 @@ TaskTableItem::Moved() {
     return false;
 }
 
-std::string
+json
 TaskTableItem::Dump() {
-    std::stringstream ss;
-    ss << "<id=" << id;
-    ss << ", task=" << task;
-    ss << ", state=" << ToString(state);
-    ss << ", timestamp=" << ToString(timestamp);
-    ss << ">";
-    return ss.str();
+    json ret{
+        {"id", id},
+        {"task", (int64_t)task.get()},
+        {"state", ToString(state)},
+        {"timestamp", timestamp.Dump()},
+    };
+    return ret;
 }
 
 std::vector<uint64_t>
@@ -268,13 +262,13 @@ TaskTable::Get(uint64_t index) {
 ////        table_.erase(table_.begin(), iterator);
 //}
 
-std::string
+json
 TaskTable::Dump() {
-    std::stringstream ss;
+    json ret;
     for (auto& item : table_) {
-        ss << item->Dump() << std::endl;
+        ret.push_back(item->Dump());
     }
-    return ss.str();
+    return ret;
 }
 
 }  // namespace scheduler

core/src/scheduler/TaskTable.h

@@ -26,6 +26,7 @@
 #include <vector>
 
 #include "event/Event.h"
+#include "interface/interfaces.h"
 #include "task/SearchTask.h"
 
 namespace milvus {
@@ -42,7 +43,7 @@ enum class TaskTableItemState {
     MOVED,      // moved, termination state
 };
 
-struct TaskTimestamp {
+struct TaskTimestamp : public interface::dumpable {
     uint64_t start = 0;
     uint64_t move = 0;
     uint64_t moved = 0;
@@ -51,9 +52,12 @@ struct TaskTimestamp {
     uint64_t execute = 0;
     uint64_t executed = 0;
     uint64_t finish = 0;
+
+    json
+    Dump() override;
 };
 
-struct TaskTableItem {
+struct TaskTableItem : public interface::dumpable {
     TaskTableItem() : id(0), task(nullptr), state(TaskTableItemState::INVALID), mutex() {
     }
 
@@ -87,13 +91,13 @@ struct TaskTableItem {
     bool
     Moved();
 
-    std::string
-    Dump();
+    json
+    Dump() override;
 };
 
 using TaskTableItemPtr = std::shared_ptr<TaskTableItem>;
 
-class TaskTable {
+class TaskTable : public interface::dumpable {
  public:
     TaskTable() = default;
 
@@ -240,8 +244,8 @@ class TaskTable {
     /*
      * Dump;
      */
-    std::string
-    Dump();
+    json
+    Dump() override;
 
  private:
     std::uint64_t id_ = 0;

core/src/scheduler/interface/interfaces.h

@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <condition_variable>
+#include <deque>
+#include <list>
+#include <memory>
+#include <mutex>
+#include <queue>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <vector>
+
+#include "utils/Json.h"
+
+namespace milvus {
+namespace interface {
+
+struct dumpable {
+    virtual ~dumpable() {
+    }
+
+    virtual json
+    Dump() = 0;
+};
+
+}  // namespace interface
+}  // namespace milvus

core/src/utils/Json.h

@@ -0,0 +1,23 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include "external/nlohmann/json.hpp"
+
+namespace milvus {
+using json = nlohmann::json;
+}

tests/milvus_benchmark/requirements.txt

@@ -1,6 +1,6 @@
 numpy==1.16.3
 pymilvus>=0.1.18
-pyyaml==3.12
+pyyaml==5.1
 docker==4.0.2
 tableprint==0.8.0
 ansicolors==1.1.8