[skip ci]Implement upgrade milvus with operator (#10260)

Signed-off-by: yanliang567 <yanliang.qiao@zilliz.com>

tests/python_client/common/cus_resource_opts.py

@@ -43,18 +43,49 @@ class CustomResourceOperations(object):
                 log.error("Exception when calling CustomObjectsApi->delete_namespaced_custom_object: %s\n" % e)
                 raise Exception(str(e))
 
+    def patch(self, metadata_name, body):
+        """patch a custom resource in k8s"""
+        config.load_kube_config()
+        api_instance = client.CustomObjectsApi()
+        try:
+            api_response = api_instance.patch_namespaced_custom_object(self.group, self.version, self.namespace,
+                                                                       plural=self.plural,
+                                                                       name=metadata_name,
+                                                                       body=body)
+            log.debug(f"patch custom resource response: {api_response}")
+        except ApiException as e:
+            log.error("Exception when calling CustomObjectsApi->patch_namespaced_custom_object: %s\n" % e)
+            raise Exception(str(e))
+        return api_response
+
     def list_all(self):
         """list all the customer resources in k8s"""
+        pretty = 'true'
         try:
             config.load_kube_config()
             api_instance = client.CustomObjectsApi()
             data = api_instance.list_namespaced_custom_object(self.group, self.version, self.namespace,
-                                                              plural=self.plural)
+                                                              plural=self.plural, pretty=pretty)
+            log.debug(f"list custom resource response: {data}")
         except ApiException as e:
             log.error("Exception when calling CustomObjectsApi->list_namespaced_custom_object: %s\n" % e)
             raise Exception(str(e))
         return data
 
+    def get(self, metadata_name):
+        """list all the customer resources in k8s"""
+        try:
+            config.load_kube_config()
+            api_instance = client.CustomObjectsApi()
+            api_response = api_instance.get_namespaced_custom_object(self.group, self.version,
+                                                                     self.namespace, self.plural,
+                                                                     name=metadata_name)
+            log.debug(f"get custom resource response: {api_response}")
+        except ApiException as e:
+            log.error("Exception when calling CustomObjectsApi->list_namespaced_custom_object: %s\n" % e)
+            raise Exception(str(e))
+        return api_response
+
     def delete_all(self):
         """delete all the customer resources in k8s"""
         cus_objects = self.list_all()
@@ -62,4 +93,3 @@ class CustomResourceOperations(object):
             for item in cus_objects["items"]:
                 metadata_name = item["metadata"]["name"]
                 self.delete(metadata_name)
-

tests/python_client/customize/milvus_operator.py

@@ -1,6 +1,7 @@
 from benedict import benedict
 from utils.util_log import test_log as log
 from common.cus_resource_opts import CustomResourceOperations as CusResource
+import time
 
 template_yaml = 'template/default.yaml'
 MILVUS_GRP = 'milvus.io'
@@ -8,58 +9,93 @@ MILVUS_VER = 'v1alpha1'
 MILVUS_PLURAL = 'milvusclusters'
 
 
-def update_configs(yaml, template):
-    if not isinstance(yaml, dict):
+def update_configs(configs, template=template_yaml, namespace='default'):
+    if not isinstance(configs, dict):
         log.error("customize configurations must be in dict type")
         return None
 
     d_configs = benedict.from_yaml(template)
 
-    for key in yaml.keys():
-        d_configs[key] = yaml[key]
+    for key in configs.keys():
+        d_configs[key] = configs[key]
+    # Overwrite namespace with input param
+    d_configs['metadata.namespace'] = namespace
 
     # return a python dict for common use
     log.info(f"customized configs: {d_configs._dict}")
     return d_configs._dict
 
 
-def install_milvus(cus_configs, template=template_yaml):
+def install_milvus(configs, template, namespace='default'):
 
-    _configs = update_configs(cus_configs, template)
+    new_configs = update_configs(configs, template, namespace)
     # apply custom resource object to deploy milvus
-    cus_res = CusResource(kind=MILVUS_PLURAL,
-                          group=MILVUS_GRP,
-                          version=MILVUS_VER,
-                          namespace='chaos-testing')
-    print(_configs)
-    return cus_res.create(_configs)
+    cus_res = CusResource(kind=MILVUS_PLURAL, group=MILVUS_GRP,
+                          version=MILVUS_VER, namespace=namespace)
+    return cus_res.create(new_configs)
 
 
 def uninstall_milvus(release_name, namespace='default'):
 
     # delete custom resource object to uninstall milvus
-    cus_res = CusResource(kind=MILVUS_PLURAL,
-                          group=MILVUS_GRP,
-                          version=MILVUS_VER,
-                          namespace=namespace)
+    cus_res = CusResource(kind=MILVUS_PLURAL, group=MILVUS_GRP,
+                          version=MILVUS_VER, namespace=namespace)
     cus_res.delete(release_name)
 
 
-def upgrade_milvus(release_name, configs):
-    # TODO: upgrade milvus with new configs
-    pass
+def upgrade_milvus(release_name, configs, namespace='default'):
+    if not isinstance(configs, dict):
+        log.error("customize configurations must be in dict type")
+        return None
+
+    d_configs = benedict()
+
+    for key in configs.keys():
+        d_configs[key] = configs[key]
+
+    cus_res = CusResource(kind=MILVUS_PLURAL, group=MILVUS_GRP,
+                          version=MILVUS_VER, namespace=namespace)
+    log.debug(f"upgrade milvus with configs: {d_configs}")
+    cus_res.patch(release_name, d_configs)
+
+
+def wait_for_milvus_healthy(release_name, namespace='default', timeout=600):
+
+    cus_res = CusResource(kind=MILVUS_PLURAL, group=MILVUS_GRP,
+                          version=MILVUS_VER, namespace=namespace)
+    starttime = time.time()
+    log.info(f"start to check healthy: {starttime}")
+    while time.time() < starttime + timeout:
+        time.sleep(10)
+        res_object = cus_res.get(release_name)
+        if res_object.get('status', None) is not None:
+            if 'Healthy' == res_object['status']['status']:
+                log.info(f"milvus healthy in {time.time()-starttime} seconds")
+                return True
+    log.info(f"end to check healthy until timeout {timeout}")
+    return False
 
 
 if __name__ == '__main__':
 
-    cus_configs = {'spec.components.image': 'milvusdb/milvus-dev:master-20211013-91d8f85',
-                   'metadata.namespace': 'chaos-testing',
-                   'metadata.name': 'milvus-dbl-testop2',
+    cus_configs = {'spec.components.image': 'milvusdb/milvus-dev:master-20211020-4d7252739',
+                   'metadata.namespace': 'def',
+                   'metadata.name': 'milvus-2739',
                    'spec.components.queryNode.replicas': 2,
                    'spec.components.queryNode.resources.limits.memory': '2048Mi'
                    }
-    milvus_instance = install_milvus(cus_configs, template_yaml)
-    print(milvus_instance)
-    # upgrade_milvus(cus_configs)
-    # uninstall_milvus('milvus-dbl-testop2', namespace='chaos-testing')
-    # update_configs(cus_configs)
+    namespace = 'chaos-testing'
+    name = 'milvus-2739'
+
+    milvus_instance = install_milvus(cus_configs, template_yaml, namespace=namespace)
+    result = wait_for_milvus_healthy(name, namespace=namespace)
+    log.info(f"install milvus healthy: {result}")
+    n_configs = {'spec.components.queryNode.replicas': 1,
+                 'spec.components.dataNode.resources.limits.memory': '2048Mi'
+                 }
+    upgrade_milvus(name, n_configs, namespace=namespace)
+    result = wait_for_milvus_healthy(name, namespace=namespace)
+    log.info(f"upgrade milvus healthy: {result}")
+
+    # uninstall_milvus(name, namespace=namespace)
+

tests/python_client/customize/template/default.yaml

@@ -9,12 +9,12 @@ spec:
     etcd:
       inCluster:
         deletionPolicy: Delete
-        persistenceKeep: false
+        pvcDeletion: true
     pulsar:
       inCluster:
         deletionPolicy: Delete
-        persistenceKeep: false
+        pvcDeletion: true
     storage:
       inCluster:
         deletionPolicy: Delete
-        persistenceKeep: false
+        pvcDeletion: true