test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-29 18:38:44 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

[test]Wait index building complete before rolling update (#26377)

Browse Source 
Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>
This commit is contained in:
zhuwenxing 2023-08-17 16:38:18 +08:00 committed by GitHub
parent 3c62acde05
commit e7d5196f68
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 52 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
tests/python_client/chaos/testcases/test_single_request_operation_for_rolling_update.py
View File

@ -1,6 +1,9 @@
import time
from pathlib import Path
import subprocess
import pytest
from time import sleep
from pymilvus import connections
from pymilvus import connections, utility
from chaos.checker import (CreateChecker,
InsertChecker,
FlushChecker,
@ -87,10 +90,20 @@ class TestOperations(TestBase):
log.info(f"do bulk insert failed: {result}")
retry_times += 1
sleep(5)
# how to make sure the bulk insert done before rolling update?
# wait for index building complete
utility.wait_for_index_building_complete(v.c_name, timeout=120)
res = utility.index_building_progress(v.c_name)
index_completed = res["pending_index_rows"] == 0
while not index_completed:
time.sleep(10)
res = utility.index_building_progress(v.c_name)
log.info(f"index building progress: {res}")
index_completed = res["pending_index_rows"] == 0
log.info(f"index building progress: {res}")
log.info("*********************Load Start**********************")
cc.start_monitor_threads(self.health_checkers)
# wait request_duration
request_duration = request_duration.replace("h", "*3600+").replace("m", "*60+").replace("s", "")
if request_duration[-1] == "+":
@ -98,6 +111,16 @@ class TestOperations(TestBase):
request_duration = eval(request_duration)
for i in range(10):
sleep(request_duration // 10)
if i == 3:
# apply rolling update after 30% time of request_duration
log.info("*********************Apply Rolling Update**********************")
file_path = f"{str(Path(__file__).parent.parent.parent)}/deploy/milvus_crd.yaml"
cmd = f"kubectl apply -f {file_path}"
log.info(f"cmd: {cmd}")
res = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = res.communicate()
log.info(f"{cmd}, stdout: {stdout}, stderr: {stderr}")
for k, v in self.health_checkers.items():
v.check_result()
for k, v in self.health_checkers.items():

27
tests/python_client/deploy/monitor_rolling_update.py Normal file
View File

@ -0,0 +1,27 @@
import argparse
import subprocess
import time
from loguru import logger as log
def run_kubectl_get_pod(duration, interval, release_name):
end_time = time.time() + duration
while time.time() < end_time:
cmd = f"kubectl get pod |grep {release_name}"
res = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = res.communicate()
output = stdout.decode("utf-8")
log.info(f"{cmd}\n{output}\n")
time.sleep(interval)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Script to run "kubectl get pod" command at regular intervals')
parser.add_argument('-d', '--duration', type=int, default=600, help='Duration in seconds (default: 600)')
parser.add_argument('-i', '--interval', type=int, default=5, help='Interval in seconds (default: 30)')
parser.add_argument('-n', '--release_name', type=str, default="", help='release name (default: "None")')
args = parser.parse_args()
run_kubectl_get_pod(args.duration, args.interval, args.release_name)