mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-12-02 11:59:00 +08:00
114 lines
4.0 KiB
Python
114 lines
4.0 KiB
Python
import os.path
|
|
import time
|
|
from kubernetes import client, config
|
|
from kubernetes.client.rest import ApiException
|
|
from utils.util_log import test_log as log
|
|
|
|
|
|
def wait_pods_ready(namespace, label_selector, expected_num=None, timeout=360):
|
|
"""
|
|
wait pods with label selector all ready
|
|
|
|
:param namespace: the namespace where the release
|
|
:type namespace: str
|
|
|
|
:param label_selector: labels to restrict which pods are waiting to be ready
|
|
:type label_selector: str
|
|
|
|
:param expected_num: expected the minimum number of pods to be ready if not None
|
|
:type expected_num: int
|
|
|
|
:param timeout: limits the duration of the call
|
|
:type timeout: int
|
|
|
|
:example:
|
|
>>> wait_pods_ready("default", "app.kubernetes.io/instance=scale-query", expected_num=9)
|
|
"""
|
|
config.load_kube_config()
|
|
api_instance = client.CoreV1Api()
|
|
try:
|
|
all_pos_ready_flag = False
|
|
time_cnt = 0
|
|
while not all_pos_ready_flag and time_cnt < timeout:
|
|
api_response = api_instance.list_namespaced_pod(namespace=namespace, label_selector=label_selector)
|
|
all_pos_ready_flag = True
|
|
if expected_num is not None and len(api_response.items) < expected_num:
|
|
all_pos_ready_flag = False
|
|
else:
|
|
for item in api_response.items:
|
|
for c in item.status.container_statuses:
|
|
log.info(f"{c.name} status is {c.ready}")
|
|
if c.ready is False:
|
|
all_pos_ready_flag = False
|
|
break
|
|
if not all_pos_ready_flag:
|
|
log.info("all pods are not ready, please wait")
|
|
time.sleep(30)
|
|
time_cnt += 30
|
|
if all_pos_ready_flag:
|
|
log.info(f"all pods in namespace {namespace} with label {label_selector} are ready")
|
|
else:
|
|
log.info(f"timeout for waiting all pods in namespace {namespace} with label {label_selector} ready")
|
|
except ApiException as e:
|
|
log.error("Exception when calling CoreV1Api->list_namespaced_pod: %s\n" % e)
|
|
raise Exception(str(e))
|
|
|
|
return all_pos_ready_flag
|
|
|
|
|
|
def get_pod_list(namespace, label_selector):
|
|
"""
|
|
get pod list with label selector
|
|
|
|
:param namespace: the namespace where the release
|
|
:type namespace: str
|
|
|
|
:param label_selector: labels to restrict which pods to list
|
|
:type label_selector: str
|
|
|
|
:example:
|
|
>>> get_pod_list("chaos-testing", "app.kubernetes.io/instance=test-proxy-pod-failure, component=proxy")
|
|
"""
|
|
config.load_kube_config()
|
|
api_instance = client.CoreV1Api()
|
|
try:
|
|
api_response = api_instance.list_namespaced_pod(namespace=namespace, label_selector=label_selector)
|
|
return api_response.items
|
|
except ApiException as e:
|
|
log.error("Exception when calling CoreV1Api->list_namespaced_pod: %s\n" % e)
|
|
raise Exception(str(e))
|
|
|
|
|
|
def export_pod_logs(namespace, label_selector):
|
|
"""
|
|
export pod logs with label selector to '/tmp/milvus'
|
|
|
|
:param namespace: the namespace where the release
|
|
:type namespace: str
|
|
|
|
:param label_selector: labels to restrict which pods logs to export
|
|
:type label_selector: str
|
|
|
|
:example:
|
|
>>> export_pod_logs("chaos-testing", "app.kubernetes.io/instance=mic-milvus")
|
|
"""
|
|
pod_log_path = '/tmp/milvus_logs'
|
|
if not os.path.isdir(pod_log_path):
|
|
os.makedirs(pod_log_path)
|
|
|
|
# get pods and export logs
|
|
items = get_pod_list(namespace, label_selector=label_selector)
|
|
try:
|
|
for item in items:
|
|
pod_name = item.metadata.name
|
|
os.system(f'kubectl logs {pod_name} > {pod_log_path}/{pod_name}.log 2>&1')
|
|
except Exception as e:
|
|
log.error(f"Exception when export pod {pod_name} logs: %s\n" % e)
|
|
raise Exception(str(e))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
label = "app.kubernetes.io/instance=test-proxy-pod-failure, component=proxy"
|
|
res = get_pod_list("chaos-testing", label_selector=label)
|
|
export_pod_logs(namespace='chaos-testing', label_selector=label)
|