2021-06-17 11:49:57 +08:00
|
|
|
import os
|
|
|
|
import threading
|
2022-10-18 17:21:26 +08:00
|
|
|
import time
|
2021-06-17 11:49:57 +08:00
|
|
|
import glob
|
2021-08-20 11:00:56 +08:00
|
|
|
from chaos import constants
|
2021-06-17 11:49:57 +08:00
|
|
|
from yaml import full_load
|
2021-06-24 11:56:08 +08:00
|
|
|
from utils.util_log import test_log as log
|
2022-06-25 15:38:15 +08:00
|
|
|
from delayed_assert import expect
|
2023-07-25 17:07:02 +08:00
|
|
|
import pytest
|
|
|
|
|
2021-06-17 11:49:57 +08:00
|
|
|
|
|
|
|
def check_config(chaos_config):
|
2023-07-25 17:07:02 +08:00
|
|
|
if not chaos_config.get("kind", None):
|
2021-09-14 13:17:51 +08:00
|
|
|
raise Exception("kind must be specified")
|
2023-07-25 17:07:02 +08:00
|
|
|
if not chaos_config.get("spec", None):
|
2021-09-14 13:17:51 +08:00
|
|
|
raise Exception("spec must be specified")
|
2023-07-25 17:07:02 +08:00
|
|
|
if "action" not in chaos_config.get("spec", None):
|
2021-09-14 13:17:51 +08:00
|
|
|
raise Exception("action must be specified in spec")
|
2023-07-25 17:07:02 +08:00
|
|
|
if "selector" not in chaos_config.get("spec", None):
|
2021-09-14 13:17:51 +08:00
|
|
|
raise Exception("selector must be specified in spec")
|
2021-06-17 11:49:57 +08:00
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
def reset_counting(checkers={}):
|
2021-10-06 20:55:17 +08:00
|
|
|
"""reset checker counts for all checker threads"""
|
2021-06-17 11:49:57 +08:00
|
|
|
for ch in checkers.values():
|
|
|
|
ch.reset()
|
|
|
|
|
|
|
|
|
|
|
|
def gen_experiment_config(yaml):
|
2021-10-09 18:09:55 +08:00
|
|
|
"""load the yaml file of chaos experiment"""
|
2021-06-17 11:49:57 +08:00
|
|
|
with open(yaml) as f:
|
|
|
|
_config = full_load(f)
|
|
|
|
f.close()
|
|
|
|
return _config
|
|
|
|
|
|
|
|
|
|
|
|
def start_monitor_threads(checkers={}):
|
2021-10-05 20:37:34 +08:00
|
|
|
"""start the threads by checkers"""
|
2023-09-08 10:09:16 +08:00
|
|
|
tasks = []
|
2021-06-24 11:56:08 +08:00
|
|
|
for k, ch in checkers.items():
|
2022-07-01 09:06:23 +08:00
|
|
|
ch._keep_running = True
|
2021-11-08 13:01:34 +08:00
|
|
|
t = threading.Thread(target=ch.keep_running, args=(), name=k, daemon=True)
|
2021-06-17 11:49:57 +08:00
|
|
|
t.start()
|
2023-09-08 10:09:16 +08:00
|
|
|
tasks.append(t)
|
|
|
|
return tasks
|
|
|
|
|
|
|
|
|
|
|
|
def check_thread_status(tasks):
|
|
|
|
"""check the status of all threads"""
|
|
|
|
for t in tasks:
|
|
|
|
if t.is_alive():
|
|
|
|
log.info(f"thread {t.name} is still running")
|
|
|
|
else:
|
|
|
|
log.info(f"thread {t.name} is not running")
|
2021-06-17 11:49:57 +08:00
|
|
|
|
|
|
|
|
|
|
|
def get_env_variable_by_name(name):
|
2023-07-25 17:07:02 +08:00
|
|
|
"""get env variable by name"""
|
2021-06-17 11:49:57 +08:00
|
|
|
try:
|
|
|
|
env_var = os.environ[name]
|
2021-06-24 11:56:08 +08:00
|
|
|
log.debug(f"env_variable: {env_var}")
|
2021-06-17 11:49:57 +08:00
|
|
|
return str(env_var)
|
|
|
|
except Exception as e:
|
2021-06-24 11:56:08 +08:00
|
|
|
log.debug(f"fail to get env variables, error: {str(e)}")
|
2021-06-17 11:49:57 +08:00
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
def get_chaos_yamls():
|
2021-10-04 08:32:05 +08:00
|
|
|
"""get chaos yaml file(s) from configured environment path"""
|
2021-06-17 11:49:57 +08:00
|
|
|
chaos_env = get_env_variable_by_name(constants.CHAOS_CONFIG_ENV)
|
|
|
|
if chaos_env is not None:
|
|
|
|
if os.path.isdir(chaos_env):
|
2021-06-24 11:56:08 +08:00
|
|
|
log.debug(f"chaos_env is a dir: {chaos_env}")
|
2023-07-25 17:07:02 +08:00
|
|
|
return glob.glob(chaos_env + "chaos_*.yaml")
|
2021-06-17 11:49:57 +08:00
|
|
|
elif os.path.isfile(chaos_env):
|
2021-06-24 11:56:08 +08:00
|
|
|
log.debug(f"chaos_env is a file: {chaos_env}")
|
2021-06-17 11:49:57 +08:00
|
|
|
return [chaos_env]
|
|
|
|
else:
|
|
|
|
# not a valid directory, return default
|
|
|
|
pass
|
2021-09-17 14:01:49 +08:00
|
|
|
log.debug("not a valid directory or file, return default chaos config path")
|
2021-06-24 11:56:08 +08:00
|
|
|
return glob.glob(constants.TESTS_CONFIG_LOCATION + constants.ALL_CHAOS_YAMLS)
|
|
|
|
|
2021-06-17 11:49:57 +08:00
|
|
|
|
2023-07-25 17:07:02 +08:00
|
|
|
def reconnect(connections, alias="default", timeout=360):
|
2021-10-03 08:50:02 +08:00
|
|
|
"""trying to connect by connection alias"""
|
2022-10-18 17:21:26 +08:00
|
|
|
is_connected = False
|
|
|
|
start = time.time()
|
|
|
|
end = time.time()
|
2023-07-25 17:07:02 +08:00
|
|
|
while not is_connected or end - start < timeout:
|
2022-10-18 17:21:26 +08:00
|
|
|
try:
|
|
|
|
connections.connect(alias)
|
|
|
|
is_connected = True
|
|
|
|
except Exception as e:
|
|
|
|
log.debug(f"fail to connect, error: {str(e)}")
|
|
|
|
time.sleep(10)
|
|
|
|
end = time.time()
|
|
|
|
else:
|
|
|
|
log.info(f"failed to reconnect after {timeout} seconds")
|
2022-07-03 12:20:18 +08:00
|
|
|
return connections.connect(alias)
|
2022-06-25 15:38:15 +08:00
|
|
|
|
|
|
|
|
2023-07-25 17:07:02 +08:00
|
|
|
def assert_statistic(
|
|
|
|
checkers, expectations={}, succ_rate_threshold=0.95, fail_rate_threshold=0.49
|
|
|
|
):
|
2022-06-25 15:38:15 +08:00
|
|
|
for k in checkers.keys():
|
|
|
|
# expect succ if no expectations
|
|
|
|
succ_rate = checkers[k].succ_rate()
|
|
|
|
total = checkers[k].total()
|
|
|
|
average_time = checkers[k].average_time
|
2023-07-25 17:07:02 +08:00
|
|
|
if expectations.get(k, "") == constants.FAIL:
|
|
|
|
log.info(
|
|
|
|
f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}"
|
|
|
|
)
|
|
|
|
pytest.assume(
|
|
|
|
succ_rate < fail_rate_threshold or total < 2,
|
|
|
|
f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}",
|
|
|
|
)
|
2022-06-25 15:38:15 +08:00
|
|
|
else:
|
2023-07-25 17:07:02 +08:00
|
|
|
log.info(
|
|
|
|
f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}"
|
|
|
|
)
|
|
|
|
pytest.assume(
|
2023-08-29 14:30:27 +08:00
|
|
|
succ_rate >= succ_rate_threshold and total > 2,
|
2023-07-25 17:07:02 +08:00
|
|
|
f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}",
|
|
|
|
)
|