mirror of
https://gitee.com/iresty/apisix.git
synced 2024-12-03 20:47:35 +08:00
refactor: remove kubectl dependency in chaos test, use k8s API instead (#3501)
This commit is contained in:
Shuyang Wu
GitHub
parent
e5cd46e38e
commit
26a4ed4ad4
10
.github/workflows/chaos.yml
vendored
10
.github/workflows/chaos.yml
vendored
@ -14,7 +14,7 @@ jobs:
|
||||
|
||||
- name: Creating minikube cluster
|
||||
run: |
|
||||
bash ./t/chaos/utils.sh start_minikube
|
||||
bash ./t/chaos/setup_chaos_utils.sh start_minikube
|
||||
|
||||
- name: Print cluster information
|
||||
run: |
|
||||
@ -29,17 +29,17 @@ jobs:
|
||||
git clone https://github.com/api7/etcd-operator.git --depth 1
|
||||
bash etcd-operator/example/rbac/create_role.sh
|
||||
kubectl create -f etcd-operator/example/deployment.yaml
|
||||
bash ./t/chaos/utils.sh ensure_pods_ready etcd-operator "True" 30
|
||||
bash ./t/chaos/setup_chaos_utils.sh ensure_pods_ready etcd-operator "True" 30
|
||||
kubectl create -f etcd-operator/example/example-etcd-cluster.yaml
|
||||
bash ./t/chaos/utils.sh ensure_pods_ready etcd "True True True" 30
|
||||
bash ./t/chaos/setup_chaos_utils.sh ensure_pods_ready etcd "True True True" 30
|
||||
|
||||
- name: Deploy APISIX
|
||||
run: |
|
||||
bash ./t/chaos/utils.sh modify_config
|
||||
bash ./t/chaos/setup_chaos_utils.sh modify_config
|
||||
kubectl create configmap apisix-gw-config.yaml --from-file=./conf/config.yaml
|
||||
kubectl apply -f ./kubernetes/deployment.yaml
|
||||
kubectl apply -f ./kubernetes/service.yaml
|
||||
bash ./t/chaos/utils.sh ensure_pods_ready apisix-gw "True" 30
|
||||
bash ./t/chaos/setup_chaos_utils.sh ensure_pods_ready apisix-gw "True" 30
|
||||
nohup kubectl port-forward svc/apisix-gw-lb 9080:9080 >/dev/null 2>&1 &
|
||||
|
||||
- name: Deploy Chaos mesh
|
||||
|
||||
@ -1,8 +1,38 @@
|
||||
module github.com/apache/apisix/t/chaos
|
||||
|
||||
require (
|
||||
github.com/chaos-mesh/chaos-mesh v1.1.1
|
||||
github.com/gavv/httpexpect/v2 v2.1.0
|
||||
github.com/onsi/gomega v1.9.0
|
||||
k8s.io/api v0.17.0
|
||||
k8s.io/apimachinery v0.17.0
|
||||
k8s.io/client-go v0.17.0
|
||||
k8s.io/kubectl v0.0.0
|
||||
sigs.k8s.io/controller-runtime v0.4.0
|
||||
)
|
||||
|
||||
replace (
|
||||
k8s.io/api => k8s.io/api v0.17.0
|
||||
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.17.0
|
||||
k8s.io/apimachinery => k8s.io/apimachinery v0.17.1-beta.0
|
||||
k8s.io/apiserver => k8s.io/apiserver v0.17.0
|
||||
k8s.io/cli-runtime => k8s.io/cli-runtime v0.17.0
|
||||
k8s.io/client-go => k8s.io/client-go v0.17.0
|
||||
k8s.io/cloud-provider => k8s.io/cloud-provider v0.17.0
|
||||
k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.17.0
|
||||
k8s.io/code-generator => k8s.io/code-generator v0.17.1-beta.0
|
||||
k8s.io/component-base => k8s.io/component-base v0.17.0
|
||||
k8s.io/cri-api => k8s.io/cri-api v0.17.1-beta.0
|
||||
k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.17.0
|
||||
k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.17.0
|
||||
k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.17.0
|
||||
k8s.io/kube-proxy => k8s.io/kube-proxy v0.17.0
|
||||
k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.17.0
|
||||
k8s.io/kubectl => k8s.io/kubectl v0.17.0
|
||||
k8s.io/kubelet => k8s.io/kubelet v0.17.0
|
||||
k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.17.0
|
||||
k8s.io/metrics => k8s.io/metrics v0.17.0
|
||||
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.17.0
|
||||
)
|
||||
|
||||
go 1.14
|
||||
|
||||
1042
t/chaos/go.sum
1042
t/chaos/go.sum
File diff suppressed because it is too large
Load Diff
@ -18,191 +18,45 @@
|
||||
package chaos
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"context"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
|
||||
"github.com/gavv/httpexpect/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
|
||||
var (
|
||||
token = "edd1c9f034335f136f87ad84b625c8f1"
|
||||
host = "http://127.0.0.1:9080"
|
||||
)
|
||||
|
||||
type httpTestCase struct {
|
||||
E *httpexpect.Expect
|
||||
Method string
|
||||
Path string
|
||||
Body string
|
||||
Headers map[string]string
|
||||
ExpectStatus int
|
||||
ExpectBody string
|
||||
}
|
||||
|
||||
func caseCheck(tc httpTestCase) *httpexpect.Response {
|
||||
e := tc.E
|
||||
var req *httpexpect.Request
|
||||
switch tc.Method {
|
||||
case http.MethodGet:
|
||||
req = e.GET(tc.Path)
|
||||
case http.MethodPut:
|
||||
req = e.PUT(tc.Path)
|
||||
default:
|
||||
}
|
||||
|
||||
if req == nil {
|
||||
panic("fail to init request")
|
||||
}
|
||||
for key, val := range tc.Headers {
|
||||
req.WithHeader(key, val)
|
||||
}
|
||||
if tc.Body != "" {
|
||||
req.WithText(tc.Body)
|
||||
}
|
||||
|
||||
resp := req.Expect()
|
||||
if tc.ExpectStatus != 0 {
|
||||
resp.Status(tc.ExpectStatus)
|
||||
}
|
||||
|
||||
if tc.ExpectBody != "" {
|
||||
resp.Body().Contains(tc.ExpectBody)
|
||||
}
|
||||
|
||||
return resp
|
||||
}
|
||||
|
||||
func setRoute(e *httpexpect.Expect, expectStatus int) {
|
||||
caseCheck(httpTestCase{
|
||||
E: e,
|
||||
Method: http.MethodPut,
|
||||
Path: "/apisix/admin/routes/1",
|
||||
Headers: map[string]string{"X-API-KEY": token},
|
||||
Body: `{
|
||||
"uri": "/hello",
|
||||
"host": "foo.com",
|
||||
"plugins": {
|
||||
"prometheus": {}
|
||||
func createEtcdKillChaos(g *WithT, cli client.Client) {
|
||||
chaos := &v1alpha1.PodChaos{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "kill-etcd",
|
||||
Namespace: metav1.NamespaceDefault,
|
||||
},
|
||||
Spec: v1alpha1.PodChaosSpec{
|
||||
Selector: v1alpha1.SelectorSpec{
|
||||
LabelSelectors: map[string]string{"app": "etcd"},
|
||||
},
|
||||
"upstream": {
|
||||
"nodes": {
|
||||
"bar.org": 1
|
||||
},
|
||||
"type": "roundrobin"
|
||||
}
|
||||
}`,
|
||||
ExpectStatus: expectStatus,
|
||||
})
|
||||
}
|
||||
|
||||
func getRoute(e *httpexpect.Expect, expectStatus int) {
|
||||
caseCheck(httpTestCase{
|
||||
E: e,
|
||||
Method: http.MethodGet,
|
||||
Path: "/hello",
|
||||
Headers: map[string]string{"Host": "foo.com"},
|
||||
ExpectStatus: expectStatus,
|
||||
})
|
||||
}
|
||||
|
||||
func deleteRoute(e *httpexpect.Expect, expectStatus int) {
|
||||
caseCheck(httpTestCase{
|
||||
E: e,
|
||||
Method: http.MethodDelete,
|
||||
Path: "/apisix/admin/routes/1",
|
||||
Headers: map[string]string{"X-API-KEY": token},
|
||||
ExpectStatus: expectStatus,
|
||||
})
|
||||
}
|
||||
|
||||
func testPrometheusEtcdMetric(e *httpexpect.Expect, expectEtcd int) {
|
||||
caseCheck(httpTestCase{
|
||||
E: e,
|
||||
Method: http.MethodGet,
|
||||
Path: "/apisix/prometheus/metrics",
|
||||
ExpectBody: fmt.Sprintf("apisix_etcd_reachable %d", expectEtcd),
|
||||
})
|
||||
}
|
||||
|
||||
// get the first line which contains the key
|
||||
func getPrometheusMetric(e *httpexpect.Expect, g *WithT, key string) string {
|
||||
resp := caseCheck(httpTestCase{
|
||||
E: e,
|
||||
Method: http.MethodGet,
|
||||
Path: "/apisix/prometheus/metrics",
|
||||
})
|
||||
resps := strings.Split(resp.Body().Raw(), "\n")
|
||||
var targetLine string
|
||||
for _, line := range resps {
|
||||
if strings.Contains(line, key) {
|
||||
targetLine = line
|
||||
break
|
||||
}
|
||||
Action: v1alpha1.PodKillAction,
|
||||
Mode: v1alpha1.AllPodMode,
|
||||
Scheduler: &v1alpha1.SchedulerSpec{
|
||||
Cron: "@every 10m",
|
||||
},
|
||||
},
|
||||
}
|
||||
targetSlice := strings.Fields(targetLine)
|
||||
g.Expect(len(targetSlice) == 2).To(BeTrue())
|
||||
return targetSlice[1]
|
||||
}
|
||||
|
||||
func getIngressBandwidthPerSecond(e *httpexpect.Expect, g *WithT) float64 {
|
||||
key := "apisix_bandwidth{type=\"ingress\","
|
||||
bandWidthString := getPrometheusMetric(e, g, key)
|
||||
bandWidthStart, err := strconv.ParseFloat(bandWidthString, 64)
|
||||
err := cli.Create(context.Background(), chaos)
|
||||
g.Expect(err).To(BeNil())
|
||||
// after etcd got killed, it would take longer time to get the metrics
|
||||
// so need to calculate the duration
|
||||
timeStart := time.Now()
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
bandWidthString = getPrometheusMetric(e, g, key)
|
||||
bandWidthEnd, err := strconv.ParseFloat(bandWidthString, 64)
|
||||
g.Expect(err).To(BeNil())
|
||||
duration := time.Now().Sub(timeStart)
|
||||
|
||||
return (bandWidthEnd - bandWidthStart) / duration.Seconds()
|
||||
}
|
||||
|
||||
func runCommand(t *testing.T, cmd string) string {
|
||||
out, err := exec.Command("bash", "-c", cmd).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("fail to run command %s: %s, %s", cmd, err.Error(), out)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func roughCompare(a float64, b float64) bool {
|
||||
ratio := a / b
|
||||
if ratio < 1.3 && ratio > 0.7 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type silentPrinter struct {
|
||||
logger httpexpect.Logger
|
||||
}
|
||||
|
||||
func newSilentPrinter(logger httpexpect.Logger) silentPrinter {
|
||||
return silentPrinter{logger}
|
||||
}
|
||||
|
||||
// Request implements Printer.Request.
|
||||
func (p silentPrinter) Request(req *http.Request) {
|
||||
}
|
||||
|
||||
// Response implements Printer.Response.
|
||||
func (silentPrinter) Response(*http.Response, time.Duration) {
|
||||
}
|
||||
|
||||
func TestGetSuccessWhenEtcdKilled(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
e := httpexpect.New(t, host)
|
||||
cliSet := initClientSet(g)
|
||||
|
||||
eSilent := httpexpect.WithConfig(httpexpect.Config{
|
||||
BaseURL: host,
|
||||
@ -233,17 +87,19 @@ func TestGetSuccessWhenEtcdKilled(t *testing.T) {
|
||||
bpsBefore := getIngressBandwidthPerSecond(e, g)
|
||||
g.Expect(bpsBefore).NotTo(BeZero())
|
||||
|
||||
podName := runCommand(t, "kubectl get pod -l app=apisix-gw -o 'jsonpath={..metadata.name}'")
|
||||
listOption := client.MatchingLabels{"app": "apisix-gw"}
|
||||
apisixPod := getPod(g, cliSet.ctrlCli, metav1.NamespaceDefault, listOption)
|
||||
|
||||
t.Run("error log not contains etcd error", func(t *testing.T) {
|
||||
errorLog := runCommand(t, fmt.Sprintf("kubectl exec -it %s -- cat logs/error.log", podName))
|
||||
errorLog := execInPod(g, cliSet.kubeCli, apisixPod, "cat logs/error.log")
|
||||
g.Expect(strings.Contains(errorLog, "failed to fetch data from etcd")).To(BeFalse())
|
||||
})
|
||||
|
||||
// TODO: use client-go
|
||||
// apply chaos to kill all etcd pods
|
||||
t.Log("kill all etcd pods")
|
||||
_ = runCommand(t, "kubectl apply -f kill-etcd.yaml")
|
||||
time.Sleep(3 * time.Second)
|
||||
t.Run("kill all etcd pods", func(t *testing.T) {
|
||||
createEtcdKillChaos(g, cliSet.ctrlCli)
|
||||
time.Sleep(3 * time.Second)
|
||||
})
|
||||
|
||||
// fail to set route since etcd is all killed
|
||||
// while get route could still succeed
|
||||
@ -252,7 +108,7 @@ func TestGetSuccessWhenEtcdKilled(t *testing.T) {
|
||||
testPrometheusEtcdMetric(e, 0)
|
||||
|
||||
t.Run("error log contains etcd error", func(t *testing.T) {
|
||||
errorLog := runCommand(t, fmt.Sprintf("kubectl exec -it %s -- cat logs/error.log", podName))
|
||||
errorLog := execInPod(g, cliSet.kubeCli, apisixPod, "cat logs/error.log")
|
||||
g.Expect(strings.Contains(errorLog, "failed to fetch data from etcd")).To(BeTrue())
|
||||
})
|
||||
|
||||
|
||||
102
t/chaos/kube_utils.go
Normal file
102
t/chaos/kube_utils.go
Normal file
@ -0,0 +1,102 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package chaos
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
clientScheme "k8s.io/client-go/kubernetes/scheme"
|
||||
"k8s.io/client-go/tools/remotecommand"
|
||||
kubectlScheme "k8s.io/kubectl/pkg/scheme"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/config"
|
||||
)
|
||||
|
||||
type clientSet struct {
|
||||
ctrlCli client.Client
|
||||
kubeCli *kubernetes.Clientset
|
||||
}
|
||||
|
||||
func initClientSet(g *WithT) *clientSet {
|
||||
scheme := runtime.NewScheme()
|
||||
v1alpha1.AddToScheme(scheme)
|
||||
clientScheme.AddToScheme(scheme)
|
||||
|
||||
restConfig := config.GetConfigOrDie()
|
||||
ctrlCli, err := client.New(restConfig, client.Options{Scheme: scheme})
|
||||
g.Expect(err).To(BeNil())
|
||||
kubeCli, err := kubernetes.NewForConfig(restConfig)
|
||||
g.Expect(err).To(BeNil())
|
||||
|
||||
return &clientSet{ctrlCli, kubeCli}
|
||||
}
|
||||
|
||||
func getPod(g *WithT, cli client.Client, ns string, listOption client.MatchingLabels) *corev1.Pod {
|
||||
podList := &corev1.PodList{}
|
||||
err := cli.List(context.Background(), podList, client.InNamespace(ns), listOption)
|
||||
g.Expect(err).To(BeNil())
|
||||
return &podList.Items[0]
|
||||
}
|
||||
|
||||
func execInPod(g *WithT, cli *kubernetes.Clientset, pod *corev1.Pod, cmd string) string {
|
||||
name := pod.GetName()
|
||||
namespace := pod.GetNamespace()
|
||||
// only get the first container, no harm for now
|
||||
containerName := pod.Spec.Containers[0].Name
|
||||
|
||||
req := cli.CoreV1().RESTClient().Post().
|
||||
Resource("pods").
|
||||
Name(name).
|
||||
Namespace(namespace).
|
||||
SubResource("exec")
|
||||
|
||||
req.VersionedParams(&corev1.PodExecOptions{
|
||||
Container: containerName,
|
||||
Command: []string{"/bin/sh", "-c", cmd},
|
||||
Stdin: false,
|
||||
Stdout: true,
|
||||
Stderr: true,
|
||||
TTY: false,
|
||||
}, kubectlScheme.ParameterCodec)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
exec, err := remotecommand.NewSPDYExecutor(config.GetConfigOrDie(), "POST", req.URL())
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("error: %s\nin creating NewSPDYExecutor for pod %s/%s", err.Error(), namespace, name))
|
||||
}
|
||||
err = exec.Stream(remotecommand.StreamOptions{
|
||||
Stdin: nil,
|
||||
Stdout: &stdout,
|
||||
Stderr: &stderr,
|
||||
})
|
||||
if stderr.String() != "" {
|
||||
panic(fmt.Sprintf("error: %s\npod: %s\ncommand: %s", strings.TrimSuffix(stderr.String(), "\n"), pod.Name, cmd))
|
||||
}
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("error: %s\nin streaming remotecommand: pod: %s/%s, command: %s", err.Error(), namespace, pod.Name, cmd))
|
||||
}
|
||||
return stdout.String()
|
||||
}
|
||||
192
t/chaos/utils.go
Normal file
192
t/chaos/utils.go
Normal file
@ -0,0 +1,192 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package chaos
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gavv/httpexpect/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var (
|
||||
token = "edd1c9f034335f136f87ad84b625c8f1"
|
||||
host = "http://127.0.0.1:9080"
|
||||
)
|
||||
|
||||
type httpTestCase struct {
|
||||
E *httpexpect.Expect
|
||||
Method string
|
||||
Path string
|
||||
Body string
|
||||
Headers map[string]string
|
||||
ExpectStatus int
|
||||
ExpectBody string
|
||||
}
|
||||
|
||||
func caseCheck(tc httpTestCase) *httpexpect.Response {
|
||||
e := tc.E
|
||||
var req *httpexpect.Request
|
||||
switch tc.Method {
|
||||
case http.MethodGet:
|
||||
req = e.GET(tc.Path)
|
||||
case http.MethodPut:
|
||||
req = e.PUT(tc.Path)
|
||||
default:
|
||||
panic("invalid HTTP method")
|
||||
}
|
||||
|
||||
if req == nil {
|
||||
panic("fail to init request")
|
||||
}
|
||||
for key, val := range tc.Headers {
|
||||
req.WithHeader(key, val)
|
||||
}
|
||||
if tc.Body != "" {
|
||||
req.WithText(tc.Body)
|
||||
}
|
||||
|
||||
resp := req.Expect()
|
||||
if tc.ExpectStatus != 0 {
|
||||
resp.Status(tc.ExpectStatus)
|
||||
}
|
||||
|
||||
if tc.ExpectBody != "" {
|
||||
resp.Body().Contains(tc.ExpectBody)
|
||||
}
|
||||
|
||||
return resp
|
||||
}
|
||||
|
||||
func setRoute(e *httpexpect.Expect, expectStatus int) {
|
||||
caseCheck(httpTestCase{
|
||||
E: e,
|
||||
Method: http.MethodPut,
|
||||
Path: "/apisix/admin/routes/1",
|
||||
Headers: map[string]string{"X-API-KEY": token},
|
||||
Body: `{
|
||||
"uri": "/hello",
|
||||
"host": "foo.com",
|
||||
"plugins": {
|
||||
"prometheus": {}
|
||||
},
|
||||
"upstream": {
|
||||
"nodes": {
|
||||
"bar.org": 1
|
||||
},
|
||||
"type": "roundrobin"
|
||||
}
|
||||
}`,
|
||||
ExpectStatus: expectStatus,
|
||||
})
|
||||
}
|
||||
|
||||
func getRoute(e *httpexpect.Expect, expectStatus int) {
|
||||
caseCheck(httpTestCase{
|
||||
E: e,
|
||||
Method: http.MethodGet,
|
||||
Path: "/hello",
|
||||
Headers: map[string]string{"Host": "foo.com"},
|
||||
ExpectStatus: expectStatus,
|
||||
})
|
||||
}
|
||||
|
||||
func deleteRoute(e *httpexpect.Expect, expectStatus int) {
|
||||
caseCheck(httpTestCase{
|
||||
E: e,
|
||||
Method: http.MethodDelete,
|
||||
Path: "/apisix/admin/routes/1",
|
||||
Headers: map[string]string{"X-API-KEY": token},
|
||||
ExpectStatus: expectStatus,
|
||||
})
|
||||
}
|
||||
|
||||
func testPrometheusEtcdMetric(e *httpexpect.Expect, expectEtcd int) {
|
||||
caseCheck(httpTestCase{
|
||||
E: e,
|
||||
Method: http.MethodGet,
|
||||
Path: "/apisix/prometheus/metrics",
|
||||
ExpectBody: fmt.Sprintf("apisix_etcd_reachable %d", expectEtcd),
|
||||
})
|
||||
}
|
||||
|
||||
// get the first line which contains the key
|
||||
func getPrometheusMetric(e *httpexpect.Expect, g *WithT, key string) string {
|
||||
resp := caseCheck(httpTestCase{
|
||||
E: e,
|
||||
Method: http.MethodGet,
|
||||
Path: "/apisix/prometheus/metrics",
|
||||
})
|
||||
resps := strings.Split(resp.Body().Raw(), "\n")
|
||||
var targetLine string
|
||||
for _, line := range resps {
|
||||
if strings.Contains(line, key) {
|
||||
targetLine = line
|
||||
break
|
||||
}
|
||||
}
|
||||
targetSlice := strings.Fields(targetLine)
|
||||
g.Expect(len(targetSlice) == 2).To(BeTrue())
|
||||
return targetSlice[1]
|
||||
}
|
||||
|
||||
func getIngressBandwidthPerSecond(e *httpexpect.Expect, g *WithT) float64 {
|
||||
key := "apisix_bandwidth{type=\"ingress\","
|
||||
bandWidthString := getPrometheusMetric(e, g, key)
|
||||
bandWidthStart, err := strconv.ParseFloat(bandWidthString, 64)
|
||||
g.Expect(err).To(BeNil())
|
||||
// after etcd got killed, it would take longer time to get the metrics
|
||||
// so need to calculate the duration
|
||||
timeStart := time.Now()
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
bandWidthString = getPrometheusMetric(e, g, key)
|
||||
bandWidthEnd, err := strconv.ParseFloat(bandWidthString, 64)
|
||||
g.Expect(err).To(BeNil())
|
||||
duration := time.Now().Sub(timeStart)
|
||||
|
||||
return (bandWidthEnd - bandWidthStart) / duration.Seconds()
|
||||
}
|
||||
|
||||
func roughCompare(a float64, b float64) bool {
|
||||
ratio := a / b
|
||||
if ratio < 1.3 && ratio > 0.7 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type silentPrinter struct {
|
||||
logger httpexpect.Logger
|
||||
}
|
||||
|
||||
func newSilentPrinter(logger httpexpect.Logger) silentPrinter {
|
||||
return silentPrinter{logger}
|
||||
}
|
||||
|
||||
// Request implements Printer.Request.
|
||||
func (p silentPrinter) Request(req *http.Request) {
|
||||
}
|
||||
|
||||
// Response implements Printer.Response.
|
||||
func (silentPrinter) Response(*http.Response, time.Duration) {
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user