test/apisix
1
0
Fork 0
mirror of https://gitee.com/iresty/apisix.git synced 2024-12-02 12:07:35 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feature: plugin limit-count support to use redis cluster (#2406)

Browse Source 
fix #638

Co-authored-by: liuheng <liuhengloveyou@gmail.com>
This commit is contained in:
YuanSheng Wang 2020-10-17 08:21:47 +08:00 committed by GitHub
parent 7cb956302f
commit c65f5c9465
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 635 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.github/workflows/build.yml vendored
View File

@ -35,6 +35,25 @@ jobs:
if: matrix.platform == 'ubuntu-18.04'
run: sudo ./.travis/${{ matrix.os_name }}_runner.sh before_install
- name: Install Redis Cluster
if: matrix.os_name == 'linux_openresty'
uses: vishnudxb/redis-cluster@1.0.5
with:
master1-port: 5000
master2-port: 5001
master3-port: 5002
slave1-port: 5003
slave2-port: 5004
slave3-port: 5005
- name: Running Redis Cluster Test
if: matrix.os_name == 'linux_openresty'
run: |
sudo apt-get install -y redis-tools
docker ps -a
redis-cli -h 127.0.0.1 -p 5000 ping
redis-cli -h 127.0.0.1 -p 5000 cluster nodes
- name: Linux Install
if: matrix.platform == 'ubuntu-18.04'
run: sudo ./.travis/${{ matrix.os_name }}_runner.sh do_install

37
apisix/plugins/limit-count.lua
View File

@ -17,10 +17,14 @@
local limit_local_new = require("resty.limit.count").new
local core = require("apisix.core")
local plugin_name = "limit-count"
local limit_redis_cluster_new
local limit_redis_new
do
local redis_src = "apisix.plugins.limit-count.limit-count-redis"
limit_redis_new = require(redis_src).new
local cluster_src = "apisix.plugins.limit-count.limit-count-redis-cluster"
limit_redis_cluster_new = require(cluster_src).new
end
@ -40,7 +44,7 @@ local schema = {
},
policy = {
type = "string",
enum = {"local", "redis"},
enum = {"local", "redis", "redis-cluster"},
default = "local",
}
},
@ -70,11 +74,31 @@ local schema = {
type = "string", minLength = 0,
},
redis_timeout = {
type = "integer", minimum = 1,
default = 1000,
type = "integer", minimum = 1, default = 1000,
},
},
required = {"redis_host"},
},
{
properties = {
policy = {
enum = {"redis-cluster"},
},
redis_cluster_nodes = {
type = "array",
minItems = 2,
items = {
type = "string", minLength = 2, maxLength = 100
},
},
redis_password = {
type = "string", minLength = 0,
},
redis_timeout = {
type = "integer", minimum = 1, default = 1000,
},
},
required = {"redis_cluster_nodes"},
}
}
}
@ -83,7 +107,7 @@ local schema = {
local _M = {
version = 0.3,
version = 0.4,
priority = 1002,
name = plugin_name,
schema = schema,
@ -119,6 +143,11 @@ local function create_limit_obj(conf)
conf.count, conf.time_window, conf)
end
if conf.policy == "redis-cluster" then
return limit_redis_cluster_new("plugin-" .. plugin_name, conf.count,
conf.time_window, conf)
end
return nil
end

134
apisix/plugins/limit-count/limit-count-redis-cluster.lua Normal file
View File

@ -0,0 +1,134 @@
--
-- Licensed to the Apache Software Foundation (ASF) under one or more
-- contributor license agreements. See the NOTICE file distributed with
-- this work for additional information regarding copyright ownership.
-- The ASF licenses this file to You under the Apache License, Version 2.0
-- (the "License"); you may not use this file except in compliance with
-- the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
local rediscluster = require("resty.rediscluster")
local core = require("apisix.core")
local resty_lock = require("resty.lock")
local setmetatable = setmetatable
local tostring = tostring
local ipairs = ipairs
local _M = {}
local mt = {
__index = _M
}
local function new_redis_cluster(conf)
local config = {
name = "apisix-redis-cluster",
serv_list = {},
read_timeout = conf.redis_timeout,
auth = conf.redis_password,
dict_name = "plugin-limit-count-redis-cluster-slot-lock",
}
for i, conf_item in ipairs(conf.redis_cluster_nodes) do
local host, port, err = core.utils.parse_addr(conf_item)
if err then
return nil, "failed to parse address: " .. conf_item
.. " err: " .. err
end
config.serv_list[i] = {ip = host, port = port}
end
local red_cli, err = rediscluster:new(config)
if not red_cli then
return nil, "failed to new redis cluster: " .. err
end
return red_cli
end
function _M.new(plugin_name, limit, window, conf)
local red_cli, err = new_redis_cluster(conf)
if not red_cli then
return nil, err
end
local self = {
limit = limit, window = window, conf = conf,
plugin_name = plugin_name, red_cli =red_cli
}
return setmetatable(self, mt)
end
function _M.incoming(self, key)
local red = self.red_cli
local limit = self.limit
local window = self.window
local remaining
key = self.plugin_name .. tostring(key)
local ret, err = red:ttl(key)
if not ret then
return false, "failed to get redis `" .. key .."` ttl: " .. err
end
core.log.info("ttl key: ", key, " ret: ", ret, " err: ", err)
if ret < 0 then
local lock, err = resty_lock:new("plugin-limit-count")
if not lock then
return false, "failed to create lock: " .. err
end
local elapsed, err = lock:lock(key)
if not elapsed then
return false, "failed to acquire the lock: " .. err
end
ret = red:ttl(key)
if ret < 0 then
local ok, err = lock:unlock()
if not ok then
return false, "failed to unlock: " .. err
end
ret, err = red:set(key, limit -1, "EX", window)
if not ret then
return nil, err
end
return 0, limit -1
end
local ok, err = lock:unlock()
if not ok then
return false, "failed to unlock: " .. err
end
end
remaining, err = red:incrby(key, -1)
if not remaining then
return nil, err
end
if remaining < 0 then
return nil, "rejected"
end
return 0, remaining
end
return _M

5
apisix/plugins/limit-count/limit-count-redis.lua
View File

@ -97,12 +97,13 @@ function _M.incoming(self, key)
return false, "failed to unlock: " .. err
end
ret, err = red:set(key, limit -1, "EX", window)
limit = limit -1
ret, err = red:set(key, limit, "EX", window)
if not ret then
return nil, err
end
return 0, limit -1
return 0, limit
end
ok, err = lock:unlock()

1
bin/apisix
View File

@ -190,6 +190,7 @@ http {
lua_shared_dict balancer_ewma 10m;
lua_shared_dict balancer_ewma_locks 10m;
lua_shared_dict balancer_ewma_last_touched_at 10m;
lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m;
# for openid-connect plugin
lua_shared_dict discovery 1m; # cache for discovery metadata documents

37
doc/plugins/limit-count.md
View File

@ -39,11 +39,12 @@ Limit request rate by a fixed number of requests in a given time window.
| time_window | integer | required | | [0,...] | the time window in seconds before the request count is reset. |
| key | string | required | | ["remote_addr", "server_addr", "http_x_real_ip", "http_x_forwarded_for"] | the user specified key to limit the rate. |
| rejected_code | integer | optional | 503 | [200,600] | The HTTP status code returned when the request exceeds the threshold is rejected, default 503. |
| policy | string | optional | "local" | ["local", "redis"] | The rate-limiting policies to use for retrieving and incrementing the limits. Available values are `local`(the counters will be stored locally in-memory on the node) and `redis`(counters are stored on a Redis server and will be shared across the nodes, usually used it to do the global speed limit). |
| policy | string | optional | "local" | ["local", "redis", "redis-cluster"] | The rate-limiting policies to use for retrieving and incrementing the limits. Available values are `local`(the counters will be stored locally in-memory on the node) and `redis`(counters are stored on a Redis server and will be shared across the nodes, usually use it to do the global speed limit). |
| redis_host | string | required for `redis` | | | When using the `redis` policy, this property specifies the address of the Redis server. |
| redis_port | integer | optional | 6379 | [1,...] | When using the `redis` policy, this property specifies the port of the Redis server. |
| redis_password | string | optional | | | When using the `redis` policy, this property specifies the password of the Redis server. |
| redis_timeout | integer | optional | 1000 | [1,...] | When using the `redis` policy, this property specifies the timeout in milliseconds of any command submitted to the Redis server. |
| redis_cluster_nodes | array | optional | | | When using `redis-cluster` policy，This property is a list of addresses of Redis cluster service nodes. |
**Key can be customized by the user, only need to modify a line of code of the plug-in to complete. It is a security consideration that is not open in the plugin.**
@ -81,7 +82,7 @@ Then add limit-count plugin:
If you need a cluster-level precision traffic limit, then we can do it with the redis server. The rate limit of the traffic will be shared between different APISIX nodes to limit the rate of cluster traffic.
Here is the example:
Here is the example if we use single `redis` policy:
```shell
curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
@ -109,6 +110,34 @@ curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335
}'
```
If using `redis-cluster` policy:
```shell
curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"uri": "/index.html",
"plugins": {
"limit-count": {
"count": 2,
"time_window": 60,
"rejected_code": 503,
"key": "remote_addr",
"policy": "redis-cluster",
"redis_cluster_nodes": [
"127.0.0.1:5000",
"127.0.0.1:5001"
]
}
},
"upstream": {
"type": "roundrobin",
"nodes": {
"39.97.63.215:80": 1
}
}
}'
```
## Test Plugin
The above configuration limits access to only 2 times in 60 seconds. The first two visits will be normally:
@ -120,7 +149,7 @@ curl -i http://127.0.0.1:9080/index.html
The response header contains `X-RateLimit-Limit` and `X-RateLimit-Remaining`,
which mean the total number of requests and the remaining number of requests that can be sent:
```
```shell
HTTP/1.1 200 OK
Content-Type: text/html
Content-Length: 13175
@ -132,7 +161,7 @@ Server: APISIX web server
When you visit for the third time, you will receive a response with the 503 HTTP code:
```
```shell
HTTP/1.1 503 Service Temporarily Unavailable
Content-Type: text/html
Content-Length: 194

58
doc/zh-cn/plugins/limit-count.md
View File

@ -26,18 +26,18 @@
## 参数
| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
| -------------- | ------- | ------------ | ------- | ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| count | integer | 必须 | | [0,...] | 指定时间窗口内的请求数量阈值 |
| time_window | integer | 必须 | | [0,...] | 时间窗口的大小（以秒为单位），超过这个时间就会重置 |
| key | string | 必须 | | ["remote_addr", "server_addr", "http_x_real_ip", "http_x_forwarded_for"] | 用来做请求计数的依据 |
| rejected_code | integer | 可选 | 503 | [200,600] | 当请求超过阈值被拒绝时，返回的 HTTP 状态码 |
| policy | string | 可选 | "local" | ["local", "redis"] | 用于检索和增加限制的速率限制策略。可选的值有：`local`(计数器被以内存方式保存在节点本地，默认选项) 和 `redis`(计数器保存在 Redis 服务节点上，从而可以跨节点共享结果，通常用它来完成全局限速) |
| redis_host | string | `redis` 必须 | | | 当使用 `redis` 限速策略时，该属性是 Redis 服务节点的地址。 |
| redis_port | integer | 可选 | 6379 | [1,...] | 当使用 `redis` 限速策略时，该属性是 Redis 服务节点的端口 |
| redis_password | string | 可选 | | | 当使用 `redis` 限速策略时，该属性是 Redis 服务节点的密码。 |
| redis_timeout | integer | 可选 | 1000 | [1,...] | 当使用 `redis` 限速策略时，该属性是 Redis 服务节点以毫秒为单位的超时时间 |
| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
| --------------- | -------- | ------------ | ------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
| count | integer | 必须 | | [0,...] | 指定时间窗口内的请求数量阈值 |
| time_window | integer | 必须 | | [0,...] | 时间窗口的大小（以秒为单位），超过这个时间就会重置 |
| key | string | 必须 | | ["remote_addr", "server_addr", "http_x_real_ip", "http_x_forwarded_for"] | 用来做请求计数的依据 |
| rejected_code | integer | 可选 | 503 | [200,600] | 当请求超过阈值被拒绝时，返回的 HTTP 状态码 |
| policy | string | 可选 | "local" | ["local", "redis", "redis-cluster"] | 用于检索和增加限制的速率限制策略。可选的值有：`local`(计数器被以内存方式保存在节点本地，默认选项) 和 `redis`(计数器保存在 Redis 服务节点上，从而可以跨节点共享结果，通常用它来完成全局限速)；以及`redis-cluster`，跟redis功能一样，只是使用redis集群方式。 |
| redis_host | string | `redis` 必须 | | | 当使用 `redis` 限速策略时，该属性是 Redis 服务节点的地址。 |
| redis_port | integer | 可选 | 6379 | [1,...] | 当使用 `redis` 限速策略时，该属性是 Redis 服务节点的端口 |
| redis_password | string | 可选 | | | 当使用 `redis` 限速策略时，该属性是 Redis 服务节点的密码。 |
| redis_timeout | integer | 可选 | 1000 | [1,...] | 当使用 `redis` 限速策略时，该属性是 Redis 服务节点以毫秒为单位的超时时间 |
| redis_cluster_nodes | array | 可选 | | | 当使用 `redis-cluster` 限速策略时，该属性是 Redis 集群服务节点的地址列表。 |
**key 是可以被用户自定义的，只需要修改插件的一行代码即可完成。并没有在插件中放开是处于安全的考虑。**
@ -76,7 +76,7 @@ curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335
如果你需要一个集群级别的流量控制，我们可以借助 redis server 来完成。不同的 APISIX 节点之间将共享流量限速结果，实现集群流量限速。
请看下面例子：
如果启用单 redis 策略，请看下面例子：
```shell
curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
@ -104,6 +104,34 @@ curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335
}'
```
如果使用 `redis-cluster` 策略:
```shell
curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"uri": "/index.html",
"plugins": {
"limit-count": {
"count": 2,
"time_window": 60,
"rejected_code": 503,
"key": "remote_addr",
"policy": "redis-cluster",
"redis_cluster_nodes": [
"127.0.0.1:5000",
"127.0.0.1:5001"
]
}
},
"upstream": {
"type": "roundrobin",
"nodes": {
"39.97.63.215:80": 1
}
}
}'
```
#### 测试插件
上述配置限制了 60 秒内只能访问 2 次，前两次访问都会正常访问：
@ -114,7 +142,7 @@ curl -i http://127.0.0.1:9080/index.html
响应头里面包含了 `X-RateLimit-Limit` 和 `X-RateLimit-Remaining`，他们的含义分别是限制的总请求数和剩余还可以发送的请求数：
```
```shell
HTTP/1.1 200 OK
Content-Type: text/html
Content-Length: 13175
@ -126,7 +154,7 @@ Server: APISIX web server
当你第三次访问的时候，就会收到包含 503 返回码的响应头：
```
```shell
HTTP/1.1 503 Service Temporarily Unavailable
Content-Type: text/html
Content-Length: 194

1
rockspec/apisix-master-0.rockspec
View File

@ -54,6 +54,7 @@ dependencies = {
"skywalking-nginx-lua-plugin = 1.0-0",
"base64 = 1.5-2",
"dkjson = 2.5-2",
"resty-redis-cluster = 1.02-4",
}
build = {

5
t/APISIX.pm vendored
View File

@ -137,6 +137,10 @@ env ENABLE_ETCD_AUTH;
env APISIX_PROFILE;
_EOC_
# set default `timeout` to 5sec
my $timeout = $block->timeout // 5;
$block->set_value("timeout", $timeout);
$block->set_value("main_config", $main_config);
my $stream_enable = $block->stream_enable;
@ -236,6 +240,7 @@ _EOC_
lua_shared_dict balancer_ewma 1m;
lua_shared_dict balancer_ewma_locks 1m;
lua_shared_dict balancer_ewma_last_touched_at 1m;
lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m;
resolver $dns_addrs_str;
resolver_timeout 5;

363
t/plugin/limit-count-redis-cluster.t Normal file
View File

@ -0,0 +1,363 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
use t::APISIX 'no_plan';
repeat_each(1);
no_long_string();
no_shuffle();
no_root_location();
run_tests;
__DATA__
=== TEST 1: set route, missing redis host
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"plugins": {
"limit-count": {
"count": 2,
"time_window": 60,
"rejected_code": 503,
"key": "remote_addr",
"policy": "redis-cluster"
}
},
"upstream": {
"nodes": {
"127.0.0.1:1980": 1
},
"type": "roundrobin"
},
"uri": "/hello"
}]]
)
if code >= 300 then
ngx.status = code
end
ngx.print(body)
}
}
--- request
GET /t
--- error_code: 400
--- response_body
{"error_msg":"failed to check the configuration of plugin limit-count err: failed to validate dependent schema for \"policy\": value should match only one schema, but matches none"}
--- no_error_log
[error]
=== TEST 2: set route, with redis host and port
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"uri": "/hello",
"plugins": {
"limit-count": {
"count": 2,
"time_window": 60,
"rejected_code": 503,
"key": "remote_addr",
"policy": "redis-cluster",
"redis_timeout": 1001,
"redis_cluster_nodes": [
"127.0.0.1:5000",
"127.0.0.1:5001"
]
}
},
"upstream": {
"nodes": {
"127.0.0.1:1980": 1
},
"type": "roundrobin"
}
}]]
)
if code >= 300 then
ngx.status = code
end
ngx.say(body)
}
}
--- request
GET /t
--- response_body
passed
--- no_error_log
[error]
=== TEST 3: set route
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"uri": "/hello",
"plugins": {
"limit-count": {
"count": 2,
"time_window": 60,
"rejected_code": 503,
"key": "remote_addr",
"policy": "redis-cluster",
"redis_cluster_nodes": [
"127.0.0.1:5000",
"127.0.0.1:5001"
]
}
},
"upstream": {
"nodes": {
"127.0.0.1:1980": 1
},
"type": "roundrobin"
}
}]],
[[{
"node": {
"value": {
"plugins": {
"limit-count": {
"count": 2,
"time_window": 60,
"rejected_code": 503,
"key": "remote_addr",
"policy": "redis-cluster",
"redis_timeout": 1000,
"redis_cluster_nodes": [
"127.0.0.1:5000",
"127.0.0.1:5001"
]
}
},
"upstream": {
"nodes": {
"127.0.0.1:1980": 1
},
"type": "roundrobin"
},
"uri": "/hello"
},
"key": "/apisix/routes/1"
},
"action": "set"
}]]
)
if code >= 300 then
ngx.status = code
end
ngx.say(body)
}
}
--- request
GET /t
--- response_body
passed
--- no_error_log
[error]
=== TEST 4: up the limit
--- pipelined_requests eval
["GET /hello", "GET /hello", "GET /hello", "GET /hello"]
--- error_code eval
[200, 200, 503, 503]
--- no_error_log
[error]
=== TEST 5: up the limit again
--- pipelined_requests eval
["GET /hello1", "GET /hello", "GET /hello2", "GET /hello", "GET /hello"]
--- error_code eval
[404, 503, 404, 503, 503]
--- no_error_log
[error]
=== TEST 6: set route, four redis nodes, only one is valid
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"uri": "/hello",
"plugins": {
"limit-count": {
"count": 9999,
"time_window": 60,
"key": "http_x_real_ip",
"policy": "redis-cluster",
"redis_cluster_nodes": [
"127.0.0.1:5000",
"127.0.0.1:8001",
"127.0.0.1:8002",
"127.0.0.1:8003"
]
}
},
"upstream": {
"nodes": {
"127.0.0.1:1980": 1
},
"type": "roundrobin"
}
}]]
)
if code >= 300 then
ngx.status = code
end
ngx.say(body)
}
}
--- request
GET /t
--- response_body
passed
--- no_error_log
[error]
=== TEST 7: hit route
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
for i = 1, 20 do
local code, body = t('/hello', ngx.HTTP_GET)
ngx.say("code: ", code)
end
}
}
--- request
GET /t
--- response_body
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
code: 200
--- no_error_log
[error]
--- timeout: 10
=== TEST 8: update route, use new limit configuration
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local function set_route(count)
t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"uri": "/hello",
"plugins": {
"limit-count": {
"count": ]] .. count .. [[,
"time_window": 60,
"key": "http_x_real_ip",
"policy": "redis-cluster",
"redis_cluster_nodes": [
"127.0.0.1:5000",
"127.0.0.1:5001"
]
}
},
"upstream": {
"nodes": {
"127.0.0.1:1980": 1
},
"type": "roundrobin"
}
}]]
)
end
set_route(2)
local t = require("lib.test_admin").test
for i = 1, 5 do
local code, body = t('/hello', ngx.HTTP_GET)
ngx.say("code: ", code)
end
set_route(3)
local t = require("lib.test_admin").test
for i = 1, 5 do
local code, body = t('/hello', ngx.HTTP_GET)
ngx.say("code: ", code)
end
}
}
--- request
GET /t
--- response_body
code: 200
code: 200
code: 503
code: 503
code: 503
code: 200
code: 200
code: 200
code: 503
code: 503
--- no_error_log
[error]