feat(ext-plugin): avoid sending conf request more times (#5183)

apisix/cli/ngx_tpl.lua

@@ -191,6 +191,9 @@ http {
     # for authz-keycloak
     lua_shared_dict access-tokens {* http.lua_shared_dict["access-tokens"] *}; # cache for service account access tokens
 
+    # for ext-plugin
+    lua_shared_dict ext-plugin {* http.lua_shared_dict["ext-plugin"] *}; # cache for ext-plugin
+
     # for custom shared dict
     {% if http.custom_lua_shared_dict then %}
     {% for cache_key, cache_size in pairs(http.custom_lua_shared_dict) do %}

apisix/plugins/ext-plugin/init.lua

@@ -60,6 +60,7 @@ local ipairs = ipairs
 local pairs = pairs
 local tostring = tostring
 local type = type
+local dict = ngx.shared["ext-plugin"]
 
 
 local events_list
@@ -292,6 +293,44 @@ local function handle_extra_info(ctx, input)
 end
 
 
+local function fetch_token(key)
+    if dict then
+        return dict:get(key)
+    else
+        core.log.error('shm "ext-plugin" not found')
+        return nil
+    end
+end
+
+
+local function store_token(key, token)
+    if dict then
+        local exp = helper.get_conf_token_cache_time()
+        -- early expiry, lrucache in critical state sends prepare_conf_req as original behaviour
+        exp = exp * 0.9
+        local success, err, forcible = dict:set(key, token, exp)
+        if not success then
+            core.log.error("ext-plugin:failed to set conf token, err: ", err)
+        end
+        if forcible then
+            core.log.warn("ext-plugin:set valid items forcibly overwritten")
+        end
+    else
+        core.log.error('shm "ext-plugin" not found')
+    end
+end
+
+
+local function flush_token()
+    if dict then
+        core.log.warn("flush conf token in shared dict")
+        dict:flush_all()
+    else
+        core.log.error('shm "ext-plugin" not found')
+    end
+end
+
+
 local rpc_call
 local rpc_handlers = {
     nil,
@@ -300,6 +339,12 @@ local rpc_handlers = {
 
         local key = builder:CreateString(unique_key)
 
+        local token = fetch_token(key)
+        if token then
+            core.log.info("fetch token from shared dict, token: ", token)
+            return token
+        end
+
         local conf_vec
         if conf.conf then
             local len = #conf.conf
@@ -344,9 +389,10 @@ local rpc_handlers = {
 
         local buf = flatbuffers.binaryArray.New(resp)
         local pcr = prepare_conf_resp.GetRootAsResp(buf, 0)
-        local token = pcr:ConfToken()
+        token = pcr:ConfToken()
 
         core.log.notice("get conf token: ", token, " conf: ", core.json.delay_encode(conf.conf))
+        store_token(key, token)
         return token
     end,
     function (conf, ctx, sock, entry)
@@ -470,7 +516,6 @@ local rpc_handlers = {
         local buf = flatbuffers.binaryArray.New(resp)
         local call_resp = http_req_call_resp.GetRootAsResp(buf, 0)
         local action_type = call_resp:ActionType()
-
         if action_type == http_req_call_action.Stop then
             local action = call_resp:Action()
             local stop = http_req_call_stop.New()
@@ -588,6 +633,8 @@ end
 
 
 local function create_lrucache()
+    flush_token()
+
     if lrucache then
         core.log.warn("flush conf token lrucache")
     end

conf/config-default.yaml

@@ -253,6 +253,7 @@ nginx_config:                     # config for render the template to generate n
       jwks: 1m
       introspection: 10m
       access-tokens: 1m
+      ext-plugin: 1m
 
 etcd:
   host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.

t/APISIX.pm

@@ -445,6 +445,7 @@ _EOC_
     lua_shared_dict plugin-api-breaker 10m;
     lua_capture_error_log 1m;    # plugin error-log-logger
     lua_shared_dict etcd-cluster-health-check 10m; # etcd health check
+    lua_shared_dict ext-plugin 1m;
 
     proxy_ssl_name \$upstream_host;
     proxy_ssl_server_name on;

t/plugin/ext-plugin/conf_token.t

@@ -0,0 +1,145 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+workers(8);
+repeat_each(1);
+no_long_string();
+no_root_location();
+no_shuffle();
+log_level("info");
+worker_connections(10240);
+
+$ENV{"PATH"} = $ENV{PATH} . ":" . $ENV{TEST_NGINX_HTML_DIR};
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+
+    $block->set_value("stream_conf_enable", 1);
+
+    if (!defined $block->extra_stream_config) {
+        my $stream_config = <<_EOC_;
+    server {
+        listen unix:\$TEST_NGINX_HTML_DIR/nginx.sock;
+
+        content_by_lua_block {
+            local ext = require("lib.ext-plugin")
+            ext.go({})
+        }
+    }
+
+_EOC_
+        $block->set_value("extra_stream_config", $stream_config);
+    }
+
+    my $unix_socket_path = $ENV{"TEST_NGINX_HTML_DIR"} . "/nginx.sock";
+    my $orig_extra_yaml_config = $block->extra_yaml_config // "";
+    my $cmd = $block->ext_plugin_cmd // "['sleep', '5s']";
+    my $extra_yaml_config = <<_EOC_;
+ext-plugin:
+    path_for_test: $unix_socket_path
+    cmd: $cmd
+_EOC_
+    $extra_yaml_config = $extra_yaml_config . $orig_extra_yaml_config;
+
+    $block->set_value("extra_yaml_config", $extra_yaml_config);
+
+    if (!$block->request) {
+        $block->set_value("request", "GET /t");
+    }
+
+    if (!$block->error_log) {
+        $block->set_value("no_error_log", "[error]\n[alert]");
+    }
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: sanity
+--- config
+    location /t {
+        content_by_lua_block {
+            local json = require("toolkit.json")
+            local t = require("lib.test_admin")
+
+            local code, message, res = t.test('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                 [[{
+                    "uri": "/hello",
+                    "plugins": {
+                        "ext-plugin-pre-req": {"a":"b"}
+                    },
+                    "upstream": {
+                        "nodes": {
+                            "127.0.0.1:1980": 1
+                        },
+                        "type": "roundrobin"
+                    }
+                }]]
+            )
+
+            if code >= 300 then
+                ngx.status = code
+                ngx.say(message)
+                return
+            end
+
+            ngx.say(message)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 2: share conf token in different workers
+--- ext_plugin_cmd
+["t/plugin/ext-plugin/runner.sh", "3600"]
+--- config
+    location /t {
+        content_by_lua_block {
+            local http = require "resty.http"
+            local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello"
+
+            local t = {}
+            for i = 1, 180 do
+                local th = assert(ngx.thread.spawn(function(i)
+                    local httpc = http.new()
+                    local res, err = httpc:request_uri(uri)
+                    if not res then
+                        ngx.log(ngx.ERR, err)
+                        return
+                    end
+                end, i))
+                table.insert(t, th)
+            end
+            for i, th in ipairs(t) do
+                ngx.thread.wait(th)
+            end
+            ngx.say("done")
+        }
+    }
+--- response_body
+done
+--- grep_error_log eval
+qr/fetch token from shared dict, token: 233/
+--- grep_error_log_out eval
+qr/(fetch token from shared dict, token: 233){1,}/
+--- no_error_log
+[error]

t/plugin/ext-plugin/sanity.t

@@ -270,6 +270,7 @@ sending rpc type: 1 data length:
 receiving rpc type: 1 data length:
 --- error_log
 flush conf token lrucache
+flush conf token in shared dict
 --- no_error_log
 [error]
 
@@ -382,6 +383,7 @@ hello world
     }
 --- error_log
 refresh cache and try again
+flush conf token in shared dict
 --- no_error_log
 [error]