From 7617e19982cde511e3669a1acbea2682f91ef450 Mon Sep 17 00:00:00 2001 From: roketyyang Date: Wed, 26 Jan 2022 11:09:42 +0800 Subject: [PATCH] feat: add opentelemetry plugin (#6119) Co-authored-by: roketyyang --- apisix/plugins/opentelemetry.lua | 347 ++++++++++++ conf/config-default.yaml | 16 + docs/en/latest/config.json | 1 + docs/en/latest/plugins/opentelemetry.md | 153 +++++ docs/zh/latest/config.json | 1 + docs/zh/latest/plugins/opentelemetry.md | 151 +++++ rockspec/apisix-master-0.rockspec | 1 + t/plugin/opentelemetry.t | 722 ++++++++++++++++++++++++ 8 files changed, 1392 insertions(+) create mode 100644 apisix/plugins/opentelemetry.lua create mode 100644 docs/en/latest/plugins/opentelemetry.md create mode 100644 docs/zh/latest/plugins/opentelemetry.md create mode 100644 t/plugin/opentelemetry.t diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua new file mode 100644 index 00000000..12d61c99 --- /dev/null +++ b/apisix/plugins/opentelemetry.lua @@ -0,0 +1,347 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local plugin_name = "opentelemetry" +local core = require("apisix.core") +local plugin = require("apisix.plugin") +local process = require("ngx.process") + +local always_off_sampler_new = require("opentelemetry.trace.sampling.always_off_sampler").new +local always_on_sampler_new = require("opentelemetry.trace.sampling.always_on_sampler").new +local parent_base_sampler_new = require("opentelemetry.trace.sampling.parent_base_sampler").new +local trace_id_ratio_sampler_new = + require("opentelemetry.trace.sampling.trace_id_ratio_sampler").new + +local exporter_client_new = require("opentelemetry.trace.exporter.http_client").new +local otlp_exporter_new = require("opentelemetry.trace.exporter.otlp").new +local batch_span_processor_new = require("opentelemetry.trace.batch_span_processor").new +local id_generator = require("opentelemetry.trace.id_generator") +local tracer_provider_new = require("opentelemetry.trace.tracer_provider").new + +local span_kind = require("opentelemetry.trace.span_kind") +local span_status = require("opentelemetry.trace.span_status") +local resource_new = require("opentelemetry.resource").new +local attr = require("opentelemetry.attribute") + +local context_storage = require("opentelemetry.context_storage") +local context = require("opentelemetry.context").new(context_storage) +local carrier_new = require("opentelemetry.trace.propagation.carrier").new +local trace_context = require("opentelemetry.trace.propagation.trace_context") + +local ngx = ngx +local ngx_var = ngx.var +local table = table +local type = type +local pairs = pairs +local ipairs = ipairs +local unpack = unpack + +local lrucache = core.lrucache.new({ + type = 'plugin', count = 128, ttl = 24 * 60 * 60, +}) + + +local attr_schema = { + type = "object", + properties = { + trace_id_source = { + type = "string", + enum = {"x-request-id", "random"}, + description = "the source of trace id", + default = "random", + }, + resource = { + type = "object", + description = "additional resource", + additionalProperties = {{type = "boolean"}, {type = "number"}, {type = "string"}}, + }, + collector = { + type = "object", + description = "opentelemetry collector", + properties = { + address = {type = "string", description = "host:port", default = "127.0.0.1:4317"}, + request_timeout = {type = "integer", description = "second uint", default = 3}, + request_headers = { + type = "object", + description = "http headers", + additionalProperties = { + one_of = {{type = "boolean"},{type = "number"}, {type = "string"}}, + }, + } + }, + default = {address = "127.0.0.1:4317", request_timeout = 3} + }, + batch_span_processor = { + type = "object", + description = "batch span processor", + properties = { + drop_on_queue_full = { + type = "boolean", + description = "if true, drop span when queue is full," + .. " otherwise force process batches", + }, + max_queue_size = { + type = "integer", + description = "maximum queue size to buffer spans for delayed processing", + }, + batch_timeout = { + type = "number", + description = "maximum duration for constructing a batch", + }, + inactive_timeout = { + type = "number", + description = "maximum duration for processing batches", + }, + max_export_batch_size = { + type = "integer", + description = "maximum number of spans to process in a single batch", + } + }, + default = {}, + }, + }, +} + +local schema = { + type = "object", + properties = { + sampler = { + type = "object", + properties = { + name = { + type = "string", + enum = {"always_on", "always_off", "trace_id_ratio", "parent_base"}, + title = "sampling strategy", + default = "always_off" + }, + options = { + type = "object", + properties = { + fraction = { + type = "number", title = "trace_id_ratio fraction", default = 0 + }, + root = { + type = "object", + title = "parent_base root sampler", + properties = { + name = { + type = "string", + enum = {"always_on", "always_off", "trace_id_ratio"}, + title = "sampling strategy", + default = "always_off" + }, + options = { + type = "object", + properties = { + fraction = { + type = "number", + title = "trace_id_ratio fraction parameter", + default = 0, + }, + }, + default = {fraction = 0} + } + }, + default = {name = "always_off", options = {fraction = 0}} + }, + }, + default = {fraction = 0, root = {name = "always_off"}} + } + }, + default = {name = "always_off", options = {fraction = 0, root = {name = "always_off"}}} + }, + additional_attributes = { + type = "array", + items = { + type = "string", + minLength = 1, + } + } + } +} + + +local _M = { + version = 0.1, + priority = -1200, -- last running plugin, but before serverless post func + name = plugin_name, + schema = schema, + attr_schema = attr_schema, +} + + +function _M.check_schema(conf) + return core.schema.check(schema, conf) +end + + +local hostname +local sampler_factory +local plugin_info + +function _M.init() + if process.type() ~= "worker" then + return + end + + sampler_factory = { + always_off = always_off_sampler_new, + always_on = always_on_sampler_new, + parent_base = parent_base_sampler_new, + trace_id_ratio = trace_id_ratio_sampler_new, + } + hostname = core.utils.gethostname() + + plugin_info = plugin.plugin_attr(plugin_name) or {} + local ok, err = core.schema.check(attr_schema, plugin_info) + if not ok then + core.log.error("failed to check the plugin_attr[", plugin_name, "]", + ": ", err) + return + end + + if plugin_info.trace_id_source == "x-request-id" then + id_generator.new_ids = function() + local trace_id = core.request.headers()["x-request-id"] or ngx_var.request_id + return trace_id, id_generator.new_span_id() + end + end +end + + +local function create_tracer_obj(conf) + -- create exporter + local exporter = otlp_exporter_new(exporter_client_new(plugin_info.collector.address, + plugin_info.collector.request_timeout, + plugin_info.collector.request_headers)) + -- create span processor + local batch_span_processor = batch_span_processor_new(exporter, + plugin_info.batch_span_processor) + -- create sampler + local sampler + local sampler_name = conf.sampler.name + local sampler_options = conf.sampler.options + if sampler_name == "parent_base" then + local root_sampler + if sampler_options.root then + local name, fraction = sampler_options.root.name, sampler_options.root.options.fraction + root_sampler = sampler_factory[name](fraction) + else + root_sampler = always_off_sampler_new() + end + sampler = sampler_factory[sampler_name](root_sampler) + else + sampler = sampler_factory[sampler_name](sampler_options.fraction) + end + local resource_attrs = {attr.string("hostname", hostname)} + if plugin_info.resource then + if not plugin_info.resource["service.name"] then + table.insert(resource_attrs, attr.string("service.name", "APISIX")) + end + for k, v in pairs(plugin_info.resource) do + if type(v) == "string" then + table.insert(resource_attrs, attr.string(k, v)) + end + if type(v) == "number" then + table.insert(resource_attrs, attr.double(k, v)) + end + if type(v) == "boolean" then + table.insert(resource_attrs, attr.bool(k, v)) + end + end + end + -- create tracer provider + local tp = tracer_provider_new(batch_span_processor, { + resource = resource_new(unpack(resource_attrs)), + sampler = sampler, + }) + -- create tracer + return tp:tracer("opentelemetry-lua") +end + + +function _M.rewrite(conf, api_ctx) + local tracer, err = core.lrucache.plugin_ctx(lrucache, api_ctx, nil, create_tracer_obj, conf) + if not tracer then + core.log.error("failed to fetch tracer object: ", err) + return + end + + -- extract trace context from the headers of downstream HTTP request + local upstream_context = trace_context.extract(context, carrier_new()) + local attributes = { + attr.string("service", api_ctx.service_name), + attr.string("route", api_ctx.route_name), + } + if conf.additional_attributes then + for _, key in ipairs(conf.additional_attributes) do + local val = api_ctx.var[key] + if val then + core.table.insert(attributes, attr.string(key, val)) + end + end + end + + local ctx = tracer:start(upstream_context, api_ctx.var.request_uri, { + kind = span_kind.client, + attributes = attributes, + }) + ctx:attach() + + -- inject trace context into the headers of upstream HTTP request + trace_context.inject(ctx, carrier_new()) +end + + +function _M.body_filter(conf, api_ctx) + if ngx.arg[2] then + local upstream_status = core.response.get_upstream_status(api_ctx) + local ctx = context:current() + ctx:detach() + + -- get span from current context + local span = ctx:span() + if upstream_status and upstream_status >= 500 then + span:set_status(span_status.error, + "upstream response status: " .. upstream_status) + end + + span:finish() + end +end + + +-- body_filter maybe not called because of empty http body response +-- so we need to check if the span has finished in log phase +function _M.log(conf, api_ctx) + local ctx = context:current() + if ctx then + -- ctx:detach() is not necessary, because of ctx is stored in ngx.ctx + local upstream_status = core.response.get_upstream_status(api_ctx) + + -- get span from current context + local span = ctx:span() + if upstream_status and upstream_status >= 500 then + span:set_status(span_status.error, + "upstream response status: " .. upstream_status) + end + + span:finish() + end +end + + +return _M diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 3383d446..2a12930d 100644 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -385,6 +385,7 @@ plugins: # plugin list (sorted by priority) # <- recommend to use priority (0, 100) for your custom plugins - example-plugin # priority: 0 #- skywalking # priority: -1100 + #- opentelemetry # priority: -1200 - aws-lambda # priority: -1899 - azure-functions # priority: -1900 - openwhisk # priority: -1901 @@ -412,6 +413,21 @@ plugin_attr: service_name: APISIX service_instance_name: APISIX Instance Name endpoint_addr: http://127.0.0.1:12800 + opentelemetry: + trace_id_source: x-request-id + resource: + service.name: APISIX + collector: + address: 127.0.0.1:4317 + request_timeout: 3 + request_headers: + Authorization: token + batch_span_processor: + drop_on_queue_full: false + max_queue_size: 1024 + batch_timeout: 2 + inactive_timeout: 1 + max_export_batch_size: 16 prometheus: export_uri: /apisix/prometheus/metrics metric_prefix: apisix_ diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index 28e48478..41cf2fc4 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -109,6 +109,7 @@ "plugins/prometheus", "plugins/zipkin", "plugins/skywalking", + "plugins/opentelemetry", "plugins/node-status", "plugins/datadog" ] diff --git a/docs/en/latest/plugins/opentelemetry.md b/docs/en/latest/plugins/opentelemetry.md new file mode 100644 index 00000000..d2610eb0 --- /dev/null +++ b/docs/en/latest/plugins/opentelemetry.md @@ -0,0 +1,153 @@ +--- +title: opentelemetry +--- + + + +## Summary + +- [**Name**](#name) +- [**Attributes**](#attributes) +- [**How To Enable**](#how-to-enable) +- [**How to set collecting**](#how-to-set-collecting) +- [**Disable Plugin**](#disable-plugin) + +## Name + +[OpenTelemetry](https://opentelemetry.io/) report Tracing data according to [opentelemetry specification](https://github.com/open-telemetry/opentelemetry-specification). + +Just support reporting in `HTTP` with `Content-Type=application/x-protobuf`, the specification: [OTLP/HTTP Request](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlphttp-request)。 + +## Attributes + +| Name | Type | Requirement | Default | Valid | Description | +| ------------ | ------ | ------ | -------- | ------------ | ----------------------------------------------------- | +| sampler | object | optional | | | sampling config +| sampler.name | string | optional | always_off | ["always_on", "always_off", "trace_id_ratio", "parent_base"] | sampling strategy，always_on：sampling all；always_off：sampling nothing；trace_id_ratio：base trace id percentage；parent_base：use parent decision, otherwise determined by root +| sampler.options | object | optional | | {fraction = 0, root = {name = "always_off"}} | sampling strategy parameters +| sampler.options.fraction | number | optional | 0 | [0, 1] | trace_id_ratio fraction +| sampler.options.root | object | optional | {name = "always_off", options = {fraction = 0}} | | parent_base root sampler +| sampler.options.root.name | string | optional | always_off | ["always_on", "always_off", "trace_id_ratio"] | sampling strategy +| sampler.options.root.options | object | optional | {fraction = 0} | | sampling strategy parameters +| sampler.options.root.options.fraction | number | optional | 0 | [0, 1] | trace_id_ratio fraction +| additional_attributes | array[string] | optional | | | attributes (variable and its value) which will be appended to the trace span +| additional_attributes[0] | string | required | | | APISIX or Nginx variable, like `http_header` or `route_id` + +## How To Enable + +First of all, enable the opentelemetry plugin in the `config.yaml`: + +```yaml +# Add this in config.yaml +plugins: + - ... # plugin you need + - opentelemetry +``` + +Then reload APISIX. + +Here's an example, enable the opentelemetry plugin on the specified route: + +```shell +curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "methods": ["GET"], + "uris": [ + "/uid/*" + ], + "plugins": { + "opentelemetry": { + sampler": { + "name": "always_on", + } + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "10.110.149.175:8089": 1 + } + } +}' +``` + +## How to set collecting + +You can set the collecting by specifying the configuration in `conf/config.yaml`. + +| Name | Type | Default | Description | +| ------------ | ------ | -------- | ----------------------------------------------------- | +| trace_id_source | enum | random | the source of trace id, the valid value is `random` or `x-request-id`. If `x-request-id` is set, the value of `x-request-id` request header will be used as trace id. Please make sure it match regex pattern `[0-9a-f]{32}` | +| resource | object | | additional [resource](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/sdk.md) append to trace | +| collector | object | {address = "127.0.0.1:4317", request_timeout = 3} | otlp collector | +| collector.address | string | 127.0.0.1:4317 | collector address | +| collector.request_timeout | integer | 3 | report request timeout(second) | +| collector.request_headers | object | | report request http headers | +| batch_span_processor | object | | trace span processor | +| batch_span_processor.drop_on_queue_full | boolean | true | drop span when queue is full, otherwise force process batches | +| batch_span_processor.max_queue_size | integer | 2048 | maximum queue size to buffer spans for delayed processing | +| batch_span_processor.batch_timeout | number | 5 | maximum duration(second) for constructing a batch | +| batch_span_processor.max_export_batch_size | integer | 256 | maximum number of spans to process in a single batch | +| batch_span_processor.inactive_timeout | number | 2 | timer interval(second) for processing batches | + +Here is an example: + +```yaml +plugin_attr: + opentelemetry: + resource: + service.name: APISIX + tenant.id: business_id + collector: + address: 192.168.8.211:4317 + request_timeout: 3 + request_headers: + foo: bar + batch_span_processor: + drop_on_queue_full: false + max_queue_size: 6 + batch_timeout: 2 + inactive_timeout: 1 + max_export_batch_size: 2 +``` + +## Disable Plugin + +When you want to disable the opentelemetry plugin on a route/service, it is very simple, +you can delete the corresponding JSON configuration in the plugin configuration, +no need to restart the service, it will take effect immediately: + +```shell +$ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "methods": ["GET"], + "uris": [ + "/uid/*" + ], + "plugins": { + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "10.110.149.175:8089": 1 + } + } +}' +``` diff --git a/docs/zh/latest/config.json b/docs/zh/latest/config.json index a0c29211..22026bed 100644 --- a/docs/zh/latest/config.json +++ b/docs/zh/latest/config.json @@ -105,6 +105,7 @@ "plugins/prometheus", "plugins/zipkin", "plugins/skywalking", + "plugins/opentelemetry", "plugins/node-status" ] }, diff --git a/docs/zh/latest/plugins/opentelemetry.md b/docs/zh/latest/plugins/opentelemetry.md new file mode 100644 index 00000000..fbf784e3 --- /dev/null +++ b/docs/zh/latest/plugins/opentelemetry.md @@ -0,0 +1,151 @@ +--- +title: opentelemetry +--- + + + +## 目录 + +- [名字](#名字) +- [属性](#属性) +- [如何启用](#如何启用) +- [如何设置数据上报](#如何设置数据上报) +- [禁用插件](#禁用插件) + +## 名字 + +[OpenTelemetry](https://opentelemetry.io/) 提供符合 [opentelemetry specification](https://github.com/open-telemetry/opentelemetry-specification) 协议规范的 Tracing 数据上报。 + +只支持 `HTTP` 协议，且请求类型为 `application/x-protobuf` 的数据上报，相关协议标准：[OTLP/HTTP Request](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlphttp-request)。 + +## 属性 + +| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | +| ------------ | ------ | ------ | -------- | ------------ | ----------------------------------------------------- | +| sampler | object | 可选 | | | 采样配置 +| sampler.name | string | 可选 | always_off | ["always_on", "always_off", "trace_id_ratio", "parent_base"] | 采样算法，always_on：全采样；always_off：不采样；trace_id_ratio：基于 trace id 的百分比采样；parent_base：如果存在 tracing 上游，则使用上游的采样决定，否则使用配置的采样算法决策 +| sampler.options | object | 可选 | | {fraction = 0, root = {name = "always_off"}} | 采样算法参数 +| sampler.options.fraction | number | 可选 | 0 | [0, 1] | trace_id_ratio 采样算法的百分比 +| sampler.options.root | object | 可选 | {name = "always_off", options = {fraction = 0}} | | parent_base 采样算法在没有上游 tracing 时，会使用 root 采样算法做决策 +| sampler.options.root.name | string | 可选 | always_off | ["always_on", "always_off", "trace_id_ratio"] | 采样算法 +| sampler.options.root.options | object | 可选 | {fraction = 0} | | 采样算法参数 +| sampler.options.root.options.fraction | number | 可选 | 0 | [0, 1] | trace_id_ratio 采样算法的百分比 +| additional_attributes | array[string] | optional | | | 追加到 trace span 的额外属性（变量名为 key，变量值为 value） +| additional_attributes[0] | string | required | | | APISIX or Nginx 变量，例如 `http_header` or `route_id` + +## 如何启用 + +首先，你需要在 `config.yaml` 里面启用 opentelemetry 插件： + +```yaml +# 加到 config.yaml +plugins: + - ... # plugin you need + - opentelemetry +``` + +然后重载 APISIX。 + +下面是一个示例，在指定的 route 上开启了 opentelemetry 插件: + +```shell +curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "methods": ["GET"], + "uris": [ + "/uid/*" + ], + "plugins": { + "opentelemetry": { + sampler": { + "name": "always_on", + } + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "10.110.149.175:8089": 1 + } + } +}' +``` + +## 如何设置数据上报 + +我们可以通过指定 `conf/config.yaml` 中的配置来设置数据上报： + +| 名称 | 类型 | 默认值 | 描述 | +| ------------ | ------ | -------- | ----------------------------------------------------- | +| trace_id_source | enum | random | 合法的取值：`random` 或 `x-request-id`，允许使用当前请求 ID 代替随机 ID 作为新的 TraceID，必须确保当前请求 ID 是符合 TraceID 规范的：`[0-9a-f]{32}` | +| resource | object | | 追加到 trace 的额外 [resource](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/sdk.md) | +| collector | object | {address = "127.0.0.1:4317", request_timeout = 3} | 数据采集服务 | +| collector.address | string | 127.0.0.1:4317 | 数据采集服务地址 | +| collector.request_timeout | integer | 3 | 数据采集服务上报请求超时时长，单位秒 | +| collector.request_headers | object | | 数据采集服务上报请求附加的 HTTP 请求头 | +| batch_span_processor | object | | trace span 处理器参数配置 | +| batch_span_processor.drop_on_queue_full | boolean | true | 当处理器缓存队列慢试，丢弃新到来的 span | +| batch_span_processor.max_queue_size | integer | 2048 | 处理器缓存队列容量最大值 | +| batch_span_processor.batch_timeout | number | 5 | 构造一批 span 超时时长，单位秒 | +| batch_span_processor.max_export_batch_size | integer | 256 | 一批 span 的数量，每次上报的 span 数量 | +| batch_span_processor.inactive_timeout | number | 2 | 每隔多长时间检查是否有一批 span 可以上报，单位秒 | + +配置示例: + +```yaml +plugin_attr: + opentelemetry: + resource: + service.name: APISIX + tenant.id: business_id + collector: + address: 192.168.8.211:4317 + request_timeout: 3 + request_headers: + foo: bar + batch_span_processor: + drop_on_queue_full: false + max_queue_size: 6 + batch_timeout: 2 + inactive_timeout: 1 + max_export_batch_size: 2 +``` + +## 禁用插件 + +当你想禁用一条路由/服务上的 opentelemetry 插件的时候，很简单，在插件的配置中把对应的 JSON 配置删除即可，无须重启服务，即刻生效： + +```shell +$ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "methods": ["GET"], + "uris": [ + "/uid/*" + ], + "plugins": { + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "10.110.149.175:8089": 1 + } + } +}' +``` diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec index 44116062..bfc07766 100644 --- a/rockspec/apisix-master-0.rockspec +++ b/rockspec/apisix-master-0.rockspec @@ -73,6 +73,7 @@ dependencies = { "inspect == 3.1.1", "lualdap = 1.2.6-1", "lua-resty-rocketmq = 0.3.0-0", + "opentelemetry-lua = 0.1-1", } build = { diff --git a/t/plugin/opentelemetry.t b/t/plugin/opentelemetry.t new file mode 100644 index 00000000..0c142667 --- /dev/null +++ b/t/plugin/opentelemetry.t @@ -0,0 +1,722 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +use t::APISIX 'no_plan'; + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->extra_yaml_config) { + my $extra_yaml_config = <<_EOC_; +plugins: + - opentelemetry +plugin_attr: + opentelemetry: + batch_span_processor: + max_export_batch_size: 1 + inactive_timeout: 0.5 +_EOC_ + $block->set_value("extra_yaml_config", $extra_yaml_config); + } + + + if (!$block->extra_init_by_lua) { + my $extra_init_by_lua = <<_EOC_; +-- mock exporter http client +local client = require("opentelemetry.trace.exporter.http_client") +client.do_request = function() + ngx.log(ngx.INFO, "opentelemetry export span") +end +_EOC_ + + $block->set_value("extra_init_by_lua", $extra_init_by_lua); + } + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + if (!defined $block->response_body) { + $block->set_value("response_body", "passed\n"); + } + + if (!$block->no_error_log && !$block->error_log) { + $block->set_value("no_error_log", "[error]"); + } + + $block; +}); + +repeat_each(1); +no_long_string(); +no_root_location(); +log_level("debug"); + +run_tests; + +__DATA__ + +=== TEST 1: add plugin +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "opentelemetry": { + "sampler": { + "name": "always_on" + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 2: trigger opentelemetry +--- request +GET /opentracing +--- response_body +opentracing +--- wait: 1 +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out +opentelemetry export span + + + +=== TEST 3: use default always_off sampler +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "opentelemetry": { + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 4: not trigger opentelemetry +--- request +GET /opentracing +--- response_body +opentracing +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out + + + +=== TEST 5: use trace_id_ratio sampler, default fraction = 0 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "opentelemetry": { + "sampler": { + "name": "trace_id_ratio" + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 6: not trigger opentelemetry +--- request +GET /opentracing +--- response_body +opentracing +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out + + + +=== TEST 7: use trace_id_ratio sampler, fraction = 1.0 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "opentelemetry": { + "sampler": { + "name": "trace_id_ratio", + "options": { + "fraction": 1.0 + } + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 8: trigger opentelemetry +--- request +GET /opentracing +--- response_body +opentracing +--- wait: 1 +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out +opentelemetry export span + + + +=== TEST 9: use parent_base sampler, default root sampler = always_off +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "opentelemetry": { + "sampler": { + "name": "parent_base" + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 10: not trigger opentelemetry +--- request +GET /opentracing +--- response_body +opentracing +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out + + + +=== TEST 11: use parent_base sampler, root sampler = always_on +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "opentelemetry": { + "sampler": { + "name": "parent_base", + "options": { + "root": { + "name": "always_on" + } + } + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 12: trigger opentelemetry +--- request +GET /opentracing +--- response_body +opentracing +--- wait: 1 +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out +opentelemetry export span + + + +=== TEST 13: use parent_base sampler, root sampler = trace_id_ratio with default fraction = 0 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "opentelemetry": { + "sampler": { + "name": "parent_base", + "options": { + "root": { + "name": "trace_id_ratio" + } + } + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 14: not trigger opentelemetry +--- request +GET /opentracing +--- response_body +opentracing +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out + + + +=== TEST 15: trigger opentelemetry, trace_flag = 1 +--- request +GET /opentracing +--- more_headers +traceparent: 00-00000000000000000000000000000001-0000000000000001-01 +--- response_body +opentracing +--- wait: 1 +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out +opentelemetry export span + + + +=== TEST 16: use parent_base sampler, root sampler = trace_id_ratio with fraction = 1 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "opentelemetry": { + "sampler": { + "name": "parent_base", + "options": { + "root": { + "name": "trace_id_ratio", + "options": { + "fraction": 1.0 + } + } + } + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 17: trigger opentelemetry +--- request +GET /opentracing +--- response_body +opentracing +--- wait: 1 +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out +opentelemetry export span + + + +=== TEST 18: not trigger opentelemetry, trace_flag = 0 +--- request +GET /opentracing +--- more_headers +traceparent: 00-00000000000000000000000000000001-0000000000000001-00 +--- response_body +opentracing +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out + + + +=== TEST 19: set additional_attributes +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/services/1', + ngx.HTTP_PUT, + [[{ + "name": "service_name", + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + } + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "name": "route_name", + "plugins": { + "opentelemetry": { + "sampler": { + "name": "always_on" + }, + "additional_attributes": [ + "http_user_agent", + "arg_foo", + "cookie_token", + "remote_addr" + ] + } + }, + "uri": "/opentracing", + "service_id": "1" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 20: trigger opentelemetry, test trace_id_source=x-request-id, custom resource, additional_attributes +--- extra_yaml_config +plugins: + - opentelemetry +plugin_attr: + opentelemetry: + trace_id_source: x-request-id + resource: + service.name: test + test_key: test_val + batch_span_processor: + max_export_batch_size: 1 + inactive_timeout: 0.5 +--- extra_init_by_lua + local core = require("apisix.core") + local otlp = require("opentelemetry.trace.exporter.otlp") + otlp.export_spans = function(self, spans) + if (#spans ~= 1) then + ngx.log(ngx.ERR, "unexpected spans length: ", #spans) + return + end + + local span = spans[1] + if span:context().trace_id ~= "01010101010101010101010101010101" then + ngx.log(ngx.ERR, "unexpected trace id: ", span:context().trace_id) + return + end + + if span.name ~= "/opentracing?foo=bar&a=b" then + ngx.log(ngx.ERR, "expect span name: /opentracing?foo=bar&a=b, but got ", span.name) + return + end + + local expected_resource_attrs = { + test_key = "test_val", + } + expected_resource_attrs["service.name"] = "test" + expected_resource_attrs["telemetry.sdk.language"] = "lua" + expected_resource_attrs["telemetry.sdk.name"] = "opentelemetry-lua" + expected_resource_attrs["telemetry.sdk.version"] = "0.1.1" + expected_resource_attrs["hostname"] = core.utils.gethostname() + local actual_resource_attrs = span.tracer.provider.resource:attributes() + if #actual_resource_attrs ~= 6 then + ngx.log(ngx.ERR, "expect len(actual_resource) = 6, but got ", #actual_resource_attrs) + return + end + for _, attr in ipairs(actual_resource_attrs) do + local expected_val = expected_resource_attrs[attr.key] + if not expected_val then + ngx.log(ngx.ERR, "unexpected resource attr key: ", attr.key) + return + end + if attr.value.string_value ~= expected_val then + ngx.log(ngx.ERR, "unexpected resource attr val: ", attr.value.string_value) + return + end + end + + local expected_attributes = { + service = "service_name", + route = "route_name", + http_user_agent = "test_nginx", + arg_foo = "bar", + cookie_token = "auth_token", + remote_addr = "127.0.0.1", + } + if #span.attributes ~= 6 then + ngx.log(ngx.ERR, "expect len(span.attributes) = 6, but got ", #span.attributes) + return + end + for _, attr in ipairs(span.attributes) do + local expected_val = expected_attributes[attr.key] + if not expected_val then + ngx.log(ngx.ERR, "unexpected attr key: ", attr.key) + return + end + if attr.value.string_value ~= expected_val then + ngx.log(ngx.ERR, "unexpected attr val: ", attr.value.string_value) + return + end + end + + ngx.log(ngx.INFO, "opentelemetry export span") + end +--- request +GET /opentracing?foo=bar&a=b +--- more_headers +X-Request-Id: 01010101010101010101010101010101 +User-Agent: test_nginx +Cookie: token=auth_token; +--- response_body +opentracing +--- wait: 1 +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out +opentelemetry export span + + + +=== TEST 21: create route for /specific_status +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/2', + ngx.HTTP_PUT, + [[{ + "name": "route_name", + "plugins": { + "opentelemetry": { + "sampler": { + "name": "always_on" + } + } + }, + "uri": "/specific_status", + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 22: 500 status, test span.status +--- extra_init_by_lua + local otlp = require("opentelemetry.trace.exporter.otlp") + otlp.export_spans = function(self, spans) + if (#spans ~= 1) then + ngx.log(ngx.ERR, "unexpected spans length: ", #spans) + return + end + + local span = spans[1] + if span.status.code ~= 2 then + ngx.log(ngx.ERR, "unexpected status.code: ", span.status.code) + end + if span.status.message ~= "upstream response status: 500" then + ngx.log(ngx.ERR, "unexpected status.message: ", span.status.message) + end + + ngx.log(ngx.INFO, "opentelemetry export span") + end +--- request +GET /specific_status +--- more_headers +X-Test-Upstream-Status: 500 +--- error_code: 500 +--- response_body +upstream status: 500 +--- wait: 1 +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out +opentelemetry export span + + + +=== TEST 23: test response empty body +--- extra_init_by_lua + local otlp = require("opentelemetry.trace.exporter.otlp") + otlp.export_spans = function(self, spans) + ngx.log(ngx.INFO, "opentelemetry export span") + end +--- request +HEAD /specific_status +--- response_body +--- wait: 1 +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out +opentelemetry export span