mirror of
https://gitee.com/dgiiot/dgiot.git
synced 2024-11-29 10:48:04 +08:00
add linux_4.3.1
This commit is contained in:
U-JOHNLIU\jonhl
parent
9bda456ed7
commit
f83db4a345
14
.ci/build_packages/Dockerfile
Normal file
14
.ci/build_packages/Dockerfile
Normal file
@ -0,0 +1,14 @@
|
||||
ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
|
||||
FROM ${BUILD_FROM}
|
||||
|
||||
ARG EMQX_NAME=emqx
|
||||
|
||||
COPY . /emqx
|
||||
|
||||
WORKDIR /emqx
|
||||
|
||||
RUN make ${EMQX_NAME}-zip || cat rebar3.crashdump
|
||||
|
||||
RUN make ${EMQX_NAME}-pkg || cat rebar3.crashdump
|
||||
|
||||
RUN /emqx/.ci/build_packages/tests.sh
|
||||
161
.ci/build_packages/tests.sh
Normal file
161
.ci/build_packages/tests.sh
Normal file
@ -0,0 +1,161 @@
|
||||
#!/bin/bash
|
||||
set -x -e -u
|
||||
export CODE_PATH=${CODE_PATH:-"/emqx"}
|
||||
export EMQX_NAME=${EMQX_NAME:-"emqx"}
|
||||
export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}"
|
||||
export RELUP_PACKAGE_PATH="${CODE_PATH}/relup_packages/${EMQX_NAME}"
|
||||
# export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1"
|
||||
# export EMQX_NODE_COOKIE=$(date +%s%N)
|
||||
|
||||
emqx_prepare(){
|
||||
mkdir -p "${PACKAGE_PATH}"
|
||||
|
||||
if [ ! -d "/paho-mqtt-testing" ]; then
|
||||
git clone -b develop-4.0 https://hub.fastgit.org/emqx/paho.mqtt.testing.git /paho-mqtt-testing
|
||||
fi
|
||||
pip3 install pytest
|
||||
}
|
||||
|
||||
emqx_test(){
|
||||
cd "${PACKAGE_PATH}"
|
||||
|
||||
for var in "$PACKAGE_PATH"/"${EMQX_NAME}"-*;do
|
||||
case ${var##*.} in
|
||||
"zip")
|
||||
packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip)
|
||||
unzip -q "${PACKAGE_PATH}/${packagename}"
|
||||
export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \
|
||||
EMQX_MQTT__MAX_TOPIC_ALIAS=10
|
||||
sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins
|
||||
|
||||
echo "running ${packagename} start"
|
||||
"${PACKAGE_PATH}"/emqx/bin/emqx start || ( tail "${PACKAGE_PATH}"/emqx/log/emqx.log.1 && exit 1 )
|
||||
IDLE_TIME=0
|
||||
while [ -z "$("${PACKAGE_PATH}"/emqx/bin/emqx_ctl status |grep 'is running'|awk '{print $1}')" ]
|
||||
do
|
||||
if [ $IDLE_TIME -gt 10 ]
|
||||
then
|
||||
echo "emqx running error"
|
||||
exit 1
|
||||
fi
|
||||
sleep 10
|
||||
IDLE_TIME=$((IDLE_TIME+1))
|
||||
done
|
||||
pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
|
||||
"${PACKAGE_PATH}"/emqx/bin/emqx stop
|
||||
echo "running ${packagename} stop"
|
||||
rm -rf "${PACKAGE_PATH}"/emqx
|
||||
;;
|
||||
"deb")
|
||||
packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.deb)
|
||||
dpkg -i "${PACKAGE_PATH}/${packagename}"
|
||||
if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ]
|
||||
then
|
||||
echo "package install error"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "running ${packagename} start"
|
||||
running_test
|
||||
echo "running ${packagename} stop"
|
||||
|
||||
dpkg -r "${EMQX_NAME}"
|
||||
if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "rc" ]
|
||||
then
|
||||
echo "package remove error"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
dpkg -P "${EMQX_NAME}"
|
||||
if dpkg -l |grep -q emqx
|
||||
then
|
||||
echo "package uninstall error"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
"rpm")
|
||||
packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm)
|
||||
rpm -ivh "${PACKAGE_PATH}/${packagename}"
|
||||
if ! rpm -q emqx | grep -q emqx; then
|
||||
echo "package install error"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "running ${packagename} start"
|
||||
running_test
|
||||
echo "running ${packagename} stop"
|
||||
|
||||
rpm -e "${EMQX_NAME}"
|
||||
if [ "$(rpm -q emqx)" != "package emqx is not installed" ];then
|
||||
echo "package uninstall error"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
running_test(){
|
||||
export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \
|
||||
EMQX_MQTT__MAX_TOPIC_ALIAS=10
|
||||
sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins
|
||||
|
||||
emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 )
|
||||
IDLE_TIME=0
|
||||
while [ -z "$(emqx_ctl status |grep 'is running'|awk '{print $1}')" ]
|
||||
do
|
||||
if [ $IDLE_TIME -gt 10 ]
|
||||
then
|
||||
echo "emqx running error"
|
||||
exit 1
|
||||
fi
|
||||
sleep 10
|
||||
IDLE_TIME=$((IDLE_TIME+1))
|
||||
done
|
||||
pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
|
||||
# shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
|
||||
emqx stop || kill "$(ps -ef | grep -E '\-progname\s.+emqx\s' |awk '{print $2}')"
|
||||
|
||||
if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \
|
||||
|| [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then
|
||||
service emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 )
|
||||
IDLE_TIME=0
|
||||
while [ -z "$(emqx_ctl status |grep 'is running'|awk '{print $1}')" ]
|
||||
do
|
||||
if [ $IDLE_TIME -gt 10 ]
|
||||
then
|
||||
echo "emqx service error"
|
||||
exit 1
|
||||
fi
|
||||
sleep 10
|
||||
IDLE_TIME=$((IDLE_TIME+1))
|
||||
done
|
||||
service emqx stop
|
||||
fi
|
||||
}
|
||||
|
||||
relup_test(){
|
||||
TARGET_VERSION="$1"
|
||||
if [ -d "${RELUP_PACKAGE_PATH}" ];then
|
||||
cd "${RELUP_PACKAGE_PATH }"
|
||||
|
||||
for var in "${EMQX_NAME}"-*-"$(uname -m)".zip;do
|
||||
packagename=$(basename "${var}")
|
||||
unzip "$packagename"
|
||||
./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 )
|
||||
./emqx/bin/emqx_ctl status
|
||||
./emqx/bin/emqx versions
|
||||
cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-$(uname -m)".zip ./emqx/releases
|
||||
./emqx/bin/emqx install "${TARGET_VERSION}"
|
||||
[ "$(./emqx/bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]")" = "${TARGET_VERSION}" ] || exit 1
|
||||
./emqx/bin/emqx_ctl status
|
||||
./emqx/bin/emqx stop
|
||||
rm -rf emqx
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
emqx_prepare
|
||||
emqx_test
|
||||
# relup_test <TODO: parameterise relup target version>
|
||||
8
.ci/docker-compose-file/.env
Normal file
8
.ci/docker-compose-file/.env
Normal file
@ -0,0 +1,8 @@
|
||||
MYSQL_TAG=8
|
||||
REDIS_TAG=6
|
||||
MONGO_TAG=4
|
||||
PGSQL_TAG=13
|
||||
LDAP_TAG=2.4.50
|
||||
|
||||
TARGET=emqx/emqx
|
||||
EMQX_TAG=build-alpine-amd64
|
||||
7
.ci/docker-compose-file/conf.cluster.env
Normal file
7
.ci/docker-compose-file/conf.cluster.env
Normal file
@ -0,0 +1,7 @@
|
||||
EMQX_NAME=emqx
|
||||
EMQX_CLUSTER__DISCOVERY=static
|
||||
EMQX_CLUSTER__STATIC__SEEDS="emqx@node1.emqx.io, emqx@node2.emqx.io"
|
||||
EMQX_LISTENER__TCP__EXTERNAL__PROXY_PROTOCOL=on
|
||||
EMQX_LISTENER__WS__EXTERNAL__PROXY_PROTOCOL=on
|
||||
EMQX_LOG__LEVEL=debug
|
||||
EMQX_LOADED_PLUGINS=emqx_sn
|
||||
13
.ci/docker-compose-file/conf.env
Normal file
13
.ci/docker-compose-file/conf.env
Normal file
@ -0,0 +1,13 @@
|
||||
EMQX_AUTH__LDAP__SERVERS=ldap_server
|
||||
EMQX_AUTH__MONGO__SERVER=mongo_server:27017
|
||||
EMQX_AUTH__MYSQL__SERVER=mysql_server:3306
|
||||
EMQX_AUTH__MYSQL__USERNAME=root
|
||||
EMQX_AUTH__MYSQL__PASSWORD=public
|
||||
EMQX_AUTH__MYSQL__DATABASE=mqtt
|
||||
EMQX_AUTH__PGSQL__SERVER=pgsql_server:5432
|
||||
EMQX_AUTH__PGSQL__USERNAME=root
|
||||
EMQX_AUTH__PGSQL__PASSWORD=public
|
||||
EMQX_AUTH__PGSQL__DATABASE=mqtt
|
||||
EMQX_AUTH__REDIS__SERVER=redis_server:6379
|
||||
EMQX_AUTH__REDIS__PASSWORD=public
|
||||
CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
|
||||
85
.ci/docker-compose-file/docker-compose-emqx-cluster.yaml
Normal file
85
.ci/docker-compose-file/docker-compose-emqx-cluster.yaml
Normal file
@ -0,0 +1,85 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
haproxy:
|
||||
container_name: haproxy
|
||||
image: haproxy:2.3
|
||||
depends_on:
|
||||
- emqx1
|
||||
- emqx2
|
||||
volumes:
|
||||
- ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
|
||||
- ../../etc/certs:/usr/local/etc/haproxy/certs
|
||||
ports:
|
||||
- "18083:18083"
|
||||
# - "1883:1883"
|
||||
# - "8883:8883"
|
||||
# - "8083:8083"
|
||||
# - "8084:8084"
|
||||
networks:
|
||||
- emqx_bridge
|
||||
working_dir: /usr/local/etc/haproxy
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
- |
|
||||
cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /usr/local/etc/haproxy/certs/emqx.pem
|
||||
haproxy -f /usr/local/etc/haproxy/haproxy.cfg
|
||||
|
||||
emqx1:
|
||||
container_name: node1.emqx.io
|
||||
image: $TARGET:$EMQX_TAG
|
||||
env_file:
|
||||
- conf.cluster.env
|
||||
environment:
|
||||
- "EMQX_HOST=node1.emqx.io"
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
|
||||
sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
|
||||
/opt/emqx/bin/emqx foreground
|
||||
healthcheck:
|
||||
test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"]
|
||||
interval: 5s
|
||||
timeout: 25s
|
||||
retries: 5
|
||||
networks:
|
||||
emqx_bridge:
|
||||
aliases:
|
||||
- node1.emqx.io
|
||||
|
||||
emqx2:
|
||||
container_name: node2.emqx.io
|
||||
image: $TARGET:$EMQX_TAG
|
||||
env_file:
|
||||
- conf.cluster.env
|
||||
environment:
|
||||
- "EMQX_HOST=node2.emqx.io"
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
|
||||
sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
|
||||
/opt/emqx/bin/emqx foreground
|
||||
healthcheck:
|
||||
test: ["CMD", "/opt/emqx/bin/emqx", "ping"]
|
||||
interval: 5s
|
||||
timeout: 25s
|
||||
retries: 5
|
||||
networks:
|
||||
emqx_bridge:
|
||||
aliases:
|
||||
- node2.emqx.io
|
||||
|
||||
networks:
|
||||
emqx_bridge:
|
||||
driver: bridge
|
||||
name: emqx_bridge
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: 172.100.239.0/24
|
||||
gateway: 172.100.239.1
|
||||
16
.ci/docker-compose-file/docker-compose-ldap-tcp.yaml
Normal file
16
.ci/docker-compose-file/docker-compose-ldap-tcp.yaml
Normal file
@ -0,0 +1,16 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
ldap_server:
|
||||
container_name: ldap
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: .ci/docker-compose-file/openldap/Dockerfile
|
||||
args:
|
||||
LDAP_TAG: ${LDAP_TAG}
|
||||
image: openldap
|
||||
ports:
|
||||
- 389:389
|
||||
restart: always
|
||||
networks:
|
||||
- emqx_bridge
|
||||
14
.ci/docker-compose-file/docker-compose-mongo-tcp.yaml
Normal file
14
.ci/docker-compose-file/docker-compose-mongo-tcp.yaml
Normal file
@ -0,0 +1,14 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
mongo_server:
|
||||
container_name: mongo
|
||||
image: mongo:${MONGO_TAG}
|
||||
restart: always
|
||||
environment:
|
||||
MONGO_INITDB_DATABASE: mqtt
|
||||
networks:
|
||||
- emqx_bridge
|
||||
command:
|
||||
--ipv6
|
||||
--bind_ip_all
|
||||
18
.ci/docker-compose-file/docker-compose-mongo-tls.yaml
Normal file
18
.ci/docker-compose-file/docker-compose-mongo-tls.yaml
Normal file
@ -0,0 +1,18 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
mongo_server:
|
||||
container_name: mongo
|
||||
image: mongo:${MONGO_TAG}
|
||||
restart: always
|
||||
environment:
|
||||
MONGO_INITDB_DATABASE: mqtt
|
||||
volumes:
|
||||
- ../../apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/mongodb.pem/:/etc/certs/mongodb.pem
|
||||
networks:
|
||||
- emqx_bridge
|
||||
command:
|
||||
--ipv6
|
||||
--bind_ip_all
|
||||
--sslMode requireSSL
|
||||
--sslPEMKeyFile /etc/certs/mongodb.pem
|
||||
20
.ci/docker-compose-file/docker-compose-mysql-tcp.yaml
Normal file
20
.ci/docker-compose-file/docker-compose-mysql-tcp.yaml
Normal file
@ -0,0 +1,20 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
mysql_server:
|
||||
container_name: mysql
|
||||
image: mysql:${MYSQL_TAG}
|
||||
restart: always
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: public
|
||||
MYSQL_DATABASE: mqtt
|
||||
networks:
|
||||
- emqx_bridge
|
||||
command:
|
||||
--bind-address "::"
|
||||
--character-set-server=utf8mb4
|
||||
--collation-server=utf8mb4_general_ci
|
||||
--explicit_defaults_for_timestamp=true
|
||||
--lower_case_table_names=1
|
||||
--max_allowed_packet=128M
|
||||
--skip-symbolic-links
|
||||
45
.ci/docker-compose-file/docker-compose-mysql-tls.yaml
Normal file
45
.ci/docker-compose-file/docker-compose-mysql-tls.yaml
Normal file
@ -0,0 +1,45 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
mysql_server:
|
||||
container_name: mysql
|
||||
image: mysql:${MYSQL_TAG}
|
||||
restart: always
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: public
|
||||
MYSQL_DATABASE: mqtt
|
||||
MYSQL_USER: ssluser
|
||||
MYSQL_PASSWORD: public
|
||||
volumes:
|
||||
- ../../apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem:/etc/certs/ca-cert.pem
|
||||
- ../../apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/server-cert.pem:/etc/certs/server-cert.pem
|
||||
- ../../apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/server-key.pem:/etc/certs/server-key.pem
|
||||
networks:
|
||||
- emqx_bridge
|
||||
command:
|
||||
--bind-address "::"
|
||||
--character-set-server=utf8mb4
|
||||
--collation-server=utf8mb4_general_ci
|
||||
--explicit_defaults_for_timestamp=true
|
||||
--lower_case_table_names=1
|
||||
--max_allowed_packet=128M
|
||||
--skip-symbolic-links
|
||||
--ssl-ca=/etc/certs/ca-cert.pem
|
||||
--ssl-cert=/etc/certs/server-cert.pem
|
||||
--ssl-key=/etc/certs/server-key.pem
|
||||
|
||||
mysql_client:
|
||||
container_name: mysql_client
|
||||
image: mysql:${MYSQL_TAG}
|
||||
networks:
|
||||
- emqx_bridge
|
||||
depends_on:
|
||||
- mysql_server
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
service mysql start
|
||||
echo "show tables;" | mysql -h mysql_server -u root -ppublic mqtt mqtt
|
||||
while [[ $$? -ne 0 ]];do echo "show tables;" | mysql -h mysql_server -u root -ppublic mqtt; done
|
||||
echo "ALTER USER 'ssluser'@'%' REQUIRE X509;" | mysql -h mysql_server -u root -ppublic mqtt
|
||||
15
.ci/docker-compose-file/docker-compose-pgsql-tcp.yaml
Normal file
15
.ci/docker-compose-file/docker-compose-pgsql-tcp.yaml
Normal file
@ -0,0 +1,15 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
pgsql_server:
|
||||
container_name: pgsql
|
||||
image: postgres:${PGSQL_TAG}
|
||||
restart: always
|
||||
environment:
|
||||
POSTGRES_PASSWORD: public
|
||||
POSTGRES_USER: root
|
||||
POSTGRES_DB: mqtt
|
||||
ports:
|
||||
- "5432:5432"
|
||||
networks:
|
||||
- emqx_bridge
|
||||
32
.ci/docker-compose-file/docker-compose-pgsql-tls.yaml
Normal file
32
.ci/docker-compose-file/docker-compose-pgsql-tls.yaml
Normal file
@ -0,0 +1,32 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
pgsql_server:
|
||||
container_name: pgsql
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: .ci/docker-compose-file/pgsql/Dockerfile
|
||||
args:
|
||||
POSTGRES_USER: postgres
|
||||
BUILD_FROM: postgres:${PGSQL_TAG}
|
||||
image: emqx_pgsql:${PGSQL_TAG}
|
||||
restart: always
|
||||
environment:
|
||||
POSTGRES_DB: mqtt
|
||||
POSTGRES_USER: root
|
||||
POSTGRES_PASSWORD: public
|
||||
ports:
|
||||
- "5432:5432"
|
||||
command:
|
||||
- -c
|
||||
- ssl=on
|
||||
- -c
|
||||
- ssl_cert_file=/var/lib/postgresql/server.crt
|
||||
- -c
|
||||
- ssl_key_file=/var/lib/postgresql/server.key
|
||||
- -c
|
||||
- ssl_ca_file=/var/lib/postgresql/root.crt
|
||||
- -c
|
||||
- hba_file=/var/lib/postgresql/pg_hba.conf
|
||||
networks:
|
||||
- emqx_bridge
|
||||
15
.ci/docker-compose-file/docker-compose-python.yaml
Normal file
15
.ci/docker-compose-file/docker-compose-python.yaml
Normal file
@ -0,0 +1,15 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
python:
|
||||
container_name: python
|
||||
image: python:3.7.2-alpine3.9
|
||||
depends_on:
|
||||
- emqx1
|
||||
- emqx2
|
||||
tty: true
|
||||
networks:
|
||||
emqx_bridge:
|
||||
volumes:
|
||||
- ./python:/scripts
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
redis_server:
|
||||
image: redis:${REDIS_TAG}
|
||||
container_name: redis
|
||||
volumes:
|
||||
- ./redis/:/data/conf
|
||||
command: bash -c "/bin/bash /data/conf/redis.sh --node cluster && tail -f /var/log/redis-server.log"
|
||||
networks:
|
||||
- emqx_bridge
|
||||
@ -0,0 +1,12 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
redis_server:
|
||||
container_name: redis
|
||||
image: redis:${REDIS_TAG}
|
||||
volumes:
|
||||
- ../../apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs:/tls
|
||||
- ./redis/:/data/conf
|
||||
command: bash -c "/bin/bash /data/conf/redis.sh --node cluster --tls-enabled && tail -f /var/log/redis-server.log"
|
||||
networks:
|
||||
- emqx_bridge
|
||||
@ -0,0 +1,11 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
redis_server:
|
||||
container_name: redis
|
||||
image: redis:${REDIS_TAG}
|
||||
volumes:
|
||||
- ./redis/:/data/conf
|
||||
command: bash -c "/bin/bash /data/conf/redis.sh --node sentinel && tail -f /var/log/redis-server.log"
|
||||
networks:
|
||||
- emqx_bridge
|
||||
@ -0,0 +1,12 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
redis_server:
|
||||
container_name: redis
|
||||
image: redis:${REDIS_TAG}
|
||||
volumes:
|
||||
- ../../apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs:/tls
|
||||
- ./redis/:/data/conf
|
||||
command: bash -c "/bin/bash /data/conf/redis.sh --node sentinel --tls-enabled && tail -f /var/log/redis-server.log"
|
||||
networks:
|
||||
- emqx_bridge
|
||||
13
.ci/docker-compose-file/docker-compose-redis-single-tcp.yaml
Normal file
13
.ci/docker-compose-file/docker-compose-redis-single-tcp.yaml
Normal file
@ -0,0 +1,13 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
redis_server:
|
||||
container_name: redis
|
||||
image: redis:${REDIS_TAG}
|
||||
command:
|
||||
- redis-server
|
||||
- "--bind 0.0.0.0 ::"
|
||||
- --requirepass public
|
||||
restart: always
|
||||
networks:
|
||||
- emqx_bridge
|
||||
19
.ci/docker-compose-file/docker-compose-redis-single-tls.yaml
Normal file
19
.ci/docker-compose-file/docker-compose-redis-single-tls.yaml
Normal file
@ -0,0 +1,19 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
redis_server:
|
||||
container_name: redis
|
||||
image: redis:${REDIS_TAG}
|
||||
volumes:
|
||||
- ../../apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs:/tls
|
||||
command:
|
||||
- redis-server
|
||||
- "--bind 0.0.0.0 ::"
|
||||
- --requirepass public
|
||||
- --tls-port 6380
|
||||
- --tls-cert-file /tls/redis.crt
|
||||
- --tls-key-file /tls/redis.key
|
||||
- --tls-ca-cert-file /tls/ca.crt
|
||||
restart: always
|
||||
networks:
|
||||
- emqx_bridge
|
||||
35
.ci/docker-compose-file/docker-compose.yaml
Normal file
35
.ci/docker-compose-file/docker-compose.yaml
Normal file
@ -0,0 +1,35 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
erlang:
|
||||
container_name: erlang
|
||||
image: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
|
||||
env_file:
|
||||
- conf.env
|
||||
environment:
|
||||
GITHUB_ACTIONS: ${GITHUB_ACTIONS}
|
||||
GITHUB_TOKEN: ${GITHUB_TOKEN}
|
||||
GITHUB_RUN_ID: ${GITHUB_RUN_ID}
|
||||
GITHUB_SHA: ${GITHUB_SHA}
|
||||
GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER}
|
||||
GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}
|
||||
GITHUB_REF: ${GITHUB_REF}
|
||||
networks:
|
||||
- emqx_bridge
|
||||
volumes:
|
||||
- ../..:/emqx
|
||||
working_dir: /emqx
|
||||
tty: true
|
||||
|
||||
networks:
|
||||
emqx_bridge:
|
||||
driver: bridge
|
||||
name: emqx_bridge
|
||||
enable_ipv6: true
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: 172.100.239.0/24
|
||||
gateway: 172.100.239.1
|
||||
- subnet: 2001:3200:3200::/64
|
||||
gateway: 2001:3200:3200::1
|
||||
109
.ci/docker-compose-file/haproxy/haproxy.cfg
Normal file
109
.ci/docker-compose-file/haproxy/haproxy.cfg
Normal file
@ -0,0 +1,109 @@
|
||||
##----------------------------------------------------------------
|
||||
## global 2021/04/05
|
||||
##----------------------------------------------------------------
|
||||
global
|
||||
log stdout format raw daemon debug
|
||||
# Replace 1024000 with deployment connections
|
||||
maxconn 1000
|
||||
nbproc 1
|
||||
nbthread 2
|
||||
cpu-map auto:1/1-2 0-1
|
||||
tune.ssl.default-dh-param 2048
|
||||
ssl-default-bind-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:SEED:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!RSAPSK:!aDH:!aECDH:!EDH-DSS-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA:!SRP
|
||||
# Enable the HAProxy Runtime API
|
||||
stats socket :9999 level admin expose-fd listeners
|
||||
|
||||
##----------------------------------------------------------------
|
||||
## defaults
|
||||
##----------------------------------------------------------------
|
||||
defaults
|
||||
log global
|
||||
mode tcp
|
||||
option tcplog
|
||||
# Replace 1024000 with deployment connections
|
||||
maxconn 1000
|
||||
timeout connect 30000
|
||||
timeout client 600s
|
||||
timeout server 600s
|
||||
|
||||
##----------------------------------------------------------------
|
||||
## API
|
||||
##----------------------------------------------------------------
|
||||
frontend emqx_mgmt
|
||||
mode tcp
|
||||
option tcplog
|
||||
bind *:8081
|
||||
default_backend emqx_mgmt_back
|
||||
|
||||
frontend emqx_dashboard
|
||||
mode tcp
|
||||
option tcplog
|
||||
bind *:18083
|
||||
default_backend emqx_dashboard_back
|
||||
|
||||
backend emqx_mgmt_back
|
||||
mode http
|
||||
# balance static-rr
|
||||
server emqx-1 node1.emqx.io:8081
|
||||
server emqx-2 node2.emqx.io:8081
|
||||
|
||||
backend emqx_dashboard_back
|
||||
mode http
|
||||
# balance static-rr
|
||||
server emqx-1 node1.emqx.io:18083
|
||||
server emqx-2 node2.emqx.io:18083
|
||||
|
||||
|
||||
##----------------------------------------------------------------
|
||||
## public
|
||||
##----------------------------------------------------------------
|
||||
frontend emqx_tcp
|
||||
mode tcp
|
||||
option tcplog
|
||||
bind *:1883
|
||||
default_backend emqx_tcp_back
|
||||
|
||||
frontend emqx_ws
|
||||
mode tcp
|
||||
option tcplog
|
||||
bind *:8083
|
||||
default_backend emqx_ws_back
|
||||
|
||||
backend emqx_tcp_back
|
||||
mode tcp
|
||||
balance static-rr
|
||||
server emqx-1 node1.emqx.io:1883 check-send-proxy send-proxy-v2
|
||||
server emqx-2 node2.emqx.io:1883 check-send-proxy send-proxy-v2
|
||||
|
||||
backend emqx_ws_back
|
||||
mode tcp
|
||||
balance static-rr
|
||||
server emqx-1 node1.emqx.io:8083 check-send-proxy send-proxy-v2
|
||||
server emqx-2 node2.emqx.io:8083 check-send-proxy send-proxy-v2
|
||||
|
||||
##----------------------------------------------------------------
|
||||
## TLS
|
||||
##----------------------------------------------------------------
|
||||
frontend emqx_ssl
|
||||
mode tcp
|
||||
option tcplog
|
||||
bind *:8883 ssl crt /usr/local/etc/haproxy/certs/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3
|
||||
default_backend emqx_ssl_back
|
||||
|
||||
frontend emqx_wss
|
||||
mode tcp
|
||||
option tcplog
|
||||
bind *:8084 ssl crt /usr/local/etc/haproxy/certs/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3
|
||||
default_backend emqx_wss_back
|
||||
|
||||
backend emqx_ssl_back
|
||||
mode tcp
|
||||
balance static-rr
|
||||
server emqx-1 node1.emqx.io:1883 check-send-proxy send-proxy-v2-ssl-cn
|
||||
server emqx-2 node2.emqx.io:1883 check-send-proxy send-proxy-v2-ssl-cn
|
||||
|
||||
backend emqx_wss_back
|
||||
mode tcp
|
||||
balance static-rr
|
||||
server emqx-1 node1.emqx.io:8083 check-send-proxy send-proxy-v2-ssl-cn
|
||||
server emqx-2 node2.emqx.io:8083 check-send-proxy send-proxy-v2-ssl-cn
|
||||
26
.ci/docker-compose-file/openldap/Dockerfile
Normal file
26
.ci/docker-compose-file/openldap/Dockerfile
Normal file
@ -0,0 +1,26 @@
|
||||
FROM buildpack-deps:stretch
|
||||
|
||||
ARG LDAP_TAG=2.4.50
|
||||
|
||||
RUN apt-get update && apt-get install -y groff groff-base
|
||||
RUN wget ftp://ftp.openldap.org/pub/OpenLDAP/openldap-release/openldap-${LDAP_TAG}.tgz \
|
||||
&& gunzip -c openldap-${LDAP_TAG}.tgz | tar xvfB - \
|
||||
&& cd openldap-${LDAP_TAG} \
|
||||
&& ./configure && make depend && make && make install \
|
||||
&& cd .. && rm -rf openldap-${LDAP_TAG}
|
||||
|
||||
COPY .ci/docker-compose-file/openldap/slapd.conf /usr/local/etc/openldap/slapd.conf
|
||||
COPY apps/emqx_auth_ldap/emqx.io.ldif /usr/local/etc/openldap/schema/emqx.io.ldif
|
||||
COPY apps/emqx_auth_ldap/emqx.schema /usr/local/etc/openldap/schema/emqx.schema
|
||||
COPY apps/emqx_auth_ldap/test/certs/*.pem /usr/local/etc/openldap/
|
||||
|
||||
RUN mkdir -p /usr/local/etc/openldap/data \
|
||||
&& slapadd -l /usr/local/etc/openldap/schema/emqx.io.ldif -f /usr/local/etc/openldap/slapd.conf
|
||||
|
||||
WORKDIR /usr/local/etc/openldap
|
||||
|
||||
EXPOSE 389 636
|
||||
|
||||
ENTRYPOINT ["/usr/local/libexec/slapd", "-h", "ldap:/// ldaps:///", "-d", "3", "-f", "/usr/local/etc/openldap/slapd.conf"]
|
||||
|
||||
CMD []
|
||||
16
.ci/docker-compose-file/openldap/slapd.conf
Normal file
16
.ci/docker-compose-file/openldap/slapd.conf
Normal file
@ -0,0 +1,16 @@
|
||||
include /usr/local/etc/openldap/schema/core.schema
|
||||
include /usr/local/etc/openldap/schema/cosine.schema
|
||||
include /usr/local/etc/openldap/schema/inetorgperson.schema
|
||||
include /usr/local/etc/openldap/schema/ppolicy.schema
|
||||
include /usr/local/etc/openldap/schema/emqx.schema
|
||||
|
||||
TLSCACertificateFile /usr/local/etc/openldap/cacert.pem
|
||||
TLSCertificateFile /usr/local/etc/openldap/cert.pem
|
||||
TLSCertificateKeyFile /usr/local/etc/openldap/key.pem
|
||||
|
||||
database bdb
|
||||
suffix "dc=emqx,dc=io"
|
||||
rootdn "cn=root,dc=emqx,dc=io"
|
||||
rootpw {SSHA}eoF7NhNrejVYYyGHqnt+MdKNBh4r1w3W
|
||||
|
||||
directory /usr/local/etc/openldap/data
|
||||
12
.ci/docker-compose-file/pgsql/Dockerfile
Normal file
12
.ci/docker-compose-file/pgsql/Dockerfile
Normal file
@ -0,0 +1,12 @@
|
||||
ARG BUILD_FROM=postgres:11
|
||||
FROM ${BUILD_FROM}
|
||||
ARG POSTGRES_USER=postgres
|
||||
COPY --chown=$POSTGRES_USER .ci/docker-compose-file/pgsql/pg_hba.conf /var/lib/postgresql/pg_hba.conf
|
||||
COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-key.pem /var/lib/postgresql/server.key
|
||||
COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-cert.pem /var/lib/postgresql/server.crt
|
||||
COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem /var/lib/postgresql/root.crt
|
||||
RUN chmod 600 /var/lib/postgresql/pg_hba.conf
|
||||
RUN chmod 600 /var/lib/postgresql/server.key
|
||||
RUN chmod 600 /var/lib/postgresql/server.crt
|
||||
RUN chmod 600 /var/lib/postgresql/root.crt
|
||||
EXPOSE 5432
|
||||
9
.ci/docker-compose-file/pgsql/pg_hba.conf
Normal file
9
.ci/docker-compose-file/pgsql/pg_hba.conf
Normal file
@ -0,0 +1,9 @@
|
||||
# TYPE DATABASE USER CIDR-ADDRESS METHOD
|
||||
local all all trust
|
||||
host all all 0.0.0.0/0 trust
|
||||
host all all ::/0 trust
|
||||
hostssl all all 0.0.0.0/0 cert
|
||||
hostssl all all ::/0 cert
|
||||
|
||||
hostssl all www-data 0.0.0.0/0 cert clientcert=1
|
||||
hostssl all postgres 0.0.0.0/0 cert clientcert=1
|
||||
24
.ci/docker-compose-file/python/pytest.sh
Normal file
24
.ci/docker-compose-file/python/pytest.sh
Normal file
@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
|
||||
## This script is to run emqx cluster smoke tests (fvt) in github action
|
||||
## This script is executed in pacho_client
|
||||
|
||||
set -x
|
||||
set +e
|
||||
|
||||
LB="haproxy"
|
||||
|
||||
apk update && apk add git curl
|
||||
git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing
|
||||
pip install pytest
|
||||
|
||||
pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$LB"
|
||||
RESULT=$?
|
||||
|
||||
pytest -v /paho.mqtt.testing/interoperability/test_client --host "$LB"
|
||||
RESULT=$(( RESULT + $? ))
|
||||
|
||||
# pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 "node1.emqx.io" --host2 "node2.emqx.io"
|
||||
# RESULT=$(( RESULT + $? ))
|
||||
|
||||
exit $RESULT
|
||||
11
.ci/docker-compose-file/redis/redis-tls.conf
Normal file
11
.ci/docker-compose-file/redis/redis-tls.conf
Normal file
@ -0,0 +1,11 @@
|
||||
daemonize yes
|
||||
bind 0.0.0.0 ::
|
||||
logfile /var/log/redis-server.log
|
||||
tls-cert-file /tls/redis.crt
|
||||
tls-key-file /tls/redis.key
|
||||
tls-ca-cert-file /tls/ca.crt
|
||||
tls-replication yes
|
||||
tls-cluster yes
|
||||
protected-mode no
|
||||
requirepass public
|
||||
masterauth public
|
||||
5
.ci/docker-compose-file/redis/redis.conf
Normal file
5
.ci/docker-compose-file/redis/redis.conf
Normal file
@ -0,0 +1,5 @@
|
||||
daemonize yes
|
||||
bind 0.0.0.0 ::
|
||||
logfile /var/log/redis-server.log
|
||||
requirepass public
|
||||
masterauth public
|
||||
125
.ci/docker-compose-file/redis/redis.sh
Normal file
125
.ci/docker-compose-file/redis/redis.sh
Normal file
@ -0,0 +1,125 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
|
||||
LOCAL_IP=$(hostname -i | grep -oE '((25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])\.){3}(25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])' | head -n 1)
|
||||
|
||||
node=single
|
||||
tls=false
|
||||
while [[ $# -gt 0 ]]
|
||||
do
|
||||
key="$1"
|
||||
|
||||
case $key in
|
||||
-n|--node)
|
||||
node="$2"
|
||||
shift # past argument
|
||||
shift # past value
|
||||
;;
|
||||
-t|--tls-enabled)
|
||||
tls="$2"
|
||||
shift # past argument
|
||||
shift # past value
|
||||
;;
|
||||
*)
|
||||
shift # past argument
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
rm -f \
|
||||
/data/conf/r7000i.log \
|
||||
/data/conf/r7001i.log \
|
||||
/data/conf/r7002i.log \
|
||||
/data/conf/nodes.7000.conf \
|
||||
/data/conf/nodes.7001.conf \
|
||||
/data/conf/nodes.7002.conf ;
|
||||
|
||||
if [ "${node}" = "cluster" ] ; then
|
||||
if $tls ; then
|
||||
redis-server /data/conf/redis-tls.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
|
||||
--tls-port 8000 --cluster-enabled yes ;
|
||||
redis-server /data/conf/redis-tls.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \
|
||||
--tls-port 8001 --cluster-enabled yes;
|
||||
redis-server /data/conf/redis-tls.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \
|
||||
--tls-port 8002 --cluster-enabled yes;
|
||||
else
|
||||
redis-server /data/conf/redis.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --cluster-enabled yes;
|
||||
redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --cluster-enabled yes;
|
||||
redis-server /data/conf/redis.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --cluster-enabled yes;
|
||||
fi
|
||||
elif [ "${node}" = "sentinel" ] ; then
|
||||
if $tls ; then
|
||||
redis-server /data/conf/redis-tls.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
|
||||
--tls-port 8000 --cluster-enabled no;
|
||||
redis-server /data/conf/redis-tls.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \
|
||||
--tls-port 8001 --cluster-enabled no --slaveof "$LOCAL_IP" 8000;
|
||||
redis-server /data/conf/redis-tls.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \
|
||||
--tls-port 8002 --cluster-enabled no --slaveof "$LOCAL_IP" 8000;
|
||||
|
||||
else
|
||||
redis-server /data/conf/redis.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
|
||||
--cluster-enabled no;
|
||||
redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \
|
||||
--cluster-enabled no --slaveof "$LOCAL_IP" 7000;
|
||||
redis-server /data/conf/redis.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \
|
||||
--cluster-enabled no --slaveof "$LOCAL_IP" 7000;
|
||||
fi
|
||||
fi
|
||||
REDIS_LOAD_FLG=true;
|
||||
|
||||
while $REDIS_LOAD_FLG;
|
||||
do
|
||||
sleep 1;
|
||||
redis-cli --pass public --no-auth-warning -p 7000 info 1> /data/conf/r7000i.log 2> /dev/null;
|
||||
if [ -s /data/conf/r7000i.log ]; then
|
||||
:
|
||||
else
|
||||
continue;
|
||||
fi
|
||||
redis-cli --pass public --no-auth-warning -p 7001 info 1> /data/conf/r7001i.log 2> /dev/null;
|
||||
if [ -s /data/conf/r7001i.log ]; then
|
||||
:
|
||||
else
|
||||
continue;
|
||||
fi
|
||||
redis-cli --pass public --no-auth-warning -p 7002 info 1> /data/conf/r7002i.log 2> /dev/null;
|
||||
if [ -s /data/conf/r7002i.log ]; then
|
||||
:
|
||||
else
|
||||
continue;
|
||||
fi
|
||||
if [ "${node}" = "cluster" ] ; then
|
||||
if $tls ; then
|
||||
yes "yes" | redis-cli --cluster create "$LOCAL_IP:8000" "$LOCAL_IP:8001" "$LOCAL_IP:8002" --pass public --no-auth-warning --tls true --cacert /tls/ca.crt --cert /tls/redis.crt --key /tls/redis.key;
|
||||
else
|
||||
yes "yes" | redis-cli --cluster create "$LOCAL_IP:7000" "$LOCAL_IP:7001" "$LOCAL_IP:7002" --pass public --no-auth-warning;
|
||||
fi
|
||||
elif [ "${node}" = "sentinel" ] ; then
|
||||
tee /_sentinel.conf>/dev/null << EOF
|
||||
port 26379
|
||||
bind 0.0.0.0 ::
|
||||
daemonize yes
|
||||
logfile /var/log/redis-server.log
|
||||
dir /tmp
|
||||
EOF
|
||||
if $tls ; then
|
||||
cat >>/_sentinel.conf<<EOF
|
||||
tls-port 26380
|
||||
tls-replication yes
|
||||
tls-cert-file /tls/redis.crt
|
||||
tls-key-file /tls/redis.key
|
||||
tls-ca-cert-file /tls/ca.crt
|
||||
sentinel monitor mymaster $LOCAL_IP 8000 1
|
||||
EOF
|
||||
else
|
||||
cat >>/_sentinel.conf<<EOF
|
||||
sentinel monitor mymaster $LOCAL_IP 7000 1
|
||||
EOF
|
||||
fi
|
||||
redis-server /_sentinel.conf --sentinel;
|
||||
fi
|
||||
REDIS_LOAD_FLG=false;
|
||||
done
|
||||
|
||||
exit 0;
|
||||
1
.ci/fvt_tests/.env
Normal file
1
.ci/fvt_tests/.env
Normal file
@ -0,0 +1 @@
|
||||
TARGET=emqx/emqx
|
||||
30
.ci/fvt_tests/http_server/README.md
Normal file
30
.ci/fvt_tests/http_server/README.md
Normal file
@ -0,0 +1,30 @@
|
||||
## http_server
|
||||
|
||||
|
||||
The http server for emqx functional validation testing
|
||||
|
||||
### Build
|
||||
|
||||
|
||||
$ rebar3 compile
|
||||
|
||||
### Getting Started
|
||||
|
||||
```
|
||||
1> http_server:start().
|
||||
Start http_server listener on 8080 successfully.
|
||||
ok
|
||||
2> http_server:stop().
|
||||
ok
|
||||
```
|
||||
|
||||
### APIS
|
||||
|
||||
+ GET `/counter`
|
||||
|
||||
返回计数器的值
|
||||
|
||||
+ POST `/counter`
|
||||
|
||||
计数器加一
|
||||
|
||||
10
.ci/fvt_tests/http_server/rebar.config
Normal file
10
.ci/fvt_tests/http_server/rebar.config
Normal file
@ -0,0 +1,10 @@
|
||||
{erl_opts, [debug_info]}.
|
||||
{deps,
|
||||
[
|
||||
{minirest, {git, "https://hub.fastgit.org/emqx/minirest.git", {tag, "0.3.1"}}}
|
||||
]}.
|
||||
|
||||
{shell, [
|
||||
% {config, "config/sys.config"},
|
||||
{apps, [http_server]}
|
||||
]}.
|
||||
17
.ci/fvt_tests/http_server/src/http_server.app.src
Normal file
17
.ci/fvt_tests/http_server/src/http_server.app.src
Normal file
@ -0,0 +1,17 @@
|
||||
{application, http_server,
|
||||
[{description, "An OTP application"},
|
||||
{vsn, "0.1.0"},
|
||||
{registered, []},
|
||||
% {mod, {http_server_app, []}},
|
||||
{modules, []},
|
||||
{applications,
|
||||
[kernel,
|
||||
stdlib,
|
||||
minirest
|
||||
]},
|
||||
{env,[]},
|
||||
{modules, []},
|
||||
|
||||
{licenses, ["Apache 2.0"]},
|
||||
{links, []}
|
||||
]}.
|
||||
50
.ci/fvt_tests/http_server/src/http_server.erl
Normal file
50
.ci/fvt_tests/http_server/src/http_server.erl
Normal file
@ -0,0 +1,50 @@
|
||||
-module(http_server).
|
||||
|
||||
-import(minirest, [ return/0
|
||||
, return/1
|
||||
]).
|
||||
|
||||
-export([ start/0
|
||||
, stop/0
|
||||
]).
|
||||
|
||||
-rest_api(#{ name => get_counter
|
||||
, method => 'GET'
|
||||
, path => "/counter"
|
||||
, func => get_counter
|
||||
, descr => "Check counter"
|
||||
}).
|
||||
-rest_api(#{ name => add_counter
|
||||
, method => 'POST'
|
||||
, path => "/counter"
|
||||
, func => add_counter
|
||||
, descr => "Counter plus one"
|
||||
}).
|
||||
|
||||
-export([ get_counter/2
|
||||
, add_counter/2
|
||||
]).
|
||||
|
||||
start() ->
|
||||
application:ensure_all_started(minirest),
|
||||
ets:new(relup_test_message, [named_table, public]),
|
||||
Handlers = [{"/", minirest:handler(#{modules => [?MODULE]})}],
|
||||
Dispatch = [{"/[...]", minirest, Handlers}],
|
||||
minirest:start_http(?MODULE, #{socket_opts => [inet, {port, 8080}]}, Dispatch).
|
||||
|
||||
stop() ->
|
||||
ets:delete(relup_test_message),
|
||||
minirest:stop_http(?MODULE).
|
||||
|
||||
get_counter(_Binding, _Params) ->
|
||||
return({ok, ets:info(relup_test_message, size)}).
|
||||
|
||||
add_counter(_Binding, Params) ->
|
||||
case lists:keymember(<<"payload">>, 1, Params) of
|
||||
true ->
|
||||
{value, {<<"id">>, ID}, Params1} = lists:keytake(<<"id">>, 1, Params),
|
||||
ets:insert(relup_test_message, {ID, Params1});
|
||||
_ ->
|
||||
ok
|
||||
end,
|
||||
return().
|
||||
149
.ci/fvt_tests/relup.lux
Normal file
149
.ci/fvt_tests/relup.lux
Normal file
@ -0,0 +1,149 @@
|
||||
[config var=PACKAGE_PATH]
|
||||
[config var=BENCH_PATH]
|
||||
[config var=ONE_MORE_EMQX_PATH]
|
||||
[config var=VSN]
|
||||
[config var=OLD_VSNS]
|
||||
|
||||
[config shell_cmd=/bin/bash]
|
||||
[config timeout=600000]
|
||||
|
||||
[loop old_vsn $OLD_VSNS]
|
||||
|
||||
[shell http_server]
|
||||
!cd http_server
|
||||
!rebar3 shell
|
||||
???Eshell
|
||||
???>
|
||||
!http_server:start().
|
||||
?Start http_server listener on 8080 successfully.
|
||||
?ok
|
||||
?>
|
||||
|
||||
[shell emqx]
|
||||
!cd $PACKAGE_PATH
|
||||
!unzip -q -o emqx-ubuntu20.04-$(echo $old_vsn | sed -r 's/[v|e]//g')-amd64.zip
|
||||
?SH-PROMPT
|
||||
|
||||
!cd emqx
|
||||
!sed -i 's|listener.wss.external[ \t]*=.*|listener.wss.external = 8085|g' etc/emqx.conf
|
||||
!sed -i '/emqx_telemetry/d' data/loaded_plugins
|
||||
!./bin/emqx start
|
||||
?EMQ X .* is started successfully!
|
||||
?SH-PROMPT
|
||||
|
||||
[shell emqx2]
|
||||
!cd $PACKAGE_PATH
|
||||
!cp -f $ONE_MORE_EMQX_PATH/one_more_emqx.sh .
|
||||
!./one_more_emqx.sh emqx2
|
||||
?SH-PROMPT
|
||||
!cd emqx2
|
||||
|
||||
!sed -i '/emqx_telemetry/d' data/loaded_plugins
|
||||
!./bin/emqx start
|
||||
?EMQ X (.*) is started successfully!
|
||||
?SH-PROMPT
|
||||
|
||||
!./bin/emqx_ctl cluster join emqx@127.0.0.1
|
||||
???Join the cluster successfully.
|
||||
?SH-PROMPT
|
||||
|
||||
!./bin/emqx_ctl cluster status
|
||||
"""???
|
||||
Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],
|
||||
stopped_nodes => []}
|
||||
"""
|
||||
?SH-PROMPT
|
||||
|
||||
!./bin/emqx_ctl resources create 'web_hook' -i 'resource:691c29ba' -c '{"url": "http://127.0.0.1:8080/counter", "method": "POST"}'
|
||||
?created
|
||||
?SH-PROMPT
|
||||
!./bin/emqx_ctl rules create 'SELECT * FROM "t/#"' '[{"name":"data_to_webserver", "params": {"$$resource": "resource:691c29ba"}}]'
|
||||
?created
|
||||
?SH-PROMPT
|
||||
|
||||
[shell emqx]
|
||||
!./bin/emqx_ctl resources list
|
||||
?691c29ba
|
||||
?SH-PROMPT
|
||||
!./bin/emqx_ctl rules list
|
||||
?691c29ba
|
||||
?SH-PROMPT
|
||||
|
||||
[shell bench]
|
||||
!cd $BENCH_PATH
|
||||
!./emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 600
|
||||
???sent
|
||||
|
||||
[shell emqx]
|
||||
!cp -f ../emqx-ubuntu20.04-$VSN-amd64.zip releases/
|
||||
!./bin/emqx install $VSN
|
||||
?SH-PROMPT
|
||||
!./bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]"
|
||||
?$VSN
|
||||
?SH-PROMPT
|
||||
|
||||
!./bin/emqx_ctl cluster status
|
||||
"""???
|
||||
Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],
|
||||
stopped_nodes => []}
|
||||
"""
|
||||
?SH-PROMPT
|
||||
|
||||
[shell emqx2]
|
||||
!cp -f ../emqx-ubuntu20.04-$VSN-amd64.zip releases/
|
||||
!./bin/emqx install $VSN
|
||||
?SH-PROMPT
|
||||
!./bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]"
|
||||
?$VSN
|
||||
?SH-PROMPT
|
||||
|
||||
!./bin/emqx_ctl cluster status
|
||||
"""???
|
||||
Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'],
|
||||
stopped_nodes => []}
|
||||
"""
|
||||
?SH-PROMPT
|
||||
|
||||
[shell bench]
|
||||
???publish complete
|
||||
??SH-PROMPT:
|
||||
!curl http://127.0.0.1:8080/counter
|
||||
???{"data":600,"code":0}
|
||||
?SH-PROMPT
|
||||
|
||||
[shell emqx2]
|
||||
!cat log/emqx.log.1 |grep -v 691c29ba |tail -n 100
|
||||
-error
|
||||
??SH-PROMPT:
|
||||
|
||||
!./bin/emqx stop
|
||||
?ok
|
||||
?SH-PROMPT:
|
||||
|
||||
!rm -rf $PACKAGE_PATH/emqx2
|
||||
?SH-PROMPT:
|
||||
|
||||
[shell emqx]
|
||||
!cat log/emqx.log.1 |grep -v 691c29ba |tail -n 100
|
||||
-error
|
||||
??SH-PROMPT:
|
||||
|
||||
!./bin/emqx stop
|
||||
?ok
|
||||
?SH-PROMPT:
|
||||
|
||||
!rm -rf $PACKAGE_PATH/emqx
|
||||
?SH-PROMPT:
|
||||
|
||||
[shell http_server]
|
||||
!http_server:stop().
|
||||
?ok
|
||||
?>
|
||||
!halt(3).
|
||||
?SH-PROMPT:
|
||||
|
||||
[endloop]
|
||||
|
||||
[cleanup]
|
||||
!echo ==$$?==
|
||||
?==0==
|
||||
27
.editorconfig
Normal file
27
.editorconfig
Normal file
@ -0,0 +1,27 @@
|
||||
# EditorConfig is awesome: https://EditorConfig.org
|
||||
|
||||
# top-most EditorConfig file
|
||||
root = true
|
||||
|
||||
# Unix-style newlines with a newline ending every file
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
|
||||
|
||||
# Matches multiple files with brace expansion notation
|
||||
# Set default charset
|
||||
[*.{erl, src, hrl}]
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
|
||||
# Tab indentation (no size specified)
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
|
||||
# Matches the exact files either package.json or .travis.yml
|
||||
[{.travis.yml}]
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
5
.gitattributes
vendored
Normal file
5
.gitattributes
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
* text=auto
|
||||
*.* text eol=lf
|
||||
*.jpg -text
|
||||
*.png -text
|
||||
*.pdf -text
|
||||
10
.github/ISSUE_TEMPLATE.md
vendored
Normal file
10
.github/ISSUE_TEMPLATE.md
vendored
Normal file
@ -0,0 +1,10 @@
|
||||
#### Environment
|
||||
|
||||
- OS:
|
||||
- Erlang/OTP:
|
||||
- EMQ:
|
||||
|
||||
#### Description
|
||||
|
||||
*A description of the issue*
|
||||
|
||||
26
.github/ISSUE_TEMPLATE/bug-report.md
vendored
Normal file
26
.github/ISSUE_TEMPLATE/bug-report.md
vendored
Normal file
@ -0,0 +1,26 @@
|
||||
---
|
||||
name: Bug Report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: Support
|
||||
assignees: tigercl
|
||||
|
||||
---
|
||||
|
||||
<!-- Please use this template while reporting a bug and provide as much info as possible. Thanks!-->
|
||||
<!-- 请使用英文描述问题 -->
|
||||
|
||||
**Environment**:
|
||||
|
||||
- EMQ X version (e.g. `emqx_ctl status`):
|
||||
- Hardware configuration (e.g. `lscpu`):
|
||||
- OS (e.g. `cat /etc/os-release`):
|
||||
- Kernel (e.g. `uname -a`):
|
||||
- Erlang/OTP version (in case you build emqx from source code):
|
||||
- Others:
|
||||
|
||||
**What happened and what you expected to happen**:
|
||||
|
||||
**How to reproduce it (as minimally and precisely as possible)**:
|
||||
|
||||
**Anything else we need to know?**:
|
||||
14
.github/ISSUE_TEMPLATE/feature-request.md
vendored
Normal file
14
.github/ISSUE_TEMPLATE/feature-request.md
vendored
Normal file
@ -0,0 +1,14 @@
|
||||
---
|
||||
name: Feature Request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: Feature
|
||||
assignees: tigercl
|
||||
|
||||
---
|
||||
|
||||
<!-- Please only use this template for submitting enhancement requests -->
|
||||
|
||||
**What would you like to be added/modified**:
|
||||
|
||||
**Why is this needed**:
|
||||
10
.github/ISSUE_TEMPLATE/support-needed.md
vendored
Normal file
10
.github/ISSUE_TEMPLATE/support-needed.md
vendored
Normal file
@ -0,0 +1,10 @@
|
||||
---
|
||||
name: Support Needed
|
||||
about: Asking a question about usages, docs or anything you're insterested in
|
||||
title: ''
|
||||
labels: Support
|
||||
assignees: tigercl
|
||||
|
||||
---
|
||||
|
||||
**Please describe your problem in detail, if necessary, you can upload the log file through the attachment**:
|
||||
15
.github/PULL_REQUEST_TEMPLATE.md
vendored
Normal file
15
.github/PULL_REQUEST_TEMPLATE.md
vendored
Normal file
@ -0,0 +1,15 @@
|
||||
<!-- Please describe the current behavior and link to a relevant issue. -->
|
||||
Fixes <issue-number>
|
||||
|
||||
**If your build fails** due to your commit message not passing the build checks, please review the guidelines here: https://hub.fastgit.org/emqx/emqx/blob/master/CONTRIBUTING.md.
|
||||
|
||||
## PR Checklist
|
||||
Please convert it to a draft if any of the following conditions are not met. Reviewers may skip over until all the items are checked:
|
||||
|
||||
- [ ] Tests for the changes have been added (for bug fixes / features)
|
||||
- [ ] Docs have been added / updated (for bug fixes / features)
|
||||
- [ ] In case of non-backward compatible changes, reviewer should check this item as a write-off, and add details in **Backward Compatibility** section
|
||||
|
||||
## Backward Compatibility
|
||||
|
||||
## More information
|
||||
7
.github/weekly-digest.yml
vendored
Normal file
7
.github/weekly-digest.yml
vendored
Normal file
@ -0,0 +1,7 @@
|
||||
# Configuration for weekly-digest - https://hub.fastgit.org/apps/weekly-digest
|
||||
publishDay: monday
|
||||
canPublishIssues: true
|
||||
canPublishPullRequests: true
|
||||
canPublishContributors: true
|
||||
canPublishStargazers: true
|
||||
canPublishCommits: true
|
||||
119
.github/workflows/.gitlint
vendored
Normal file
119
.github/workflows/.gitlint
vendored
Normal file
@ -0,0 +1,119 @@
|
||||
# Edit this file as you like.
|
||||
#
|
||||
# All these sections are optional. Each section with the exception of [general] represents
|
||||
# one rule and each key in it is an option for that specific rule.
|
||||
#
|
||||
# Rules and sections can be referenced by their full name or by id. For example
|
||||
# section "[body-max-line-length]" could also be written as "[B1]". Full section names are
|
||||
# used in here for clarity.
|
||||
#
|
||||
[general]
|
||||
# Ignore certain rules, this example uses both full name and id
|
||||
ignore=title-trailing-punctuation, T1, T2, T3, T4, T5, T6, T8, B1, B2, B3, B4, B5, B6, B7, B8
|
||||
|
||||
# verbosity should be a value between 1 and 3, the commandline -v flags take precedence over this
|
||||
# verbosity = 2
|
||||
|
||||
# By default gitlint will ignore merge, revert, fixup and squash commits.
|
||||
# ignore-merge-commits=true
|
||||
# ignore-revert-commits=true
|
||||
# ignore-fixup-commits=true
|
||||
# ignore-squash-commits=true
|
||||
|
||||
# Ignore any data send to gitlint via stdin
|
||||
# ignore-stdin=true
|
||||
|
||||
# Fetch additional meta-data from the local repository when manually passing a
|
||||
# commit message to gitlint via stdin or --commit-msg. Disabled by default.
|
||||
# staged=true
|
||||
|
||||
# Enable debug mode (prints more output). Disabled by default.
|
||||
# debug=true
|
||||
|
||||
# Enable community contributed rules
|
||||
# See http://jorisroovers.github.io/gitlint/contrib_rules for details
|
||||
# contrib=contrib-title-conventional-commits,CC1
|
||||
|
||||
# Set the extra-path where gitlint will search for user defined rules
|
||||
# See http://jorisroovers.github.io/gitlint/user_defined_rules for details
|
||||
# extra-path=examples/
|
||||
|
||||
# This is an example of how to configure the "title-max-length" rule and
|
||||
# set the line-length it enforces to 80
|
||||
# [title-max-length]
|
||||
# line-length=50
|
||||
|
||||
# Conversely, you can also enforce minimal length of a title with the
|
||||
# "title-min-length" rule:
|
||||
# [title-min-length]
|
||||
# min-length=5
|
||||
|
||||
# [title-must-not-contain-word]
|
||||
# Comma-separated list of words that should not occur in the title. Matching is case
|
||||
# insensitive. It's fine if the keyword occurs as part of a larger word (so "WIPING"
|
||||
# will not cause a violation, but "WIP: my title" will.
|
||||
# words=wip
|
||||
|
||||
[title-match-regex]
|
||||
# python-style regex that the commit-msg title must match
|
||||
# Note that the regex can contradict with other rules if not used correctly
|
||||
# (e.g. title-must-not-contain-word).
|
||||
regex=^(feat|fix|docs|style|refactor|test|build|ci|revert|chore|perf)(\(.+\))*: .+
|
||||
|
||||
# [body-max-line-length]
|
||||
# line-length=72
|
||||
|
||||
# [body-min-length]
|
||||
# min-length=5
|
||||
|
||||
# [body-is-missing]
|
||||
# Whether to ignore this rule on merge commits (which typically only have a title)
|
||||
# default = True
|
||||
# ignore-merge-commits=false
|
||||
|
||||
# [body-changed-file-mention]
|
||||
# List of files that need to be explicitly mentioned in the body when they are changed
|
||||
# This is useful for when developers often erroneously edit certain files or git submodules.
|
||||
# By specifying this rule, developers can only change the file when they explicitly reference
|
||||
# it in the commit message.
|
||||
# files=gitlint/rules.py,README.md
|
||||
|
||||
# [body-match-regex]
|
||||
# python-style regex that the commit-msg body must match.
|
||||
# E.g. body must end in My-Commit-Tag: foo
|
||||
# regex=My-Commit-Tag: foo$
|
||||
|
||||
# [author-valid-email]
|
||||
# python-style regex that the commit author email address must match.
|
||||
# For example, use the following regex if you only want to allow email addresses from foo.com
|
||||
# regex=[^@]+@foo.com
|
||||
|
||||
[ignore-by-title]
|
||||
# Ignore certain rules for commits of which the title matches a regex
|
||||
# E.g. Match commit titles that start with "Release"
|
||||
# regex=^Release(.*)
|
||||
|
||||
# Ignore certain rules, you can reference them by their id or by their full name
|
||||
# Use 'all' to ignore all rules
|
||||
# ignore=T1,body-min-length
|
||||
|
||||
[ignore-by-body]
|
||||
# Ignore certain rules for commits of which the body has a line that matches a regex
|
||||
# E.g. Match bodies that have a line that that contain "release"
|
||||
# regex=(.*)release(.*)
|
||||
#
|
||||
# Ignore certain rules, you can reference them by their id or by their full name
|
||||
# Use 'all' to ignore all rules
|
||||
# ignore=T1,body-min-length
|
||||
|
||||
# [ignore-body-lines]
|
||||
# Ignore certain lines in a commit body that match a regex.
|
||||
# E.g. Ignore all lines that start with 'Co-Authored-By'
|
||||
# regex=^Co-Authored-By
|
||||
|
||||
# This is a contrib rule - a community contributed rule. These are disabled by default.
|
||||
# You need to explicitly enable them one-by-one by adding them to the "contrib" option
|
||||
# under [general] section above.
|
||||
# [contrib-title-conventional-commits]
|
||||
# Specify allowed commit types. For details see: https://www.conventionalcommits.org/
|
||||
# types = bugfix,user-story,epic
|
||||
12
.github/workflows/apps_version_check.yaml
vendored
Normal file
12
.github/workflows/apps_version_check.yaml
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
name: Check Apps Version
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
check_apps_version:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: Check apps version
|
||||
run: ./scripts/apps-version-check.sh
|
||||
480
.github/workflows/build_packages.yaml
vendored
Normal file
480
.github/workflows/build_packages.yaml
vendored
Normal file
@ -0,0 +1,480 @@
|
||||
name: Cross build packages
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 */6 * * *'
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
- e*
|
||||
release:
|
||||
types:
|
||||
- published
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
runs-on: ubuntu-20.04
|
||||
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
|
||||
|
||||
outputs:
|
||||
profiles: ${{ steps.set_profile.outputs.profiles}}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
path: source
|
||||
- name: set profile
|
||||
id: set_profile
|
||||
shell: bash
|
||||
run: |
|
||||
if make -C source emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
echo "::set-output name=profiles::[\"emqx-ee\"]"
|
||||
else
|
||||
echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
|
||||
fi
|
||||
- name: get_all_deps
|
||||
if: endsWith(github.repository, 'emqx')
|
||||
run: |
|
||||
make -C source deps-all
|
||||
zip -ryq source.zip source
|
||||
- name: get_all_deps
|
||||
if: endsWith(github.repository, 'enterprise')
|
||||
run: |
|
||||
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org" > $HOME/.git-credentials
|
||||
git config --global credential.helper store
|
||||
echo "${{ secrets.CI_GIT_TOKEN }}" >> source/scripts/git-token
|
||||
make -C source deps-all
|
||||
zip -ryq source.zip source
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: source
|
||||
path: source.zip
|
||||
|
||||
windows:
|
||||
runs-on: windows-2019
|
||||
|
||||
needs: prepare
|
||||
if: endsWith(github.repository, 'emqx')
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
|
||||
exclude:
|
||||
- profile: emqx-edge
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: source
|
||||
path: .
|
||||
- name: unzip source code
|
||||
run: Expand-Archive -Path source.zip -DestinationPath ./
|
||||
- uses: ilammy/msvc-dev-cmd@v1
|
||||
- uses: gleam-lang/setup-erlang@v1.1.0
|
||||
id: install_erlang
|
||||
with:
|
||||
otp-version: 23.2
|
||||
- name: build
|
||||
env:
|
||||
PYTHON: python
|
||||
run: |
|
||||
$env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH"
|
||||
|
||||
$version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
|
||||
if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
|
||||
$regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
|
||||
$pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip"
|
||||
}
|
||||
else {
|
||||
$pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip"
|
||||
}
|
||||
cd source
|
||||
## We do not build/release bcrypt for windows package
|
||||
Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/
|
||||
if (Test-Path rebar.lock) {
|
||||
Remove-Item -Force -Path rebar.lock
|
||||
}
|
||||
make ${{ matrix.profile }}
|
||||
mkdir -p _packages/${{ matrix.profile }}
|
||||
Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name
|
||||
mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }}
|
||||
Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}' > _packages/${{ matrix.profile }}/$pkg_name.sha256
|
||||
- name: run emqx
|
||||
timeout-minutes: 1
|
||||
run: |
|
||||
cd source
|
||||
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
|
||||
Start-Sleep -s 5
|
||||
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
|
||||
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
|
||||
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
path: source/_packages/${{ matrix.profile }}/.
|
||||
|
||||
mac:
|
||||
runs-on: macos-10.15
|
||||
|
||||
needs: prepare
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
|
||||
erl_otp:
|
||||
- 23.2.7.2
|
||||
exclude:
|
||||
- profile: emqx-edge
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: source
|
||||
path: .
|
||||
- name: unzip source code
|
||||
run: unzip -q source.zip
|
||||
- name: prepare
|
||||
run: |
|
||||
brew update
|
||||
brew install curl zip unzip gnu-sed kerl unixodbc freetds
|
||||
echo "/usr/local/bin" >> $GITHUB_PATH
|
||||
git config --global credential.helper store
|
||||
- uses: actions/cache@v2
|
||||
id: cache
|
||||
with:
|
||||
path: ~/.kerl
|
||||
key: erl${{ matrix.erl_otp }}-macos10.15
|
||||
- name: build erlang
|
||||
if: steps.cache.outputs.cache-hit != 'true'
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
kerl build ${{ matrix.erl_otp }}
|
||||
kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
|
||||
- name: build
|
||||
run: |
|
||||
. $HOME/.kerl/${{ matrix.erl_otp }}/activate
|
||||
make -C source ${{ matrix.profile }}-zip
|
||||
- name: test
|
||||
run: |
|
||||
cd source
|
||||
pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip)
|
||||
unzip -q _packages/${{ matrix.profile }}/$pkg_name
|
||||
gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
|
||||
./emqx/bin/emqx start || cat emqx/log/erlang.log.1
|
||||
ready='no'
|
||||
for i in {1..10}; do
|
||||
if curl -fs 127.0.0.1:18083 > /dev/null; then
|
||||
ready='yes'
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$ready" != "yes" ]; then
|
||||
echo "Timed out waiting for emqx to be ready"
|
||||
cat emqx/log/erlang.log.1
|
||||
exit 1
|
||||
fi
|
||||
./emqx/bin/emqx_ctl status
|
||||
./emqx/bin/emqx stop
|
||||
rm -rf emqx
|
||||
openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}' > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
path: source/_packages/${{ matrix.profile }}/.
|
||||
|
||||
linux:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
needs: prepare
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
|
||||
arch:
|
||||
- amd64
|
||||
- arm64
|
||||
os:
|
||||
- ubuntu20.04
|
||||
- ubuntu18.04
|
||||
- ubuntu16.04
|
||||
- debian10
|
||||
- debian9
|
||||
# - opensuse
|
||||
- centos8
|
||||
- centos7
|
||||
- centos6
|
||||
- raspbian10
|
||||
- raspbian9
|
||||
exclude:
|
||||
- os: centos6
|
||||
arch: arm64
|
||||
- os: raspbian9
|
||||
arch: amd64
|
||||
- os: raspbian10
|
||||
arch: amd64
|
||||
- os: raspbian9
|
||||
profile: emqx
|
||||
- os: raspbian10
|
||||
profile: emqx
|
||||
- os: raspbian9
|
||||
profile: emqx-ee
|
||||
- os: raspbian10
|
||||
profile: emqx-ee
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
steps:
|
||||
- name: prepare docker
|
||||
run: |
|
||||
mkdir -p $HOME/.docker
|
||||
echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
|
||||
echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
docker info
|
||||
docker buildx create --use --name mybuild
|
||||
docker run --rm --privileged tonistiigi/binfmt --install all
|
||||
- uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: source
|
||||
path: .
|
||||
- name: unzip source code
|
||||
run: unzip -q source.zip
|
||||
- name: downloads emqx zip packages
|
||||
env:
|
||||
PROFILE: ${{ matrix.profile }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
SYSTEM: ${{ matrix.os }}
|
||||
run: |
|
||||
set -e -u -x
|
||||
cd source
|
||||
if [ $PROFILE = "emqx" ];then broker="emqx-ce"; else broker="$PROFILE"; fi
|
||||
if [ $PROFILE = "emqx-ee" ];then edition='enterprise'; else edition='opensource'; fi
|
||||
|
||||
vsn="$(./pkg-vsn.sh)"
|
||||
pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
|
||||
if [ $PROFILE = "emqx-ee" ]; then
|
||||
old_vsns=($(git tag -l "e$pre_vsn.[0-9]" | sed "s/e$vsn//"))
|
||||
else
|
||||
old_vsns=($(git tag -l "v$pre_vsn.[0-9]" | sed "s/v$vsn//"))
|
||||
fi
|
||||
|
||||
mkdir -p _upgrade_base
|
||||
cd _upgrade_base
|
||||
for tag in ${old_vsns[@]};do
|
||||
if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
|
||||
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip
|
||||
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256
|
||||
echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1
|
||||
fi
|
||||
done
|
||||
cd -
|
||||
- name: build emqx packages
|
||||
env:
|
||||
ERL_OTP: erl23.2.7.2-emqx-2
|
||||
PROFILE: ${{ matrix.profile }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
SYSTEM: ${{ matrix.os }}
|
||||
run: |
|
||||
set -e -u
|
||||
cd source
|
||||
docker buildx build --no-cache \
|
||||
--platform=linux/$ARCH \
|
||||
-t cross_build_emqx_for_$SYSTEM \
|
||||
-f .ci/build_packages/Dockerfile \
|
||||
--build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \
|
||||
--build-arg EMQX_NAME=$PROFILE \
|
||||
--output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar .
|
||||
|
||||
mkdir -p /tmp/packages/$PROFILE
|
||||
tar -xvf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar --wildcards emqx/_packages/$PROFILE/*
|
||||
mv emqx/_packages/$PROFILE/* /tmp/packages/$PROFILE/
|
||||
rm -rf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar
|
||||
|
||||
docker rm -f $(docker ps -a -q)
|
||||
docker volume prune -f
|
||||
- name: create sha256
|
||||
env:
|
||||
PROFILE: ${{ matrix.profile}}
|
||||
run: |
|
||||
if [ -d /tmp/packages/$PROFILE ]; then
|
||||
cd /tmp/packages/$PROFILE
|
||||
for var in $(ls emqx-* ); do
|
||||
bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"
|
||||
done
|
||||
cd -
|
||||
fi
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
path: /tmp/packages/${{ matrix.profile }}/.
|
||||
|
||||
docker:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
needs: prepare
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
|
||||
arch:
|
||||
- [amd64, x86_64]
|
||||
- [arm64v8, aarch64]
|
||||
- [arm32v7, arm]
|
||||
- [i386, i386]
|
||||
- [s390x, s390x]
|
||||
exclude:
|
||||
- profile: emqx-ee
|
||||
arch: [i386, i386]
|
||||
- profile: emqx-ee
|
||||
arch: [s390x, s390x]
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: source
|
||||
path: .
|
||||
- name: unzip source code
|
||||
run: unzip -q source.zip
|
||||
- name: build emqx docker image
|
||||
env:
|
||||
PROFILE: ${{ matrix.profile }}
|
||||
ARCH: ${{ matrix.arch[0] }}
|
||||
QEMU_ARCH: ${{ matrix.arch[1] }}
|
||||
run: |
|
||||
sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
||||
|
||||
cd source
|
||||
sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH make docker
|
||||
cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
path: source/_packages/${{ matrix.profile }}/.
|
||||
|
||||
delete-artifact:
|
||||
runs-on: ubuntu-20.04
|
||||
needs: [prepare, mac, linux, docker]
|
||||
steps:
|
||||
- uses: geekyeggo/delete-artifact@v1
|
||||
with:
|
||||
name: source
|
||||
|
||||
upload:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
|
||||
needs: [prepare, mac, linux, docker]
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: get_version
|
||||
run: |
|
||||
echo 'version<<EOF' >> $GITHUB_ENV
|
||||
echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g" >> $GITHUB_ENV
|
||||
echo 'EOF' >> $GITHUB_ENV
|
||||
- uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
path: ./_packages/${{ matrix.profile }}
|
||||
- name: install dos2unix
|
||||
run: sudo apt-get update && sudo apt install -y dos2unix
|
||||
- name: get packages
|
||||
run: |
|
||||
set -e -u
|
||||
cd _packages/${{ matrix.profile }}
|
||||
for var in $( ls |grep emqx |grep -v sha256); do
|
||||
dos2unix $var.sha256
|
||||
echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
|
||||
done
|
||||
cd -
|
||||
- name: upload aws s3
|
||||
run: |
|
||||
set -e -u
|
||||
if [ "${{ matrix.profile }}" == "emqx" ];then
|
||||
broker="emqx-ce"
|
||||
else
|
||||
broker=${{ matrix.profile }}
|
||||
fi
|
||||
aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws configure set default.region ${{ secrets.AWS_DEFAULT_REGION }}
|
||||
|
||||
aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }}
|
||||
aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*"
|
||||
- uses: Rory-Z/upload-release-asset@v1
|
||||
if: github.event_name == 'release' && matrix.profile != 'emqx-ee'
|
||||
with:
|
||||
repo: emqx
|
||||
path: "_packages/${{ matrix.profile }}/emqx-*"
|
||||
token: ${{ github.token }}
|
||||
- uses: Rory-Z/upload-release-asset@v1
|
||||
if: github.event_name == 'release' && matrix.profile == 'emqx-ee'
|
||||
with:
|
||||
repo: emqx-enterprise
|
||||
path: "_packages/${{ matrix.profile }}/emqx-*"
|
||||
token: ${{ github.token }}
|
||||
- name: update to emqx.io
|
||||
if: github.event_name == 'release'
|
||||
run: |
|
||||
set -e -x -u
|
||||
curl -w %{http_code} \
|
||||
--insecure \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
|
||||
-X POST \
|
||||
-d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
|
||||
${{ secrets.EMQX_IO_RELEASE_API }}
|
||||
- name: push docker image to docker hub
|
||||
if: github.event_name == 'release'
|
||||
run: |
|
||||
set -e -x -u
|
||||
sudo make docker-prepare
|
||||
cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd -
|
||||
echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
|
||||
sudo TARGET=emqx/${{ matrix.profile }} make docker-push
|
||||
sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list
|
||||
- name: update repo.emqx.io
|
||||
if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
|
||||
run: |
|
||||
curl --silent --show-error \
|
||||
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
-X POST \
|
||||
-d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \
|
||||
"https://api.hub.fastgit.org/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
|
||||
- name: update repo.emqx.io
|
||||
if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
|
||||
run: |
|
||||
curl --silent --show-error \
|
||||
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
-X POST \
|
||||
-d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \
|
||||
"https://api.hub.fastgit.org/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
|
||||
- name: update homebrew packages
|
||||
if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
|
||||
run: |
|
||||
if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
|
||||
curl --silent --show-error \
|
||||
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
-X POST \
|
||||
-d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \
|
||||
"https://api.hub.fastgit.org/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
|
||||
fi
|
||||
- uses: geekyeggo/delete-artifact@v1
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
116
.github/workflows/build_slim_packages.yaml
vendored
Normal file
116
.github/workflows/build_slim_packages.yaml
vendored
Normal file
@ -0,0 +1,116 @@
|
||||
name: Build slim packages
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
- e*
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
erl_otp:
|
||||
- erl23.2.7.2-emqx-2
|
||||
os:
|
||||
- ubuntu20.04
|
||||
- centos7
|
||||
|
||||
container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: prepare
|
||||
run: |
|
||||
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org" > $HOME/.git-credentials
|
||||
git config --global credential.helper store
|
||||
echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token
|
||||
echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
|
||||
else
|
||||
echo "EMQX_NAME=emqx" >> $GITHUB_ENV
|
||||
fi
|
||||
- name: build zip packages
|
||||
run: make ${EMQX_NAME}-zip
|
||||
- name: build deb/rpm packages
|
||||
run: make ${EMQX_NAME}-pkg
|
||||
- name: pakcages test
|
||||
run: |
|
||||
export CODE_PATH=$GITHUB_WORKSPACE
|
||||
.ci/build_packages/tests.sh
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ matrix.os }}
|
||||
path: _packages/**/*.zip
|
||||
|
||||
mac:
|
||||
runs-on: macos-10.15
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
erl_otp:
|
||||
- 23.2.7.2
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: prepare
|
||||
run: |
|
||||
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org" > $HOME/.git-credentials
|
||||
git config --global credential.helper store
|
||||
echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token
|
||||
echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
|
||||
else
|
||||
echo "EMQX_NAME=emqx" >> $GITHUB_ENV
|
||||
fi
|
||||
- name: prepare
|
||||
run: |
|
||||
brew update
|
||||
brew install curl zip unzip gnu-sed kerl unixodbc freetds
|
||||
echo "/usr/local/bin" >> $GITHUB_PATH
|
||||
git config --global credential.helper store
|
||||
- uses: actions/cache@v2
|
||||
id: cache
|
||||
with:
|
||||
path: ~/.kerl
|
||||
key: erl${{ matrix.erl_otp }}-macos10.15
|
||||
- name: build erlang
|
||||
if: steps.cache.outputs.cache-hit != 'true'
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
kerl build ${{ matrix.erl_otp }}
|
||||
kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
|
||||
- name: build
|
||||
run: |
|
||||
. $HOME/.kerl/${{ matrix.erl_otp }}/activate
|
||||
make ${EMQX_NAME}-zip
|
||||
- name: test
|
||||
run: |
|
||||
pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip)
|
||||
unzip -q _packages/${EMQX_NAME}/$pkg_name
|
||||
gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
|
||||
./emqx/bin/emqx start || cat emqx/log/erlang.log.1
|
||||
ready='no'
|
||||
for i in {1..10}; do
|
||||
if curl -fs 127.0.0.1:18083 > /dev/null; then
|
||||
ready='yes'
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$ready" != "yes" ]; then
|
||||
echo "Timed out waiting for emqx to be ready"
|
||||
cat emqx/log/erlang.log.1
|
||||
exit 1
|
||||
fi
|
||||
./emqx/bin/emqx_ctl status
|
||||
./emqx/bin/emqx stop
|
||||
rm -rf emqx
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: macos
|
||||
path: _packages/**/*.zip
|
||||
13
.github/workflows/check_deps_integrity.yaml
vendored
Normal file
13
.github/workflows/check_deps_integrity.yaml
vendored
Normal file
@ -0,0 +1,13 @@
|
||||
name: Check Rebar Dependencies
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
check_deps_integrity:
|
||||
runs-on: ubuntu-20.04
|
||||
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Run check-deps-integrity.escript
|
||||
run: ./scripts/check-deps-integrity.escript
|
||||
16
.github/workflows/elvis_lint.yaml
vendored
Normal file
16
.github/workflows/elvis_lint.yaml
vendored
Normal file
@ -0,0 +1,16 @@
|
||||
name: Elvis Linter
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set git token
|
||||
if: endsWith(github.repository, 'enterprise')
|
||||
run: |
|
||||
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org" > $HOME/.git-credentials
|
||||
git config --global credential.helper store
|
||||
- run: |
|
||||
./scripts/elvis-check.sh $GITHUB_BASE_REF
|
||||
42
.github/workflows/git_sync.yaml
vendored
Normal file
42
.github/workflows/git_sync.yaml
vendored
Normal file
@ -0,0 +1,42 @@
|
||||
name: Sync to enterprise
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
sync_to_enterprise:
|
||||
runs-on: ubuntu-20.04
|
||||
if: github.repository == 'emqx/emqx'
|
||||
steps:
|
||||
- name: git-sync
|
||||
uses: Rory-Z/git-sync@v3.0.1
|
||||
with:
|
||||
source_repo: ${{ github.repository }}
|
||||
source_branch: ${{ github.ref }}
|
||||
destination_repo: "${{ github.repository_owner }}/emqx-enterprise"
|
||||
destination_branch: ${{ github.ref }}
|
||||
destination_ssh_private_key: "${{ secrets.CI_SSH_PRIVATE_KEY }}"
|
||||
- name: create pull request
|
||||
id: create_pull_request
|
||||
run: |
|
||||
set -euo pipefail
|
||||
R=$(curl --silent --show-error \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
|
||||
-X POST \
|
||||
-d '{"title": "Sync code into enterprise from opensource", "head": "master", "base":"enterprise"}' \
|
||||
https://api.hub.fastgit.org/repos/${{ github.repository_owner }}/emqx-enterprise/pulls)
|
||||
echo $R | jq
|
||||
echo "::set-output name=url::$(echo $R | jq '.url')"
|
||||
- name: request reviewers for a pull request
|
||||
if: steps.create_pull_request.outputs.url != 'null'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl --silent --show-error \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
|
||||
-X POST \
|
||||
-d '{"team_reviewers":["emqx-devs"]}' \
|
||||
${{ steps.create_pull_request.outputs.url }}/requested_reviewers
|
||||
28
.github/workflows/gitee-repos-mirror.yml
vendored
28
.github/workflows/gitee-repos-mirror.yml
vendored
@ -1,28 +0,0 @@
|
||||
name: Gitee repos mirror periodic job
|
||||
|
||||
on:
|
||||
push:
|
||||
watch:
|
||||
types: started
|
||||
schedule:
|
||||
- cron: "0 23 * * *"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
|
||||
- name: Mirror the Github organization repos to Gitee.
|
||||
uses: Yikun/hub-mirror-action@v1.0
|
||||
with:
|
||||
src: github/dgiot
|
||||
dst: gitee/dgiiot
|
||||
dst_key: ${{ secrets.PRIVATE_KEY }}
|
||||
dst_token: ${{ secrets.TOKEN }}
|
||||
account_type: org
|
||||
timeout: 600
|
||||
debug: true
|
||||
force_update: true
|
||||
black_list: "dgiot.github.io, issue-generator, dgiot_amazedtu"
|
||||
406
.github/workflows/run_cts_tests.yaml
vendored
Normal file
406
.github/workflows/run_cts_tests.yaml
vendored
Normal file
@ -0,0 +1,406 @@
|
||||
name: Compatibility Test Suite
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
- e*
|
||||
release:
|
||||
types:
|
||||
- published
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
ldap:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
ldap_tag:
|
||||
- 2.4.50
|
||||
network_type:
|
||||
- ipv4
|
||||
- ipv6
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: docker compose up
|
||||
env:
|
||||
LDAP_TAG: ${{ matrix.ldap_tag }}
|
||||
run: |
|
||||
docker-compose \
|
||||
-f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose.yaml \
|
||||
up -d --build
|
||||
- name: setup
|
||||
if: matrix.network_type == 'ipv4'
|
||||
run: |
|
||||
echo EMQX_AUTH__LDAP__SERVERS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ldap) >> "$GITHUB_ENV"
|
||||
- name: setup
|
||||
if: matrix.network_type == 'ipv6'
|
||||
run: |
|
||||
echo EMQX_AUTH__LDAP__SERVERS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' ldap) >> "$GITHUB_ENV"
|
||||
- name: set git token
|
||||
run: |
|
||||
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org\" > /root/.git-credentials && git config --global credential.helper store"
|
||||
fi
|
||||
- name: run test cases
|
||||
run: |
|
||||
export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
|
||||
printenv > .env
|
||||
docker exec -i erlang sh -c "make ensure-rebar3"
|
||||
docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_ldap"
|
||||
docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap"
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: logs_ldap${{ matrix.ldap_tag }}_${{ matrix.network_type }}
|
||||
path: _build/test/logs
|
||||
|
||||
mongo:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mongo_tag:
|
||||
- 3
|
||||
- 4
|
||||
network_type:
|
||||
- ipv4
|
||||
- ipv6
|
||||
connect_type:
|
||||
- tls
|
||||
- tcp
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: docker-compose up
|
||||
run: |
|
||||
docker-compose \
|
||||
-f .ci/docker-compose-file/docker-compose-mongo-${{ matrix.connect_type }}.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose.yaml \
|
||||
up -d --build
|
||||
- name: setup
|
||||
env:
|
||||
MONGO_TAG: ${{ matrix.mongo_tag }}
|
||||
if: matrix.connect_type == 'tls'
|
||||
run: |
|
||||
cat <<-EOF >> "$GITHUB_ENV"
|
||||
EMQX_AUTH__MONGO__SSL=on
|
||||
EMQX_AUTH__MONGO__SSL__CACERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem
|
||||
EMQX_AUTH__MONGO__SSL__CERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem
|
||||
EMQX_AUTH__MONGO__SSL__KEYFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem
|
||||
EMQX_AUTH__MONGO__SSL__VERIFY=true
|
||||
EMQX_AUTH__MONGO__SSL__SERVER_NAME_INDICATION=disable
|
||||
EOF
|
||||
- name: setup
|
||||
env:
|
||||
MONGO_TAG: ${{ matrix.mongo_tag }}
|
||||
if: matrix.connect_type == 'tcp'
|
||||
run: |
|
||||
echo EMQX_AUTH__MONGO__SSL=off >> "$GITHUB_ENV"
|
||||
- name: setup
|
||||
if: matrix.network_type == 'ipv4'
|
||||
run: |
|
||||
echo "EMQX_AUTH__MONGO__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo):27017" >> "$GITHUB_ENV"
|
||||
- name: setup
|
||||
if: matrix.network_type == 'ipv6'
|
||||
run: |
|
||||
echo "EMQX_AUTH__MONGO__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mongo):27017" >> "$GITHUB_ENV"
|
||||
- name: set git token
|
||||
run: |
|
||||
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org\" > /root/.git-credentials && git config --global credential.helper store"
|
||||
fi
|
||||
- name: run test cases
|
||||
run: |
|
||||
export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
|
||||
printenv > .env
|
||||
docker exec -i erlang sh -c "make ensure-rebar3"
|
||||
docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mongo"
|
||||
docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mongo"
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: logs_mongo${{ matrix.mongo_tag }}_${{ matrix.network_type }}_${{ matrix.connect_type }}
|
||||
path: _build/test/logs
|
||||
|
||||
mysql:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mysql_tag:
|
||||
- 5.7
|
||||
- 8
|
||||
network_type:
|
||||
- ipv4
|
||||
- ipv6
|
||||
connect_type:
|
||||
- tls
|
||||
- tcp
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: docker-compose up
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
docker-compose \
|
||||
-f .ci/docker-compose-file/docker-compose-mysql-${{ matrix.connect_type }}.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose.yaml \
|
||||
up -d --build
|
||||
while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
|
||||
!= $(docker ps -a --filter name=client | wc -l) ]; do
|
||||
sleep 5
|
||||
done
|
||||
- name: setup
|
||||
env:
|
||||
MYSQL_TAG: ${{ matrix.mysql_tag }}
|
||||
if: matrix.connect_type == 'tls'
|
||||
run: |
|
||||
cat <<-EOF >> "$GITHUB_ENV"
|
||||
EMQX_AUTH__MYSQL__USERNAME=ssluser
|
||||
EMQX_AUTH__MYSQL__PASSWORD=public
|
||||
EMQX_AUTH__MYSQL__DATABASE=mqtt
|
||||
EMQX_AUTH__MYSQL__SSL=on
|
||||
EMQX_AUTH__MYSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem
|
||||
EMQX_AUTH__MYSQL__SSL__CERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem
|
||||
EMQX_AUTH__MYSQL__SSL__KEYFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem
|
||||
EMQX_AUTH__MYSQL__SSL__VERIFY=true
|
||||
EMQX_AUTH__MYSQL__SSL__SERVER_NAME_INDICATION=disable
|
||||
EOF
|
||||
- name: setup
|
||||
env:
|
||||
MYSQL_TAG: ${{ matrix.mysql_tag }}
|
||||
if: matrix.connect_type == 'tcp'
|
||||
run: |
|
||||
cat <<-EOF >> "$GITHUB_ENV"
|
||||
EMQX_AUTH__MYSQL__USERNAME=root
|
||||
EMQX_AUTH__MYSQL__PASSWORD=public
|
||||
EMQX_AUTH__MYSQL__DATABASE=mqtt
|
||||
EMQX_AUTH__MYSQL__SSL=off
|
||||
EOF
|
||||
- name: setup
|
||||
if: matrix.network_type == 'ipv4'
|
||||
run: |
|
||||
echo "EMQX_AUTH__MYSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql):3306" >> "$GITHUB_ENV"
|
||||
- name: setup
|
||||
if: matrix.network_type == 'ipv6'
|
||||
run: |
|
||||
echo "EMQX_AUTH__MYSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mysql):3306" >> "$GITHUB_ENV"
|
||||
- name: set git token
|
||||
run: |
|
||||
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org\" > /root/.git-credentials && git config --global credential.helper store"
|
||||
fi
|
||||
- name: run test cases
|
||||
run: |
|
||||
export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
|
||||
printenv > .env
|
||||
docker exec -i erlang sh -c "make ensure-rebar3"
|
||||
docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mysql"
|
||||
docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mysql"
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: logs_mysql${{ matrix.mysql_tag }}_${{ matrix.network_type }}_${{ matrix.connect_type }}
|
||||
path: _build/test/logs
|
||||
|
||||
pgsql:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
pgsql_tag:
|
||||
- 9
|
||||
- 10
|
||||
- 11
|
||||
- 12
|
||||
- 13
|
||||
network_type:
|
||||
- ipv4
|
||||
- ipv6
|
||||
connect_type:
|
||||
- tls
|
||||
- tcp
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: docker-compose up
|
||||
run: |
|
||||
docker-compose \
|
||||
-f .ci/docker-compose-file/docker-compose-pgsql-${{ matrix.connect_type }}.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose.yaml \
|
||||
up -d --build
|
||||
- name: setup
|
||||
env:
|
||||
PGSQL_TAG: ${{ matrix.pgsql_tag }}
|
||||
if: matrix.connect_type == 'tls'
|
||||
run: |
|
||||
cat <<-EOF >> "$GITHUB_ENV"
|
||||
EMQX_AUTH__PGSQL__SSL=on
|
||||
EMQX_AUTH__PGSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem
|
||||
EMQX_AUTH__PGSQL__SSL__CERTFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem
|
||||
EMQX_AUTH__PGSQL__SSL__KEYFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem
|
||||
EMQX_AUTH__PGSQL__SSL__VERIFY=true
|
||||
EMQX_AUTH__PGSQL__SSL__SERVER_NAME_INDICATION=disable
|
||||
EOF
|
||||
- name: setup
|
||||
env:
|
||||
PGSQL_TAG: ${{ matrix.pgsql_tag }}
|
||||
if: matrix.connect_type == 'tcp'
|
||||
run: |
|
||||
echo EMQX_AUTH__PGSQL__SSL=off >> "$GITHUB_ENV"
|
||||
- name: setup
|
||||
if: matrix.network_type == 'ipv4'
|
||||
run: |
|
||||
echo "EMQX_AUTH__PGSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql):5432" >> "$GITHUB_ENV"
|
||||
- name: setup
|
||||
if: matrix.network_type == 'ipv6'
|
||||
run: |
|
||||
echo "EMQX_AUTH__PGSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' pgsql):5432" >> "$GITHUB_ENV"
|
||||
- name: set git token
|
||||
run: |
|
||||
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org\" > /root/.git-credentials && git config --global credential.helper store"
|
||||
fi
|
||||
- name: run test cases
|
||||
run: |
|
||||
export EMQX_AUTH__PGSQL__USERNAME=root \
|
||||
EMQX_AUTH__PGSQL__PASSWORD=public \
|
||||
EMQX_AUTH__PGSQL__DATABASE=mqtt \
|
||||
CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
|
||||
printenv > .env
|
||||
docker exec -i erlang sh -c "make ensure-rebar3"
|
||||
docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_pgsql"
|
||||
docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_pgsql"
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: logs_pgsql${{ matrix.pgsql_tag }}_${{ matrix.network_type }}_${{ matrix.connect_type }}
|
||||
path: _build/test/logs
|
||||
|
||||
redis:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
redis_tag:
|
||||
- 5
|
||||
- 6
|
||||
network_type:
|
||||
- ipv4
|
||||
- ipv6
|
||||
connect_type:
|
||||
- tls
|
||||
- tcp
|
||||
node_type:
|
||||
- single
|
||||
- sentinel
|
||||
- cluster
|
||||
exclude:
|
||||
- redis_tag: 5
|
||||
connect_type: tls
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: docker-compose up
|
||||
run: |
|
||||
docker-compose \
|
||||
-f .ci/docker-compose-file/docker-compose-redis-${{ matrix.node_type }}-${{ matrix.connect_type }}.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose.yaml \
|
||||
up -d --build
|
||||
- name: setup
|
||||
env:
|
||||
REDIS_TAG: ${{ matrix.redis_tag }}
|
||||
if: matrix.connect_type == 'tls'
|
||||
run: |
|
||||
cat <<-EOF >> "$GITHUB_ENV"
|
||||
EMQX_AUTH__REDIS__SSL=on
|
||||
EMQX_AUTH__REDIS__SSL__CACERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt
|
||||
EMQX_AUTH__REDIS__SSL__CERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt
|
||||
EMQX_AUTH__REDIS__SSL__KEYFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key
|
||||
EMQX_AUTH__REDIS__SSL__VERIFY=true
|
||||
EMQX_AUTH__REDIS__SSL__SERVER_NAME_INDICATION=disable
|
||||
EOF
|
||||
- name: setup
|
||||
env:
|
||||
REDIS_TAG: ${{ matrix.redis_tag }}
|
||||
if: matrix.connect_type == 'tcp'
|
||||
run: |
|
||||
echo EMQX_AUTH__REDIS__SSL=off >> "$GITHUB_ENV"
|
||||
- name: get server address
|
||||
run: |
|
||||
ipv4_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' redis)
|
||||
ipv6_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' redis)
|
||||
cat <<-EOF >> "$GITHUB_ENV"
|
||||
redis_ipv4_address=$ipv4_address
|
||||
redis_ipv6_address=$ipv6_address
|
||||
EOF
|
||||
- name: setup
|
||||
if: matrix.node_type == 'single' && matrix.connect_type == 'tcp'
|
||||
run: |
|
||||
cat <<-EOF >> "$GITHUB_ENV"
|
||||
EMQX_AUTH__REDIS__TYPE=single
|
||||
EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:6379
|
||||
EOF
|
||||
- name: setup
|
||||
if: matrix.node_type == 'single' && matrix.connect_type == 'tls'
|
||||
run: |
|
||||
cat <<-EOF >> "$GITHUB_ENV"
|
||||
EMQX_AUTH__REDIS__TYPE=single
|
||||
EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:6380
|
||||
EOF
|
||||
- name: setup
|
||||
if: matrix.node_type == 'sentinel' && matrix.connect_type == 'tcp'
|
||||
run: |
|
||||
cat <<-EOF >> "$GITHUB_ENV"
|
||||
EMQX_AUTH__REDIS__TYPE=sentinel
|
||||
EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:26379
|
||||
EMQX_AUTH__REDIS__SENTINEL=mymaster
|
||||
EOF
|
||||
- name: setup
|
||||
if: matrix.node_type == 'sentinel' && matrix.connect_type == 'tls'
|
||||
run: |
|
||||
cat <<-EOF >> "$GITHUB_ENV"
|
||||
EMQX_AUTH__REDIS__TYPE=sentinel
|
||||
EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:26380
|
||||
EMQX_AUTH__REDIS__SENTINEL=mymaster
|
||||
EOF
|
||||
- name: setup
|
||||
if: matrix.node_type == 'cluster' && matrix.connect_type == 'tcp'
|
||||
run: |
|
||||
cat <<-EOF >> "$GITHUB_ENV"
|
||||
EMQX_AUTH__REDIS__TYPE=cluster
|
||||
EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:7000
|
||||
EOF
|
||||
- name: setup
|
||||
if: matrix.node_type == 'cluster' && matrix.connect_type == 'tls'
|
||||
run: |
|
||||
cat <<-EOF >> "$GITHUB_ENV"
|
||||
EMQX_AUTH__REDIS__TYPE=cluster
|
||||
EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:8000
|
||||
EOF
|
||||
- name: set git token
|
||||
run: |
|
||||
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org\" > /root/.git-credentials && git config --global credential.helper store"
|
||||
fi
|
||||
- name: run test cases
|
||||
run: |
|
||||
export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
|
||||
export EMQX_AUTH__REIDS__PASSWORD=public
|
||||
printenv > .env
|
||||
docker exec -i erlang sh -c "make ensure-rebar3"
|
||||
docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_redis"
|
||||
docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_redis"
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: logs_redis${{ matrix.redis_tag }}_${{ matrix.node_type }}_${{ matrix.network_type }}_${{ matrix.connect_type }}
|
||||
path: _build/test/logs
|
||||
273
.github/workflows/run_fvt_tests.yaml
vendored
Normal file
273
.github/workflows/run_fvt_tests.yaml
vendored
Normal file
@ -0,0 +1,273 @@
|
||||
name: Functional Verification Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
- e*
|
||||
release:
|
||||
types:
|
||||
- published
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
docker_test:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: prepare
|
||||
run: |
|
||||
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org" > $HOME/.git-credentials
|
||||
git config --global credential.helper store
|
||||
echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
|
||||
make deps-emqx-ee
|
||||
echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
|
||||
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
|
||||
else
|
||||
echo "TARGET=emqx/emqx" >> $GITHUB_ENV
|
||||
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
|
||||
fi
|
||||
- name: make emqx image
|
||||
run: make docker
|
||||
- name: run emqx
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -e -u -x
|
||||
echo "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env
|
||||
echo "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" >> .ci/docker-compose-file/conf.cluster.env
|
||||
echo "EMQX_MQTT__MAX_TOPIC_ALIAS=10" >> .ci/docker-compose-file/conf.cluster.env
|
||||
docker-compose \
|
||||
-f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-python.yaml \
|
||||
up -d
|
||||
while ! docker exec -i node1.emqx.io bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1; do
|
||||
echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx";
|
||||
sleep 5;
|
||||
done
|
||||
- name: verify EMQX_LOADED_PLUGINS override working
|
||||
run: |
|
||||
expected="{emqx_sn, true}."
|
||||
output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1)
|
||||
if [ "$expected" != "$output" ]; then
|
||||
exit 1
|
||||
fi
|
||||
- name: make paho tests
|
||||
run: |
|
||||
if ! docker exec -i python /scripts/pytest.sh; then
|
||||
docker logs node1.emqx.io
|
||||
docker logs node2.emqx.io
|
||||
exit 1
|
||||
fi
|
||||
|
||||
helm_test:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: prepare
|
||||
run: |
|
||||
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org" > $HOME/.git-credentials
|
||||
git config --global credential.helper store
|
||||
echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
|
||||
make deps-emqx-ee
|
||||
echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
|
||||
else
|
||||
echo "TARGET=emqx/emqx" >> $GITHUB_ENV
|
||||
fi
|
||||
- name: make emqx image
|
||||
run: make docker
|
||||
- name: install k3s
|
||||
env:
|
||||
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
|
||||
run: |
|
||||
sudo sh -c "echo \"127.0.0.1 $(hostname)\" >> /etc/hosts"
|
||||
curl -sfL https://get.k3s.io | sh -
|
||||
sudo chmod 644 /etc/rancher/k3s/k3s.yaml
|
||||
kubectl cluster-info
|
||||
- name: install helm
|
||||
env:
|
||||
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
|
||||
run: |
|
||||
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
|
||||
sudo chmod 700 get_helm.sh
|
||||
sudo ./get_helm.sh
|
||||
helm version
|
||||
- name: run emqx on chart
|
||||
env:
|
||||
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
version=$(./pkg-vsn.sh)
|
||||
sudo docker save ${TARGET}:$version -o emqx.tar.gz
|
||||
sudo k3s ctr image import emqx.tar.gz
|
||||
|
||||
sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml
|
||||
sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml
|
||||
|
||||
helm install emqx \
|
||||
--set image.repository=${TARGET} \
|
||||
--set image.pullPolicy=Never \
|
||||
--set emqxAclConfig="" \
|
||||
--set image.pullPolicy=Never \
|
||||
--set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s \
|
||||
--set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \
|
||||
deploy/charts/emqx \
|
||||
--debug
|
||||
|
||||
while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \
|
||||
!= "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
|
||||
echo "==============================";
|
||||
kubectl get pods;
|
||||
echo "==============================";
|
||||
echo "waiting emqx started";
|
||||
sleep 10;
|
||||
done
|
||||
- name: get pods log
|
||||
if: failure()
|
||||
env:
|
||||
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
|
||||
run: kubectl describe pods emqx-0
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
repository: emqx/paho.mqtt.testing
|
||||
ref: develop-4.0
|
||||
path: paho.mqtt.testing
|
||||
- name: install pytest
|
||||
run: |
|
||||
pip install pytest
|
||||
echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
- name: run paho test
|
||||
env:
|
||||
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
|
||||
run: |
|
||||
emqx_svc=$(kubectl get svc --namespace default emqx -o jsonpath="{.spec.clusterIP}")
|
||||
emqx1=$(kubectl get pods emqx-1 -o jsonpath='{.status.podIP}')
|
||||
emqx2=$(kubectl get pods emqx-2 -o jsonpath='{.status.podIP}')
|
||||
|
||||
pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host $emqx_svc
|
||||
RESULT=$?
|
||||
pytest -v paho.mqtt.testing/interoperability/test_cluster --host1 $emqx1 --host2 $emqx2
|
||||
RESULT=$((RESULT + $?))
|
||||
if [ 0 -ne $RESULT ]; then
|
||||
kubectl logs emqx-1
|
||||
kubectl logs emqx-2
|
||||
fi
|
||||
exit $RESULT
|
||||
|
||||
relup_test:
|
||||
runs-on: ubuntu-20.04
|
||||
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
steps:
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.8'
|
||||
architecture: 'x64'
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
repository: emqx/paho.mqtt.testing
|
||||
ref: develop-4.0
|
||||
path: paho.mqtt.testing
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
repository: terry-xiaoyu/one_more_emqx
|
||||
ref: master
|
||||
path: one_more_emqx
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
repository: emqx/emqtt-bench
|
||||
ref: master
|
||||
path: emqtt-bench
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
repository: hawk/lux
|
||||
ref: lux-2.4
|
||||
path: lux
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
repository: ${{ github.repository }}
|
||||
path: emqx
|
||||
fetch-depth: 0
|
||||
- name: prepare
|
||||
run: |
|
||||
if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org" > $HOME/.git-credentials
|
||||
git config --global credential.helper store
|
||||
echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token
|
||||
echo "PROFILE=emqx-ee" >> $GITHUB_ENV
|
||||
else
|
||||
echo "PROFILE=emqx" >> $GITHUB_ENV
|
||||
fi
|
||||
- name: get version
|
||||
run: |
|
||||
set -e -x -u
|
||||
cd emqx
|
||||
if [ $PROFILE = "emqx" ];then
|
||||
broker="emqx-ce"
|
||||
edition='opensource'
|
||||
else
|
||||
broker="emqx-ee"
|
||||
edition='enterprise'
|
||||
fi
|
||||
echo "BROKER=$broker" >> $GITHUB_ENV
|
||||
|
||||
vsn="$(./pkg-vsn.sh)"
|
||||
echo "VSN=$vsn" >> $GITHUB_ENV
|
||||
|
||||
pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
|
||||
if [ $PROFILE = "emqx" ]; then
|
||||
old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
|
||||
else
|
||||
old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
|
||||
fi
|
||||
echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
|
||||
- name: download emqx
|
||||
run: |
|
||||
set -e -x -u
|
||||
mkdir -p emqx/_upgrade_base
|
||||
cd emqx/_upgrade_base
|
||||
old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
|
||||
for old_vsn in ${old_vsns[@]}; do
|
||||
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip
|
||||
done
|
||||
- name: build emqx
|
||||
run: make -C emqx ${PROFILE}-zip
|
||||
- name: build emqtt-bench
|
||||
run: make -C emqtt-bench
|
||||
- name: build lux
|
||||
run: |
|
||||
set -e -u -x
|
||||
cd lux
|
||||
autoconf
|
||||
./configure
|
||||
make
|
||||
make install
|
||||
- name: run relup test
|
||||
run: |
|
||||
set -e -x -u
|
||||
if [ -n "$OLD_VSNS" ]; then
|
||||
mkdir -p packages
|
||||
cp emqx/_packages/emqx/*.zip packages
|
||||
cp emqx/_upgrade_base/*.zip packages
|
||||
lux -v \
|
||||
--timeout 600000 \
|
||||
--var PACKAGE_PATH=$(pwd)/packages \
|
||||
--var BENCH_PATH=$(pwd)/emqtt-bench \
|
||||
--var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
|
||||
--var VSN="$VSN" \
|
||||
--var OLD_VSNS="$OLD_VSNS" \
|
||||
emqx/.ci/fvt_tests/relup.lux
|
||||
fi
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: lux_logs
|
||||
path: lux_logs
|
||||
|
||||
|
||||
|
||||
44
.github/workflows/run_gitlint.yaml
vendored
Normal file
44
.github/workflows/run_gitlint.yaml
vendored
Normal file
@ -0,0 +1,44 @@
|
||||
name: Run gitlint
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
run_gitlint:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout source code
|
||||
uses: actions/checkout@master
|
||||
- name: Install gitlint
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt install gitlint
|
||||
- name: Set auth header
|
||||
if: endsWith(github.repository, 'enterprise')
|
||||
run: |
|
||||
echo 'AUTH_HEADER<<EOF' >> $GITHUB_ENV
|
||||
echo "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" >> $GITHUB_ENV
|
||||
echo 'EOF' >> $GITHUB_ENV
|
||||
- name: Run gitlint
|
||||
shell: bash
|
||||
run: |
|
||||
pr_number=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }')
|
||||
messages="$(curl --silent --show-error \
|
||||
--header "${{ env.AUTH_HEADER }}" \
|
||||
--header "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.hub.fastgit.org/repos/${GITHUB_REPOSITORY}/pulls/${pr_number}/commits")"
|
||||
len=$(echo $messages | jq length)
|
||||
result=true
|
||||
for i in $( seq 0 $(($len - 1)) ); do
|
||||
message=$(echo $messages | jq -r .[$i].commit.message)
|
||||
echo "commit message: $message"
|
||||
status=0
|
||||
echo $message | gitlint -C ./.github/workflows/.gitlint || status=$?
|
||||
if [ $status -ne 0 ]; then
|
||||
result=false
|
||||
fi
|
||||
done
|
||||
if ! ${result} ; then
|
||||
echo "Some of the commit messages are not structured as The Conventional Commits specification. Please check CONTRIBUTING.md for our process on PR."
|
||||
exit 1
|
||||
fi
|
||||
echo "success"
|
||||
154
.github/workflows/run_test_cases.yaml
vendored
Normal file
154
.github/workflows/run_test_cases.yaml
vendored
Normal file
@ -0,0 +1,154 @@
|
||||
name: Run test case
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
- e*
|
||||
release:
|
||||
types:
|
||||
- published
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
run_static_analysis:
|
||||
runs-on: ubuntu-20.04
|
||||
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: set git credentials
|
||||
run: |
|
||||
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org" > $HOME/.git-credentials
|
||||
git config --global credential.helper store
|
||||
fi
|
||||
- name: xref
|
||||
run: make xref
|
||||
- name: dialyzer
|
||||
run: make dialyzer
|
||||
|
||||
run_proper_test:
|
||||
runs-on: ubuntu-20.04
|
||||
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: set git credentials
|
||||
run: |
|
||||
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org" > $HOME/.git-credentials
|
||||
git config --global credential.helper store
|
||||
fi
|
||||
- name: proper
|
||||
run: make proper
|
||||
|
||||
run_common_test:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: set edition
|
||||
id: set_edition
|
||||
run: |
|
||||
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
||||
echo "EDITION=enterprise" >> $GITHUB_ENV
|
||||
else
|
||||
echo "EDITION=opensource" >> $GITHUB_ENV
|
||||
fi
|
||||
- name: docker compose up
|
||||
if: env.EDITION == 'opensource'
|
||||
env:
|
||||
MYSQL_TAG: 8
|
||||
REDIS_TAG: 6
|
||||
MONGO_TAG: 4
|
||||
PGSQL_TAG: 13
|
||||
LDAP_TAG: 2.4.50
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
docker-compose \
|
||||
-f .ci/docker-compose-file/docker-compose.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
|
||||
up -d --build
|
||||
- name: docker compose up
|
||||
if: env.EDITION == 'enterprise'
|
||||
env:
|
||||
MYSQL_TAG: 8
|
||||
REDIS_TAG: 6
|
||||
MONGO_TAG: 4
|
||||
PGSQL_TAG: 13
|
||||
LDAP_TAG: 2.4.50
|
||||
OPENTSDB_TAG: latest
|
||||
INFLUXDB_TAG: 1.7.6
|
||||
DYNAMODB_TAG: 1.11.477
|
||||
TIMESCALE_TAG: latest-pg11
|
||||
CASSANDRA_TAG: 3.11.6
|
||||
RABBITMQ_TAG: 3.7
|
||||
KAFKA_TAG: 2.5.0
|
||||
PULSAR_TAG: 2.3.2
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
docker-compose \
|
||||
-f .ci/docker-compose-file/docker-compose.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-enterprise.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-enterprise-influxdb-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-enterprise-kafka-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-enterprise-opentsdb-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-enterprise-pulsar-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-enterprise-rabbit-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-enterprise-timescale-tcp.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-enterprise-mysql-client.yaml \
|
||||
-f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \
|
||||
up -d --build
|
||||
docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@hub.fastgit.org\" > /root/.git-credentials && git config --global credential.helper store"
|
||||
while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
|
||||
!= $(docker ps -a --filter name=client | wc -l) ]; do
|
||||
sleep 5
|
||||
done
|
||||
- name: run eunit
|
||||
run: |
|
||||
docker exec -i erlang bash -c "make eunit"
|
||||
- name: run common test
|
||||
run: |
|
||||
docker exec -i erlang bash -c "make ct"
|
||||
- name: run cover
|
||||
run: |
|
||||
printenv > .env
|
||||
docker exec -i erlang bash -c "make cover"
|
||||
docker exec --env-file .env -i erlang bash -c "make coveralls"
|
||||
- name: cat rebar.crashdump
|
||||
if: failure()
|
||||
run: if [ -f 'rebar3.crashdump' ];then cat 'rebar3.crashdump' fi
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: logs
|
||||
path: _build/test/logs
|
||||
- uses: actions/upload-artifact@v1
|
||||
with:
|
||||
name: cover
|
||||
path: _build/test/cover
|
||||
|
||||
finish:
|
||||
needs: run_common_test
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Coveralls Finished
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
curl -v -k https://coveralls.io/webhook \
|
||||
--header "Content-Type: application/json" \
|
||||
--data "{\"repo_name\":\"$GITHUB_REPOSITORY\",\"repo_token\":\"$GITHUB_TOKEN\",\"payload\":{\"build_num\":$GITHUB_RUN_ID,\"status\":\"done\"}}" || true
|
||||
18
.github/workflows/shellcheck.yaml
vendored
Normal file
18
.github/workflows/shellcheck.yaml
vendored
Normal file
@ -0,0 +1,18 @@
|
||||
name: Shellcheck
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
shellcheck:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout source code
|
||||
uses: actions/checkout@master
|
||||
- name: Install shellcheck
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt install shellcheck
|
||||
- name: Run shellcheck
|
||||
run: |
|
||||
./scripts/shellcheck.sh
|
||||
echo "success"
|
||||
55
.gitignore
vendored
55
.gitignore
vendored
@ -1,32 +1,49 @@
|
||||
.eunit
|
||||
test-data/
|
||||
deps
|
||||
!deps/.placeholder
|
||||
*.o
|
||||
*.beam
|
||||
*.plt
|
||||
erl_crash.dump
|
||||
rebar3.crashdump
|
||||
ebin
|
||||
_rel/*
|
||||
!ebin/.placeholder
|
||||
.concrete/DEV_MODE
|
||||
.rebar
|
||||
test/ebin/*.beam
|
||||
.exrc
|
||||
plugins/*/ebin
|
||||
*.swp
|
||||
*.so
|
||||
.erlang.mk/
|
||||
etc/plugins/
|
||||
data/configs/*.config
|
||||
data/configs/*.conf
|
||||
data/configs/*.args
|
||||
rel/
|
||||
log/
|
||||
ct/logs/*
|
||||
.DS_Store
|
||||
cover/
|
||||
eunit.coverdata
|
||||
test/ct.cover.spec
|
||||
ct.coverdata
|
||||
.idea/
|
||||
*.d
|
||||
*.iml
|
||||
vars.config
|
||||
erlang.mk
|
||||
|
||||
rebar.lock
|
||||
test/
|
||||
_build
|
||||
.rebar3
|
||||
rebar3.crashdump
|
||||
.DS_Store
|
||||
etc/gen.emqx.conf
|
||||
compile_commands.json
|
||||
cuttlefish
|
||||
xrefr
|
||||
*.coverdata
|
||||
etc/emqx.conf.rendered
|
||||
Mnesia.*/
|
||||
*.DS_Store
|
||||
_checkouts
|
||||
rebar.config.rendered
|
||||
/rebar3
|
||||
rebar.lock
|
||||
.stamp
|
||||
tmp/
|
||||
_packages
|
||||
deploy/docker/tmp
|
||||
.vscode/settings.json
|
||||
elvis
|
||||
emqx_dialyzer_*_plt
|
||||
*/emqx_dashboard/priv/www
|
||||
dist.zip
|
||||
scripts/git-token
|
||||
etc/*.seg
|
||||
_upgrade_base/
|
||||
|
||||
1
.tool-versions
Normal file
1
.tool-versions
Normal file
@ -0,0 +1 @@
|
||||
erlang 23.2.7.2-emqx-1
|
||||
82
CONTRIBUTING.md
Normal file
82
CONTRIBUTING.md
Normal file
@ -0,0 +1,82 @@
|
||||
# Contributing
|
||||
|
||||
You are welcome to submit any bugs, issues and feature requests on this repository.
|
||||
|
||||
|
||||
## Commit Message Guidelines
|
||||
|
||||
We have very precise rules over how our git commit messages can be formatted. This leads to **more readable messages** that are easy to follow when looking through the **project history**.
|
||||
|
||||
### Commit Message Format
|
||||
|
||||
Each commit message consists of a **header**, a **body** and a **footer**. The header has a special format that includes a **type**, a **scope** and a **subject**:
|
||||
|
||||
```
|
||||
<type>(<scope>): <subject>
|
||||
<BLANK LINE>
|
||||
<body>
|
||||
<BLANK LINE>
|
||||
<footer>
|
||||
```
|
||||
|
||||
The **header** with **type** is mandatory. The **scope** of the header is optional. This repository has no predefined scopes. A custom scope can be used for clarity if desired.
|
||||
|
||||
Any line of the commit message cannot be longer 100 characters! This allows the message to be easier to read on GitHub as well as in various git tools.
|
||||
|
||||
The footer should contain a [closing reference to an issue](https://help.hub.fastgit.org/articles/closing-issues-via-commit-messages/) if any.
|
||||
|
||||
Example 1:
|
||||
|
||||
```
|
||||
feat: add Fuji release compose files
|
||||
```
|
||||
|
||||
```
|
||||
fix(script): correct run script to use the right ports
|
||||
|
||||
Previously device services used wrong port numbers. This commit fixes the port numbers to use the latest port numbers.
|
||||
|
||||
Closes: #123, #245, #992
|
||||
```
|
||||
|
||||
### Revert
|
||||
|
||||
If the commit reverts a previous commit, it should begin with `revert: `, followed by the header of the reverted commit. In the body it should say: `This reverts commit <hash>.`, where the hash is the SHA of the commit being reverted.
|
||||
|
||||
### Type
|
||||
|
||||
Must be one of the following:
|
||||
|
||||
- **feat**: New feature for the user, not a new feature for build script
|
||||
- **fix**: Bug fix for the user, not a fix to a build script
|
||||
- **docs**: Documentation only changes
|
||||
- **style**: Formatting, missing semi colons, etc; no production code change
|
||||
- **refactor**: Refactoring production code, eg. renaming a variable
|
||||
- **chore**: Updating grunt tasks etc; no production code change
|
||||
- **perf**: A code change that improves performance
|
||||
- **test**: Adding missing tests, refactoring tests; no production code change
|
||||
- **build**: Changes that affect the CI/CD pipeline or build system or external dependencies (example scopes: travis, jenkins, makefile)
|
||||
- **ci**: Changes provided by DevOps for CI purposes.
|
||||
- **revert**: Reverts a previous commit.
|
||||
|
||||
### Scope
|
||||
|
||||
There are no predefined scopes for this repository. A custom scope can be provided for clarity.
|
||||
|
||||
### Subject
|
||||
|
||||
The subject contains a succinct description of the change:
|
||||
|
||||
- use the imperative, present tense: "change" not "changed" nor "changes"
|
||||
- don't capitalize the first letter
|
||||
- no dot (.) at the end
|
||||
|
||||
### Body
|
||||
|
||||
Just as in the **subject**, use the imperative, present tense: "change" not "changed" nor "changes". The body should include the motivation for the change and contrast this with previous behavior.
|
||||
|
||||
### Footer
|
||||
|
||||
The footer should contain any information about **Breaking Changes** and is also the place to reference GitHub issues that this commit **Closes**.
|
||||
|
||||
**Breaking Changes** should start with the word `BREAKING CHANGE:` with a space or two newlines. The rest of the commit message is then used for this.
|
||||
245
Makefile
245
Makefile
@ -1,32 +1,21 @@
|
||||
## shallow clone for speed
|
||||
|
||||
REBAR_GIT_CLONE_OPTIONS += --depth 1
|
||||
export REBAR_GIT_CLONE_OPTIONS
|
||||
|
||||
TAG = $(shell git tag -l --points-at HEAD)
|
||||
|
||||
CUR_BRANCH := $(shell git branch | grep -e "^*" | cut -d' ' -f 2)
|
||||
|
||||
EMQX_DEPS_DEFAULT_VSN = release-4.2
|
||||
ifeq ($(EMQX_DEPS_DEFAULT_VSN),)
|
||||
ifneq ($(TAG),)
|
||||
EMQX_DEPS_DEFAULT_VSN ?= $(lastword 1, $(TAG))
|
||||
else
|
||||
EMQX_DEPS_DEFAULT_VSN ?= $(CUR_BRANCH)
|
||||
endif
|
||||
$(shell $(CURDIR)/scripts/git-hooks-init.sh)
|
||||
REBAR_VERSION = 3.14.3-emqx-7
|
||||
REBAR = $(CURDIR)/rebar3
|
||||
BUILD = $(CURDIR)/build
|
||||
SCRIPTS = $(CURDIR)/scripts
|
||||
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
|
||||
export EMQX_DESC ?= EMQ X
|
||||
export EMQX_CE_DASHBOARD_VERSION ?= v4.3.1
|
||||
ifeq ($(OS),Windows_NT)
|
||||
export REBAR_COLOR=none
|
||||
endif
|
||||
|
||||
REBAR = $(CURDIR)/rebar3
|
||||
PROFILE ?= emqx
|
||||
REL_PROFILES := emqx emqx-edge
|
||||
PKG_PROFILES := emqx-pkg emqx-edge-pkg
|
||||
PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default
|
||||
|
||||
REBAR_URL = https://s3.amazonaws.com/rebar3/rebar3
|
||||
|
||||
export EMQX_DEPS_DEFAULT_VSN
|
||||
|
||||
PROFILE ?= dgiot
|
||||
PROFILES := dgiot dgiot-edge
|
||||
PKG_PROFILES := dgiot-pkg dgiot-edge-pkg
|
||||
|
||||
CT_APPS := dgiot
|
||||
export REBAR_GIT_CLONE_OPTIONS += --depth=1
|
||||
|
||||
.PHONY: default
|
||||
default: $(REBAR) $(PROFILE)
|
||||
@ -34,114 +23,132 @@ default: $(REBAR) $(PROFILE)
|
||||
.PHONY: all
|
||||
all: $(REBAR) $(PROFILES)
|
||||
|
||||
.PHONY: distclean
|
||||
distclean:
|
||||
@rm -rf _build
|
||||
@rm -f data/app.*.config data/vm.*.args rebar.lock
|
||||
@rm -rf _checkouts
|
||||
.PHONY: ensure-rebar3
|
||||
ensure-rebar3:
|
||||
@$(SCRIPTS)/fail-on-old-otp-version.escript
|
||||
@$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION)
|
||||
|
||||
.PHONY: $(PROFILES)
|
||||
$(PROFILES:%=%): $(REBAR)
|
||||
ifneq ($(OS),Windows_NT)
|
||||
@ln -snf _build/$(@)/lib ./_checkouts
|
||||
endif
|
||||
@if [ $$(echo $(@) |grep edge) ];then export EMQX_DESC="EMQ X Edge";else export EMQX_DESC="EMQ X Broker"; fi;\
|
||||
$(REBAR) as $(@) release
|
||||
$(REBAR): ensure-rebar3
|
||||
|
||||
.PHONY: $(PROFILES:%=build-%)
|
||||
$(PROFILES:%=build-%): $(REBAR)
|
||||
$(REBAR) as $(@:build-%=%) compile
|
||||
.PHONY: get-dashboard
|
||||
get-dashboard:
|
||||
@$(SCRIPTS)/get-dashboard.sh
|
||||
|
||||
.PHONY: deps-all
|
||||
deps-all: $(REBAR) $(PROFILES:%=deps-%) $(PKG_PROFILES:%=deps-%)
|
||||
.PHONY: eunit
|
||||
eunit: $(REBAR)
|
||||
@ENABLE_COVER_COMPILE=1 $(REBAR) eunit -v -c
|
||||
|
||||
.PHONY: $(PROFILES:%=deps-%)
|
||||
$(PROFILES:%=deps-%): $(REBAR)
|
||||
$(REBAR) as $(@:deps-%=%) get-deps
|
||||
.PHONY: proper
|
||||
proper: $(REBAR)
|
||||
@ENABLE_COVER_COMPILE=1 $(REBAR) proper -d test/props -c
|
||||
|
||||
.PHONY: $(PKG_PROFILES:%=deps-%)
|
||||
$(PKG_PROFILES:%=deps-%): $(REBAR)
|
||||
$(REBAR) as $(@:deps-%=%) get-deps
|
||||
.PHONY: ct
|
||||
ct: $(REBAR)
|
||||
@ENABLE_COVER_COMPILE=1 $(REBAR) ct --name 'test@127.0.0.1' -c -v
|
||||
|
||||
.PHONY: run $(PROFILES:%=run-%)
|
||||
run: run-$(PROFILE)
|
||||
$(PROFILES:%=run-%): $(REBAR)
|
||||
ifneq ($(OS),Windows_NT)
|
||||
@ln -snf _build/$(@:run-%=%)/lib ./_checkouts
|
||||
endif
|
||||
$(REBAR) as $(@:run-%=%) run
|
||||
APPS=$(shell $(CURDIR)/scripts/find-apps.sh)
|
||||
|
||||
## app/name-ct targets are intended for local tests hence cover is not enabled
|
||||
.PHONY: $(APPS:%=%-ct)
|
||||
define gen-app-ct-target
|
||||
$1-ct:
|
||||
$(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
|
||||
endef
|
||||
$(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
|
||||
|
||||
## apps/name-prop targets
|
||||
.PHONY: $(APPS:%=%-prop)
|
||||
define gen-app-prop-target
|
||||
$1-prop:
|
||||
$(REBAR) proper -d test/props -v -m $(shell $(CURDIR)/scripts/find-props.sh $1)
|
||||
endef
|
||||
$(foreach app,$(APPS),$(eval $(call gen-app-prop-target,$(app))))
|
||||
|
||||
.PHONY: cover
|
||||
cover: $(REBAR)
|
||||
@ENABLE_COVER_COMPILE=1 $(REBAR) cover
|
||||
|
||||
.PHONY: coveralls
|
||||
coveralls: $(REBAR)
|
||||
@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
|
||||
|
||||
.PHONY: $(REL_PROFILES)
|
||||
$(REL_PROFILES:%=%): $(REBAR) get-dashboard
|
||||
@$(REBAR) as $(@) do compile,release
|
||||
|
||||
## Not calling rebar3 clean because
|
||||
## 1. rebar3 clean relies on rebar3, meaning it reads config, fetches dependencies etc.
|
||||
## 2. it's slow
|
||||
## NOTE: this does not force rebar3 to fetch new version dependencies
|
||||
## make clean-all to delete all fetched dependencies for a fresh start-over
|
||||
.PHONY: clean $(PROFILES:%=clean-%)
|
||||
clean: $(PROFILES:%=clean-%)
|
||||
$(PROFILES:%=clean-%): $(REBAR)
|
||||
@rm -rf _build/$(@:clean-%=%)
|
||||
@rm -rf _build/$(@:clean-%=%)+test
|
||||
$(PROFILES:%=clean-%):
|
||||
@if [ -d _build/$(@:clean-%=%) ]; then \
|
||||
rm -rf _build/$(@:clean-%=%)/rel; \
|
||||
find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
|
||||
fi
|
||||
|
||||
.PHONY: $(PROFILES:%=checkout-%)
|
||||
$(PROFILES:%=checkout-%): $(REBAR) build-$(PROFILE)
|
||||
ln -s -f _build/$(@:checkout-%=%)/lib ./_checkouts
|
||||
.PHONY: clean-all
|
||||
clean-all:
|
||||
@rm -rf _build
|
||||
|
||||
# Checkout current profile
|
||||
.PHONY: checkout
|
||||
checkout:
|
||||
@ln -s -f _build/$(PROFILE)/lib ./_checkouts
|
||||
.PHONY: deps-all
|
||||
deps-all: $(REBAR) $(PROFILES:%=deps-%)
|
||||
|
||||
# Run ct for an app in current profile
|
||||
.PHONY: $(REBAR) $(CT_APPS:%=ct-%)
|
||||
ct: $(CT_APPS:%=ct-%)
|
||||
$(CT_APPS:%=ct-%): checkout-$(PROFILE)
|
||||
-make -C _build/dgiot/lib/$(@:ct-%=%) ct
|
||||
@mkdir -p tests/logs/$(@:ct-%=%)
|
||||
@if [ -d _build/dgiot/lib/$(@:ct-%=%)/_build/test/logs ]; then cp -r _build/dgiot/lib/$(@:ct-%=%)/_build/test/logs/* tests/logs/$(@:ct-%=%); fi
|
||||
## deps-<profile> is used in CI scripts to download deps and the
|
||||
## share downloads between CI steps and/or copied into containers
|
||||
## which may not have the right credentials
|
||||
.PHONY: $(PROFILES:%=deps-%)
|
||||
$(PROFILES:%=deps-%): $(REBAR) get-dashboard
|
||||
@$(REBAR) as $(@:deps-%=%) get-deps
|
||||
|
||||
$(REBAR):
|
||||
ifneq ($(wildcard rebar3),rebar3)
|
||||
@curl -Lo rebar3 $(REBAR_URL) || wget $(REBAR_URL)
|
||||
endif
|
||||
@chmod a+x rebar3
|
||||
.PHONY: xref
|
||||
xref: $(REBAR)
|
||||
@$(REBAR) as check xref
|
||||
|
||||
# Build packages
|
||||
.PHONY: dialyzer
|
||||
dialyzer: $(REBAR)
|
||||
@$(REBAR) as check dialyzer
|
||||
|
||||
COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS)
|
||||
|
||||
## rel target is to create release package without relup
|
||||
.PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel)
|
||||
$(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS)
|
||||
@$(BUILD) $(subst -rel,,$(@)) rel
|
||||
|
||||
## relup target is to create relup instructions
|
||||
.PHONY: $(REL_PROFILES:%=%-relup)
|
||||
define gen-relup-target
|
||||
$1-relup: $(COMMON_DEPS)
|
||||
@$(BUILD) $1 relup
|
||||
endef
|
||||
ALL_ZIPS = $(REL_PROFILES)
|
||||
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-relup-target,$(zt))))
|
||||
|
||||
## zip target is to create a release package .zip with relup
|
||||
.PHONY: $(REL_PROFILES:%=%-zip)
|
||||
define gen-zip-target
|
||||
$1-zip: $1-relup
|
||||
@$(BUILD) $1 zip
|
||||
endef
|
||||
ALL_ZIPS = $(REL_PROFILES)
|
||||
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-zip-target,$(zt))))
|
||||
|
||||
## A pkg target depend on a regular release
|
||||
.PHONY: $(PKG_PROFILES)
|
||||
$(PKG_PROFILES:%=%): $(REBAR)
|
||||
ln -snf _build/$(@)/lib ./_checkouts
|
||||
@if [ $$(echo $(@) |grep edge) ];then export DGIOT_DESC="DGIOT X Edge";else export DGIOT_DESC="DGIOT X Broker"; fi;\
|
||||
$(REBAR) as $(@) release
|
||||
DGIOT_REL=$$(pwd) DGIOT_BUILD=$(@) EMQX_DEPS_DEFAULT_VSN=$(EMQX_DEPS_DEFAULT_VSN) make -C deploy/packages
|
||||
define gen-pkg-target
|
||||
$1: $1-rel
|
||||
@$(BUILD) $1 pkg
|
||||
endef
|
||||
$(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))
|
||||
|
||||
# Build docker image
|
||||
.PHONY: $(PROFILES:%=%-docker-build)
|
||||
$(PROFILES:%=%-docker-build):
|
||||
@if [ ! -z `echo $(@) |grep -oE edge` ]; then \
|
||||
TARGET=dgiot/dgiot-edge EMQX_DEPS_DEFAULT_VSN=$(EMQX_DEPS_DEFAULT_VSN) make -C deploy/docker; \
|
||||
else \
|
||||
TARGET=dgiot/dgiot EMQX_DEPS_DEFAULT_VSN=$(EMQX_DEPS_DEFAULT_VSN) make -C deploy/docker; \
|
||||
fi;
|
||||
.PHONY: run
|
||||
run: $(PROFILE) quickrun
|
||||
|
||||
# Save docker images
|
||||
.PHONY: $(PROFILES:%=%-docker-save)
|
||||
$(PROFILES:%=%-docker-save):
|
||||
@if [ ! -z `echo $(@) |grep -oE edge` ]; then \
|
||||
TARGET=dgiot/dgiot-edge EMQX_DEPS_DEFAULT_VSN=$(EMQX_DEPS_DEFAULT_VSN) make -C deploy/docker save; \
|
||||
else \
|
||||
TARGET=dgiot/dgiot EMQX_DEPS_DEFAULT_VSN=$(EMQX_DEPS_DEFAULT_VSN) make -C deploy/docker save; \
|
||||
fi;
|
||||
.PHONY: quickrun
|
||||
quickrun:
|
||||
./_build/$(PROFILE)/rel/emqx/bin/emqx console
|
||||
|
||||
# Push docker image
|
||||
.PHONY: $(PROFILES:%=%-docker-push)
|
||||
$(PROFILES:%=%-docker-push):
|
||||
@if [ ! -z `echo $(@) |grep -oE edge` ]; then \
|
||||
TARGET=dgiot/dgiot-edge EMQX_DEPS_DEFAULT_VSN=$(EMQX_DEPS_DEFAULT_VSN) make -C deploy/docker push; \
|
||||
TARGET=dgiot/dgiot-edge EMQX_DEPS_DEFAULT_VSN=$(EMQX_DEPS_DEFAULT_VSN) make -C deploy/docker manifest_list; \
|
||||
else \
|
||||
TARGET=dgiot/dgiot EMQX_DEPS_DEFAULT_VSN=$(EMQX_DEPS_DEFAULT_VSN) make -C deploy/docker push; \
|
||||
TARGET=dgiot/dgiot EMQX_DEPS_DEFAULT_VSN=$(EMQX_DEPS_DEFAULT_VSN) make -C deploy/docker manifest_list; \
|
||||
fi;
|
||||
|
||||
# Clean docker image
|
||||
.PHONY: $(PROFILES:%=%-docker-clean)
|
||||
$(PROFILES:%=%-docker-clean):
|
||||
@if [ ! -z `echo $(@) |grep -oE edge` ]; then \
|
||||
TARGET=dgiot/dgiot-edge EMQX_DEPS_DEFAULT_VSN=$(EMQX_DEPS_DEFAULT_VSN) make -C deploy/docker clean; \
|
||||
else \
|
||||
TARGET=dgiot/dgiot EMQX_DEPS_DEFAULT_VSN=$(EMQX_DEPS_DEFAULT_VSN) make -C deploy/docker clean; \
|
||||
fi;
|
||||
include docker.mk
|
||||
|
||||
151
README-CN.md
Normal file
151
README-CN.md
Normal file
@ -0,0 +1,151 @@
|
||||
# EMQ X Broker
|
||||
|
||||
[](https://hub.fastgit.org/emqx/emqx/releases)
|
||||
[](https://travis-ci.org/emqx/emqx)
|
||||
[](https://coveralls.io/github/emqx/emqx)
|
||||
[](https://hub.docker.com/r/emqx/emqx)
|
||||
[](https://slack-invite.emqx.io)
|
||||
[](https://twitter.com/EMQTech)
|
||||
[](https://askemq.com)
|
||||
|
||||
[](https://careers.emqx.cn/)
|
||||
|
||||
[English](./README.md) | 简体中文 | [日本語](./README-JP.md) | [русский](./README-RU.md)
|
||||
|
||||
*EMQ X* 是一款完全开源，高度可伸缩，高可用的分布式 MQTT 消息服务器，适用于 IoT、M2M 和移动应用程序，可处理千万级别的并发客户端。
|
||||
|
||||
从 3.0 版本开始，*EMQ X* 完整支持 MQTT V5.0 协议规范，向下兼容 MQTT V3.1 和 V3.1.1，并支持 MQTT-SN、CoAP、LwM2M、WebSocket 和 STOMP 等通信协议。EMQ X 3.0 单集群可支持千万级别的 MQTT 并发连接。
|
||||
|
||||
- 新功能的完整列表，请参阅 [EMQ X Release Notes](https://hub.fastgit.org/emqx/emqx/releases)。
|
||||
- 获取更多信息，请访问 [EMQ X 官网](https://www.emqx.cn/)。
|
||||
|
||||
## 安装
|
||||
|
||||
*EMQ X* 是跨平台的，支持 Linux、Unix、macOS 以及 Windows。这意味着 *EMQ X* 可以部署在 x86_64 架构的服务器上，也可以部署在 Raspberry Pi 这样的 ARM 设备上。
|
||||
|
||||
Windows 上编译和运行 *EMQ X* 的详情参考：[Windows.md](./Windows.md)
|
||||
|
||||
#### EMQ X Docker 镜像安装
|
||||
|
||||
```
|
||||
docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
|
||||
```
|
||||
|
||||
#### 二进制软件包安装
|
||||
|
||||
需从 [EMQ X 下载](https://www.emqx.cn/downloads) 页面获取相应操作系统的二进制软件包。
|
||||
|
||||
- [单节点安装文档](https://docs.emqx.cn/broker/latest/getting-started/install.html)
|
||||
- [集群配置文档](https://docs.emqx.cn/broker/latest/advanced/cluster.html)
|
||||
|
||||
## 从源码构建
|
||||
|
||||
3.0 版本开始，构建 *EMQ X* 需要 Erlang/OTP R21+。
|
||||
|
||||
4.3 及以后的版本：
|
||||
|
||||
```bash
|
||||
git clone https://hub.fastgit.org/emqx/emqx.git
|
||||
cd emqx
|
||||
make
|
||||
_build/emqx/rel/emqx/bin console
|
||||
```
|
||||
|
||||
对于 4.3 之前的版本，通过另外一个仓库构建：
|
||||
|
||||
```bash
|
||||
git clone https://hub.fastgit.org/emqx/emqx-rel.git
|
||||
cd emqx-rel
|
||||
make
|
||||
_build/emqx/rel/emqx/bin/emqx console
|
||||
```
|
||||
|
||||
## 快速入门
|
||||
|
||||
如果 emqx 从源码编译，`cd _build/emqx/rel/emqx`。
|
||||
如果 emqx 通过 zip 包安装，则切换到 emqx 的根目录。
|
||||
|
||||
```
|
||||
# Start emqx
|
||||
./bin/emqx start
|
||||
|
||||
# Check Status
|
||||
./bin/emqx_ctl status
|
||||
|
||||
# Stop emqx
|
||||
./bin/emqx stop
|
||||
```
|
||||
|
||||
*EMQ X* 启动，可以使用浏览器访问 http://localhost:18083 来查看 Dashboard。
|
||||
|
||||
## 测试
|
||||
|
||||
### 执行所有测试
|
||||
|
||||
```
|
||||
make eunit ct
|
||||
```
|
||||
|
||||
### 执行部分应用的 common tests
|
||||
|
||||
```bash
|
||||
make apps/emqx_bridge_mqtt-ct
|
||||
```
|
||||
|
||||
### 静态分析(Dialyzer)
|
||||
##### 分析所有应用程序
|
||||
```
|
||||
make dialyzer
|
||||
```
|
||||
|
||||
##### 要分析特定的应用程序，（用逗号分隔的应用程序列表）
|
||||
```
|
||||
DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
|
||||
```
|
||||
|
||||
## 社区
|
||||
|
||||
### FAQ
|
||||
|
||||
访问 [EMQ X FAQ](https://docs.emqx.cn/broker/latest/faq/faq.html) 以获取常见问题的帮助。
|
||||
|
||||
### 问答
|
||||
|
||||
[GitHub Discussions](https://hub.fastgit.org/emqx/emqx/discussions)
|
||||
[EMQ 中文问答社区](https://askemq.com)
|
||||
|
||||
### 参与设计
|
||||
|
||||
如果对 EMQ X 有改进建议，可以向[EIP](https://hub.fastgit.org/emqx/eip) 提交 PR 和 ISSUE
|
||||
|
||||
### 插件开发
|
||||
|
||||
如果想集成或开发你自己的插件，参考 [lib-extra/README.md](./lib-extra/README.md)
|
||||
|
||||
|
||||
### 联系我们
|
||||
|
||||
你可通过以下途径与 EMQ 社区及开发者联系:
|
||||
|
||||
- [Slack](https://slack-invite.emqx.io)
|
||||
- [Twitter](https://twitter.com/EMQTech)
|
||||
- [Facebook](https://www.facebook.com/emqxmqtt)
|
||||
- [Reddit](https://www.reddit.com/r/emqx/)
|
||||
- [Weibo](https://weibo.com/emqtt)
|
||||
- [Blog](https://www.emqx.cn/blog)
|
||||
|
||||
欢迎你将任何 bug、问题和功能请求提交到 [emqx/emqx](https://hub.fastgit.org/emqx/emqx/issues)。
|
||||
|
||||
## MQTT 规范
|
||||
|
||||
你可以通过以下链接了解与查阅 MQTT 协议:
|
||||
|
||||
[MQTT Version 3.1.1](https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html)
|
||||
|
||||
[MQTT Version 5.0](https://docs.oasis-open.org/mqtt/mqtt/v5.0/cs02/mqtt-v5.0-cs02.html)
|
||||
|
||||
[MQTT SN](http://mqtt.org/new/wp-content/uploads/2009/06/MQTT-SN_spec_v1.2.pdf)
|
||||
|
||||
## 开源许可
|
||||
|
||||
Apache License 2.0, 详见 [LICENSE](./LICENSE)。
|
||||
131
README-JP.md
Normal file
131
README-JP.md
Normal file
@ -0,0 +1,131 @@
|
||||
# EMQ X Broker
|
||||
|
||||
[](https://hub.fastgit.org/emqx/emqx/releases)
|
||||
[](https://travis-ci.org/emqx/emqx)
|
||||
[](https://coveralls.io/github/emqx/emqx)
|
||||
[](https://hub.docker.com/r/emqx/emqx)
|
||||
[](https://slack-invite.emqx.io)
|
||||
[](https://twitter.com/EMQTech)
|
||||
|
||||
[](https://www.emqx.io/careers)
|
||||
|
||||
[English](./README.md) | [简体中文](./README-CN.md) | 日本語 | [русский](./README-RU.md)
|
||||
|
||||
*EMQ X* は、高い拡張性と可用性をもつ、分散型のMQTTブローカーです。数千万のクライアントを同時に処理するIoT、M2M、モバイルアプリケーション向けです。
|
||||
|
||||
version 3.0 以降、*EMQ X* は MQTT V5.0 の仕様を完全にサポートしており、MQTT V3.1およびV3.1.1とも下位互換性があります。
|
||||
MQTT-SN、CoAP、LwM2M、WebSocket、STOMPなどの通信プロトコルをサポートしています。 MQTTの同時接続数は1つのクラスター上で1,000万以上にまでスケールできます。
|
||||
|
||||
- 新機能の一覧については、[EMQ Xリリースノート](https://hub.fastgit.org/emqx/emqx/releases)を参照してください。
|
||||
- 詳細はこちら[EMQ X公式ウェブサイト](https://www.emqx.io/)をご覧ください。
|
||||
|
||||
## インストール
|
||||
|
||||
*EMQ X* はクロスプラットフォームで、Linux、Unix、macOS、Windowsをサポートしています。
|
||||
そのため、x86_64アーキテクチャサーバー、またはRaspberryPiなどのARMデバイスに *EMQ X* をデプロイすることもできます。
|
||||
|
||||
Windows上における *EMQ X* のビルドと実行については、[Windows.md](./Windows.md)をご参照ください。
|
||||
|
||||
#### Docker イメージによる EMQ X のインストール
|
||||
|
||||
```
|
||||
docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
|
||||
```
|
||||
|
||||
#### バイナリパッケージによるインストール
|
||||
|
||||
それぞれのOSに対応したバイナリソフトウェアパッケージは、[EMQ Xのダウンロード](https://www.emqx.io/downloads)ページから取得できます。
|
||||
|
||||
- [シングルノードインストール](https://docs.emqx.io/broker/latest/en/getting-started/installation.html)
|
||||
- [マルチノードインストール](https://docs.emqx.io/broker/latest/en/advanced/cluster.html)
|
||||
|
||||
## ソースからビルド
|
||||
|
||||
version 3.0 以降の *EMQ X* をビルドするには Erlang/OTP R21+ が必要です。
|
||||
|
||||
version 4.3 以降の場合：
|
||||
|
||||
```bash
|
||||
git clone https://hub.fastgit.org/emqx/emqx-rel.git
|
||||
cd emqx-rel
|
||||
make
|
||||
_build/emqx/rel/emqx/bin/emqx console
|
||||
```
|
||||
|
||||
## クイックスタート
|
||||
|
||||
emqx をソースコードからビルドした場合は、
|
||||
`cd _build/emqx/rel/emqx`でリリースビルドのディレクトリに移動してください。
|
||||
|
||||
リリースパッケージからインストールした場合は、インストール先のルートディレクトリに移動してください。
|
||||
|
||||
```
|
||||
# Start emqx
|
||||
./bin/emqx start
|
||||
|
||||
# Check Status
|
||||
./bin/emqx_ctl status
|
||||
|
||||
# Stop emqx
|
||||
./bin/emqx stop
|
||||
```
|
||||
|
||||
*EMQ X* の起動後、ブラウザで http://localhost:18083 にアクセスするとダッシュボードが表示されます。
|
||||
|
||||
## テスト
|
||||
|
||||
### 全てのテストケースを実行する
|
||||
|
||||
```
|
||||
make eunit ct
|
||||
```
|
||||
|
||||
### common test の一部を実行する
|
||||
|
||||
```bash
|
||||
make apps/emqx_bridge_mqtt-ct
|
||||
```
|
||||
|
||||
### Dialyzer
|
||||
##### アプリケーションの型情報を解析する
|
||||
```
|
||||
make dialyzer
|
||||
```
|
||||
|
||||
##### 特定のアプリケーションのみ解析する（アプリケーション名をコンマ区切りで入力）
|
||||
```
|
||||
DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
|
||||
```
|
||||
|
||||
## コミュニティ
|
||||
|
||||
### FAQ
|
||||
|
||||
よくある質問については、[EMQ X FAQ](https://docs.emqx.io/broker/latest/en/faq/faq.html)をご確認ください。
|
||||
|
||||
### 質問する
|
||||
|
||||
質問や知識共有の場として[GitHub Discussions](https://hub.fastgit.org/emqx/emqx/discussions)を用意しています。
|
||||
|
||||
### 提案
|
||||
|
||||
大規模な改善のご提案がある場合は、[EIP](https://hub.fastgit.org/emqx/eip)にPRをどうぞ。
|
||||
|
||||
### 自作プラグイン
|
||||
|
||||
プラグインを自作することができます。[lib-extra/README.md](./lib-extra/README.md)をご確認ください。
|
||||
|
||||
|
||||
## MQTTの仕様について
|
||||
|
||||
下記のサイトで、MQTTのプロトコルについて学習・確認できます。
|
||||
|
||||
[MQTT Version 3.1.1](https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html)
|
||||
|
||||
[MQTT Version 5.0](https://docs.oasis-open.org/mqtt/mqtt/v5.0/cs02/mqtt-v5.0-cs02.html)
|
||||
|
||||
[MQTT SN](http://mqtt.org/new/wp-content/uploads/2009/06/MQTT-SN_spec_v1.2.pdf)
|
||||
|
||||
## License
|
||||
|
||||
Apache License 2.0, see [LICENSE](https://hub.fastgit.org/emqx/MQTTX/blob/master/LICENSE).
|
||||
141
README-RU.md
Normal file
141
README-RU.md
Normal file
@ -0,0 +1,141 @@
|
||||
# Брокер EMQ X
|
||||
|
||||
[](https://hub.fastgit.org/emqx/emqx/releases)
|
||||
[](https://travis-ci.org/emqx/emqx)
|
||||
[](https://coveralls.io/github/emqx/emqx?branch=master)
|
||||
[](https://hub.docker.com/r/emqx/emqx)
|
||||
[](https://slack-invite.emqx.io)
|
||||
[](https://twitter.com/EMQTech)
|
||||
[](https://hub.fastgit.org/emqx/emqx/discussions)
|
||||
|
||||
[](https://www.emqx.io/careers)
|
||||
|
||||
[English](./README.md) | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | русский
|
||||
|
||||
*EMQ X* — это масштабируемый, высоко доступный, распределённый MQTT брокер с полностью открытым кодом для интернета вещей, межмашинного взаимодействия и мобильных приложений, который поддерживает миллионы одновременных подключений.
|
||||
|
||||
Начиная с релиза 3.0, брокер *EMQ X* полностью поддерживает протокол MQTT версии 5.0, и обратно совместим с версиями 3.1 и 3.1.1, а также протоколами MQTT-SN, CoAP, LwM2M, WebSocket и STOMP. Начиная с релиза 3.0, брокер *EMQ X* может масштабироваться до более чем 10 миллионов одновременных MQTT соединений на один кластер.
|
||||
|
||||
- Полный список возможностей доступен по ссылке: [EMQ X Release Notes](https://hub.fastgit.org/emqx/emqx/releases).
|
||||
- Более подробная информация доступна на нашем сайте: [EMQ X homepage](https://www.emqx.io).
|
||||
|
||||
## Установка
|
||||
|
||||
Брокер *EMQ X* кросплатформенный, и поддерживает Linux, Unix, macOS и Windows. Он может работать на серверах с архитектурой x86_64 и устройствах на архитектуре ARM, таких как Raspberry Pi.
|
||||
|
||||
Более подробная информация о запуске на Windows по ссылке: [Windows.md](./Windows.md)
|
||||
|
||||
#### Установка EMQ X с помощью Docker-образа
|
||||
|
||||
```
|
||||
docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
|
||||
```
|
||||
|
||||
#### Установка бинарного пакета
|
||||
|
||||
Сборки для различных операционных систем: [Загрузить EMQ X](https://www.emqx.io/downloads).
|
||||
|
||||
- [Установка на одном сервере](https://docs.emqx.io/en/broker/latest/getting-started/install.html)
|
||||
- [Установка на кластере](https://docs.emqx.io/en/broker/latest/advanced/cluster.html)
|
||||
|
||||
|
||||
## Сборка из исходного кода
|
||||
|
||||
Начиная с релиза 3.0, для сборки требуется Erlang/OTP R21 или выше.
|
||||
|
||||
Инструкция для сборки версии 4.3 и выше:
|
||||
|
||||
```bash
|
||||
git clone https://hub.fastgit.org/emqx/emqx.git
|
||||
cd emqx
|
||||
make
|
||||
_build/emqx/rel/emqx/bin console
|
||||
```
|
||||
|
||||
Более ранние релизы могут быть собраны с помощью другого репозитория:
|
||||
|
||||
```bash
|
||||
git clone https://hub.fastgit.org/emqx/emqx-rel.git
|
||||
cd emqx-rel
|
||||
make
|
||||
_build/emqx/rel/emqx/bin/emqx console
|
||||
```
|
||||
|
||||
## Первый запуск
|
||||
|
||||
Если emqx был собран из исходников: `cd _build/emqx/rel/emqx`.
|
||||
Или перейдите в директорию, куда emqx был установлен из бинарного пакета.
|
||||
|
||||
```bash
|
||||
# Запуск:
|
||||
./bin/emqx start
|
||||
|
||||
# Проверка статуса:
|
||||
./bin/emqx_ctl status
|
||||
|
||||
# Остановка:
|
||||
./bin/emqx stop
|
||||
```
|
||||
|
||||
Веб-интерфейс брокера будет доступен по ссылке: http://localhost:18083
|
||||
|
||||
## Тесты
|
||||
|
||||
### Полное тестирование
|
||||
|
||||
```
|
||||
make eunit ct
|
||||
```
|
||||
|
||||
### Запуск части тестов
|
||||
|
||||
Пример:
|
||||
|
||||
```bash
|
||||
make apps/emqx_bridge_mqtt-ct
|
||||
```
|
||||
|
||||
### Dialyzer
|
||||
##### Статический анализ всех приложений
|
||||
```
|
||||
make dialyzer
|
||||
```
|
||||
|
||||
##### Статический анализ части приложений (список через запятую)
|
||||
```
|
||||
DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
|
||||
```
|
||||
|
||||
## Сообщество
|
||||
|
||||
### FAQ
|
||||
|
||||
Наиболее частые проблемы разобраны в [EMQ X FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html).
|
||||
|
||||
|
||||
### Вопросы
|
||||
|
||||
Задать вопрос или поделиться идеей можно в [GitHub Discussions](https://hub.fastgit.org/emqx/emqx/discussions).
|
||||
|
||||
### Предложения
|
||||
|
||||
Более масштабные предложения можно присылать в виде pull request в репозиторий [EIP](https://hub.fastgit.org/emqx/eip).
|
||||
|
||||
### Разработка плагинов
|
||||
|
||||
Инструкция по разработке собственных плагинов доступна по ссылке: [lib-extra/README.md](./lib-extra/README.md)
|
||||
|
||||
|
||||
## Спецификации стандарта MQTT
|
||||
|
||||
Следующие ссылки содержат спецификации стандартов:
|
||||
|
||||
[MQTT Version 3.1.1](https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html)
|
||||
|
||||
[MQTT Version 5.0](https://docs.oasis-open.org/mqtt/mqtt/v5.0/cs02/mqtt-v5.0-cs02.html)
|
||||
|
||||
[MQTT SN](http://mqtt.org/new/wp-content/uploads/2009/06/MQTT-SN_spec_v1.2.pdf)
|
||||
|
||||
## Лицензия
|
||||
|
||||
Apache License 2.0, см. [LICENSE](./LICENSE).
|
||||
32
README.md
32
README.md
@ -1,35 +1,35 @@
|
||||
# 数蛙工业互联网持续集成平台
|
||||
|
||||
## 前言
|
||||
|
||||
|
||||
数蛙团队2016年之前，在互联网和移动互联网爬坑多年，2016年开始进入物联网爬坑，希望通过这个开源平台把多年爬坑经验共享出来，让多学科交叉的工业互联网项目变得更简单。
|
||||
|
||||
|
||||
- 让丰富工程人员可以通过视窗交互可以完成需求较简单的工业互联网项目
|
||||
|
||||
|
||||
- 让广大的初级前端工程师通过serverless的方式可以承接需求较复杂的工业互联网项目
|
||||
|
||||
|
||||
- 让Python、Java、Go、C初级后台工程师通过web编程开发通道来承接复杂的工业互联网项目
|
||||
|
||||
|
||||
工业互联网落地过程是IT和OT持续融合的过程，也是技术领域和业务领域持续融合的过程。会者不难，难着不会，要把工业互联网项目变得简单一些，主要还是需要依赖于各个领域的专家围绕业务场景进行信息流的无缝路由，构建一张多学科融合的知识路由表。
|
||||
|
||||
|
||||
基于数蛙工业物联网持续集成平台可以让学者多学科交叉的知识路由表中发布学术论文，技者在实战中练就快速消化项目的本领和收入回报，商者围绕实际项目找到盈利商机。
|
||||
|
||||
|
||||
# 愿景
|
||||
|
||||
数蛙团队希望通过数蛙工业互联网持续集成平台达成下面一些愿景：
|
||||
|
||||
|
||||
+ 通过工程人员、前端工程师、初级后台工程师在不超过1个月的实际完成中小型的工业互联网项目
|
||||
|
||||
|
||||
+ 通过代码开源、软件免费、文档共享、技术认证、产品认证、运维托管等多种方式保证高质量的交付
|
||||
|
||||
|
||||
+ 技术领域专家不断持续集成业界优秀技术框架、业务领域专家不断持续优化业务模型和流程、构建多学科交叉的开放平台
|
||||
|
||||
|
||||
# 产品
|
||||
|
||||
|
||||
+ 边缘侧系列产品
|
||||
|
||||
|
||||
+ 云端系统产品
|
||||
|
||||
|
||||
# 构建
|
||||
|
||||
+ Linux/Unix/Mac构建
|
||||
@ -40,7 +40,7 @@ make
|
||||
+ windows构建
|
||||
```
|
||||
下载msys64
|
||||
|
||||
|
||||
git clone https://hub.fastgit.org/dgiot/dgiot_server.git
|
||||
make
|
||||
``
|
||||
@ -61,7 +61,7 @@ make
|
||||
|
||||
| 联系方式 | 地址 |
|
||||
| -------------- | ----------------------------------------------------------------------------------------- |
|
||||
| github | [https://github.com/dgiot](https://github.com/dgiot?from=git) |
|
||||
| github | [https://hub.fastgit.org/dgiot](https://hub.fastgit.org/dgiot?from=git) |
|
||||
| gitee | [https://gitee.com/dgiot](https://gitee.com/dgiiot?from=git) |
|
||||
| 官网 | [https://www.iotn2n.com](https://www.iotn2n.com?from=git) |
|
||||
| 博客 | [https://tech.iotn2n.com](https://tech.iotn2n.com?from=git) |
|
||||
|
||||
127
Windows.md
Normal file
127
Windows.md
Normal file
@ -0,0 +1,127 @@
|
||||
# Build and run EMQ X on Windows
|
||||
|
||||
NOTE: The instructions and examples are based on Windows 10.
|
||||
|
||||
## Build Environment
|
||||
|
||||
### Visual studio for C/C++ compile and link
|
||||
|
||||
EMQ X includes Erlang NIF (Native Implmented Function) components, implemented
|
||||
in C/C++. To compile and link C/C++ libraries, the easiest way is perhaps to
|
||||
install Visual Studio.
|
||||
|
||||
Visual Studio 2019 is used in our tests.
|
||||
If you are like me (@zmstone), do not know where to start,
|
||||
please follow this OTP guide:
|
||||
https://hub.fastgit.org/erlang/otp/blob/master/HOWTO/INSTALL-WIN32.md
|
||||
|
||||
NOTE: To avoid surprises, you may need to add below two paths to `Path` environment variable
|
||||
and order them before other paths.
|
||||
|
||||
```
|
||||
C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29910\bin\Hostx64\x64
|
||||
C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build
|
||||
```
|
||||
|
||||
Depending on your visual studio version and OS, the paths may differ.
|
||||
The first path is for rebar3 port compiler to find `cl.exe` and `link.exe`
|
||||
The second path is for Powershell or CMD to setup environment variables.
|
||||
|
||||
### Erlang/OTP
|
||||
|
||||
Install Erlang/OTP 23.2 from https://www.erlang.org/downloads
|
||||
You may need to edit the `Path` environment variable to allow running
|
||||
Erlang commands such as `erl` from powershell.
|
||||
|
||||
To validate Erlang installation in CMD or powershell:
|
||||
|
||||
* Start (or restart) CMD or powershell
|
||||
|
||||
* Execute `erl` command to enter Erlang shell
|
||||
|
||||
* Evaluate Erlang expression `halt().` to exit Erlang shell.
|
||||
|
||||
e.g.
|
||||
|
||||
```
|
||||
PS C:\Users\zmsto> erl
|
||||
Eshell V11.1.4 (abort with ^G)
|
||||
1> halt().
|
||||
```
|
||||
|
||||
### bash
|
||||
|
||||
All EMQ X build/run scripts are either in `bash` or `escript`.
|
||||
`escript` is installed as a part of Erlang. To install a `bash`
|
||||
environment in Windows, there are quite a few options.
|
||||
|
||||
Cygwin is what we tested with.
|
||||
|
||||
* Add `cygwin\bin` dir to `Path` environment variable
|
||||
To do so, search for Edit environment variable in control pannel and
|
||||
add `C:\tools\cygwin\bin` (depending on the location where it was installed)
|
||||
to `Path` list.
|
||||
|
||||
* Validate installation.
|
||||
Start (restart) CMD or powershell console and execute `which bash`, it should
|
||||
print out `/usr/bin/bash`
|
||||
|
||||
### Other tools
|
||||
|
||||
Some of the unix world tools are required to build EMQ X. Including:
|
||||
|
||||
* git
|
||||
* curl
|
||||
* make
|
||||
* jq
|
||||
* zip / unzip
|
||||
|
||||
We recommend using [scoop](https://scoop.sh/), or [Chocolatey](https://chocolatey.org/install) to install the tools.
|
||||
|
||||
When using scoop:
|
||||
|
||||
```
|
||||
scoop install git curl make jq zip unzip
|
||||
```
|
||||
|
||||
## Build EMQ X source code
|
||||
|
||||
* Clone the repo: `git clone https://hub.fastgit.org/emqx/emqx.git`
|
||||
|
||||
* Start CMD or Powershell
|
||||
|
||||
* Execute `vcvarsall.bat x86_amd64` to load environment variables
|
||||
|
||||
* Change to emqx directory and execute `make`
|
||||
|
||||
### Possible errors
|
||||
|
||||
* `'cl.exe' is not recognized as an internal or external command`
|
||||
This error is likely due to Visual Studio executables are not set in `Path` environment variable.
|
||||
To fix it, either add path like `C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29910\bin\Hostx64\x64`
|
||||
to `Paht`. Or make sure `vcvarsall.bat x86_amd64` is executed prior to the `make` command
|
||||
|
||||
* `fatal error C1083: Cannot open include file: 'assert.h': No such file or directory`
|
||||
If Visual Studio is installed correctly, this is likely `LIB` and `LIB_PATH` environment
|
||||
variables are not set. Make sure `vcvarsall.bat x86_amd64` is executed prior to the `make` command
|
||||
|
||||
* `link: extra operand 'some.obj'`
|
||||
This is likely due ot the usage of GNU `lnik.exe` but not the one from Visual Studio.
|
||||
Exeucte `link.exe --version` to inspect which one is in use. The one installed from
|
||||
Visual Studio should print out `Microsoft (R) Incremental Linker`.
|
||||
To fix it, Visual Studio's bin paths should be ordered prior to Cygwin's (or similar installation's)
|
||||
bin paths in `Path` environment variable.
|
||||
|
||||
## Run EMQ X
|
||||
|
||||
To start EMQ X broker.
|
||||
|
||||
Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQ X.
|
||||
|
||||
Then execute `_build\emqx\rel\emqx>.\bin\emqx_ctl status` to check status.
|
||||
If everything works fine, it should print out
|
||||
|
||||
```
|
||||
Node 'emqx@127.0.0.1' 4.3-beta.1 is started
|
||||
Application emqx 4.3.0 is running
|
||||
```
|
||||
0
apps/.gitkeep
Normal file
0
apps/.gitkeep
Normal file
21
apps/emqx_bridge_mqtt/.gitignore
vendored
Normal file
21
apps/emqx_bridge_mqtt/.gitignore
vendored
Normal file
@ -0,0 +1,21 @@
|
||||
.eunit
|
||||
deps
|
||||
*.o
|
||||
*.beam
|
||||
*.plt
|
||||
erl_crash.dump
|
||||
ebin/*.beam
|
||||
rel
|
||||
_build
|
||||
.concrete/DEV_MODE
|
||||
.rebar
|
||||
.erlang.mk
|
||||
data
|
||||
ebin
|
||||
emqx_bridge_mqtt.d
|
||||
*.rendered
|
||||
.rebar3/
|
||||
*.coverdata
|
||||
rebar.lock
|
||||
.DS_Store
|
||||
Mnesia.*/
|
||||
265
apps/emqx_bridge_mqtt/README.md
Normal file
265
apps/emqx_bridge_mqtt/README.md
Normal file
@ -0,0 +1,265 @@
|
||||
# EMQ Bridge MQTT
|
||||
|
||||
The concept of **Bridge** means that EMQ X supports forwarding messages
|
||||
of one of its own topics to another MQTT Broker in some way.
|
||||
|
||||
**Bridge** differs from **Cluster** in that the bridge does not
|
||||
replicate the topic trie and routing tables and only forwards MQTT
|
||||
messages based on bridging rules.
|
||||
|
||||
At present, the bridging methods supported by EMQ X are as follows:
|
||||
|
||||
- RPC bridge: RPC Bridge only supports message forwarding and does not
|
||||
support subscribing to the topic of remote nodes to synchronize
|
||||
data;
|
||||
- MQTT Bridge: MQTT Bridge supports both forwarding and data
|
||||
synchronization through subscription topic.
|
||||
|
||||
These concepts are shown below:
|
||||
|
||||
|
||||
|
||||
In addition, the EMQ X message broker supports multi-node bridge mode interconnection
|
||||
|
||||
```
|
||||
--------- --------- ---------
|
||||
Publisher --> | Node1 | --Bridge Forward--> | Node2 | --Bridge Forward--> | Node3 | --> Subscriber
|
||||
--------- --------- ---------
|
||||
```
|
||||
|
||||
In EMQ X, bridge is configured by modifying `etc/emqx.conf`. EMQ X distinguishes between different bridges based on different names. E.g
|
||||
|
||||
```
|
||||
## Bridge address: node name for local bridge, host:port for remote.
|
||||
bridge.mqtt.aws.address = 127.0.0.1:1883
|
||||
```
|
||||
|
||||
This configuration declares a bridge named `aws` and specifies that it is bridged to the MQTT broker of 127.0.0.1:1883 by MQTT mode.
|
||||
|
||||
In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary (such as bridge.$name.address, where $name refers to the name of bridge)
|
||||
|
||||
The next two sections describe how to create a bridge in RPC and MQTT mode respectively and create a forwarding rule that forwards the messages from sensors. Assuming that two EMQ X nodes are running on two hosts:
|
||||
|
||||
|
||||
| Name | Node | MQTT Port |
|
||||
|------|-------------------|-----------|
|
||||
| emqx1| emqx1@192.168.1.1.| 1883 |
|
||||
| emqx2| emqx2@192.168.1.2 | 1883 |
|
||||
|
||||
|
||||
## EMQ X RPC Bridge Configuration
|
||||
|
||||
The following is the basic configuration of RPC bridging. A simplest RPC bridging only requires the following three items
|
||||
|
||||
```
|
||||
## Bridge Address: Use node name (nodename@host) for rpc bridging, and host:port for mqtt connection
|
||||
bridge.mqtt.emqx2.address = emqx2@192.168.1.2
|
||||
|
||||
## Forwarding topics of the message
|
||||
bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/#
|
||||
|
||||
## bridged mountpoint
|
||||
bridge.mqtt.emqx2.mountpoint = bridge/emqx2/${node}/
|
||||
```
|
||||
|
||||
If the messages received by the local node emqx1 matches the topic `sersor1/#` or `sensor2/#`, these messages will be forwarded to the `sensor1/#` or `sensor2/#` topic of the remote node emqx2.
|
||||
|
||||
`forwards` is used to specify topics. Messages of the in `forwards` specified topics on local node are forwarded to the remote node.
|
||||
|
||||
`mountpoint` is used to add a topic prefix when forwarding a message. To use `mountpoint`, the `forwards` directive must be set. In the above example, a message with the topic `sensor1/hello` received by the local node will be forwarded to the remote node with the topic `bridge/emqx2/emqx1@192.168.1.1/sensor1/hello`.
|
||||
|
||||
Limitations of RPC bridging:
|
||||
|
||||
1. The RPC bridge of emqx can only forward local messages to the remote node, and cannot synchronize the messages of the remote node to the local node;
|
||||
|
||||
2. RPC bridge can only bridge two EMQ X broker together and cannot bridge EMQ X broker to other MQTT brokers.
|
||||
|
||||
## EMQ X MQTT Bridge Configuration
|
||||
|
||||
EMQ X 3.0 officially introduced MQTT bridge, so that EMQ X can bridge any MQTT broker. Because of the characteristics of the MQTT protocol, EMQ X can subscribe to the remote mqtt broker's topic through MQTT bridge, and then synchronize the remote MQTT broker's message to the local.
|
||||
|
||||
EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and connect this MQTT client to the remote MQTT broker. Therefore, in the MQTT bridge configuration, following fields may be set for the EMQ X to connect to the remote broker as an mqtt client
|
||||
|
||||
```
|
||||
## Bridge Address: Use node name for rpc bridging, use host:port for mqtt connection
|
||||
bridge.mqtt.emqx2.address = 192.168.1.2:1883
|
||||
|
||||
## Bridged Protocol Version
|
||||
## Enumeration value: mqttv3 | mqttv4 | mqttv5
|
||||
bridge.mqtt.emqx2.proto_ver = mqttv4
|
||||
|
||||
## mqtt client's clientid
|
||||
bridge.mqtt.emqx2.clientid = bridge_emq
|
||||
|
||||
## mqtt client's clean_start field
|
||||
## Note: Some MQTT Brokers need to set the clean_start value as `true`
|
||||
bridge.mqtt.emqx2.clean_start = true
|
||||
|
||||
## mqtt client's username field
|
||||
bridge.mqtt.emqx2.username = user
|
||||
|
||||
## mqtt client's password field
|
||||
bridge.mqtt.emqx2.password = passwd
|
||||
|
||||
## Whether the mqtt client uses ssl to connect to a remote serve or not
|
||||
bridge.mqtt.emqx2.ssl = off
|
||||
|
||||
## CA Certificate of Client SSL Connection (PEM format)
|
||||
bridge.mqtt.emqx2.cacertfile = etc/certs/cacert.pem
|
||||
|
||||
## SSL certificate of Client SSL connection
|
||||
bridge.mqtt.emqx2.certfile = etc/certs/client-cert.pem
|
||||
|
||||
## Key file of Client SSL connection
|
||||
bridge.mqtt.emqx2.keyfile = etc/certs/client-key.pem
|
||||
|
||||
## SSL encryption
|
||||
bridge.mqtt.emqx2.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384
|
||||
|
||||
## TTLS PSK password
|
||||
## Note 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot be configured at the same time
|
||||
##
|
||||
## See 'https://tools.ietf.org/html/rfc4279#section-2'.
|
||||
## bridge.mqtt.emqx2.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA
|
||||
|
||||
## Client's heartbeat interval
|
||||
bridge.mqtt.emqx2.keepalive = 60s
|
||||
|
||||
## Supported TLS version
|
||||
bridge.mqtt.emqx2.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1
|
||||
|
||||
## Forwarding topics of the message
|
||||
bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/#
|
||||
|
||||
## Bridged mountpoint
|
||||
bridge.mqtt.emqx2.mountpoint = bridge/emqx2/${node}/
|
||||
|
||||
## Subscription topic for bridging
|
||||
bridge.mqtt.emqx2.subscription.1.topic = cmd/topic1
|
||||
|
||||
## Subscription qos for bridging
|
||||
bridge.mqtt.emqx2.subscription.1.qos = 1
|
||||
|
||||
## Subscription topic for bridging
|
||||
bridge.mqtt.emqx2.subscription.2.topic = cmd/topic2
|
||||
|
||||
## Subscription qos for bridging
|
||||
bridge.mqtt.emqx2.subscription.2.qos = 1
|
||||
|
||||
## Bridging reconnection interval
|
||||
## Default: 30s
|
||||
bridge.mqtt.emqx2.reconnect_interval = 30s
|
||||
|
||||
## QoS1 message retransmission interval
|
||||
bridge.mqtt.emqx2.retry_interval = 20s
|
||||
|
||||
## Inflight Size.
|
||||
bridge.mqtt.emqx2.max_inflight_batches = 32
|
||||
```
|
||||
|
||||
## Bridge Cache Configuration
|
||||
|
||||
The bridge of EMQ X has a message caching mechanism. The caching mechanism is applicable to both RPC bridging and MQTT bridging. When the bridge is disconnected (such as when the network connection is unstable), the messages with a topic specified in `forwards` can be cached to the local message queue. Until the bridge is restored, these messages are re-forwarded to the remote node. The configuration of the cache queue is as follows
|
||||
|
||||
```
|
||||
## emqx_bridge internal number of messages used for batch
|
||||
bridge.mqtt.emqx2.queue.batch_count_limit = 32
|
||||
|
||||
## emqx_bridge internal number of message bytes used for batch
|
||||
bridge.mqtt.emqx2.queue.batch_bytes_limit = 1000MB
|
||||
|
||||
## The path for placing replayq queue. If it is not specified, then replayq will run in `mem-only` mode and messages will not be cached on disk.
|
||||
bridge.mqtt.emqx2.queue.replayq_dir = data/emqx_emqx2_bridge/
|
||||
|
||||
## Replayq data segment size
|
||||
bridge.mqtt.emqx2.queue.replayq_seg_bytes = 10MB
|
||||
```
|
||||
|
||||
`bridge.mqtt.emqx2.queue.replayq_dir` is a configuration parameter for specifying the path of the bridge storage queue.
|
||||
|
||||
`bridge.mqtt.emqx2.queue.replayq_seg_bytes` is used to specify the size of the largest single file of the message queue that is cached on disk. If the message queue size exceeds the specified value, a new file is created to store the message queue.
|
||||
|
||||
## CLI for EMQ X Bridge MQTT
|
||||
|
||||
CLI for EMQ X Bridge MQTT:
|
||||
|
||||
``` bash
|
||||
$ cd emqx1/ && ./bin/emqx_ctl bridges
|
||||
bridges list # List bridges
|
||||
bridges start <Name> # Start a bridge
|
||||
bridges stop <Name> # Stop a bridge
|
||||
bridges forwards <Name> # Show a bridge forward topic
|
||||
bridges add-forward <Name> <Topic> # Add bridge forward topic
|
||||
bridges del-forward <Name> <Topic> # Delete bridge forward topic
|
||||
bridges subscriptions <Name> # Show a bridge subscriptions topic
|
||||
bridges add-subscription <Name> <Topic> <Qos> # Add bridge subscriptions topic
|
||||
```
|
||||
|
||||
List all bridge states
|
||||
|
||||
``` bash
|
||||
$ ./bin/emqx_ctl bridges list
|
||||
name: emqx status: Stopped $ ./bin/emqx_ctl bridges list
|
||||
name: emqx status: Stopped
|
||||
```
|
||||
|
||||
Start the specified bridge
|
||||
|
||||
``` bash
|
||||
$ ./bin/emqx_ctl bridges start emqx
|
||||
Start bridge successfully.
|
||||
```
|
||||
|
||||
Stop the specified bridge
|
||||
|
||||
``` bash
|
||||
$ ./bin/emqx_ctl bridges stop emqx
|
||||
Stop bridge successfully.
|
||||
```
|
||||
List the forwarding topics for the specified bridge
|
||||
|
||||
``` bash
|
||||
$ ./bin/emqx_ctl bridges forwards emqx
|
||||
topic: topic1/#
|
||||
topic: topic2/#
|
||||
```
|
||||
|
||||
Add a forwarding topic for the specified bridge
|
||||
|
||||
``` bash
|
||||
$ ./bin/emqx_ctl bridges add-forwards emqx topic3/#
|
||||
Add-forward topic successfully.
|
||||
```
|
||||
|
||||
Delete the forwarding topic for the specified bridge
|
||||
|
||||
|
||||
``` bash
|
||||
$ ./bin/emqx_ctl bridges del-forwards emqx topic3/#
|
||||
Del-forward topic successfully.
|
||||
```
|
||||
|
||||
List subscriptions for the specified bridge
|
||||
|
||||
``` bash
|
||||
$ ./bin/emqx_ctl bridges subscriptions emqx
|
||||
topic: cmd/topic1, qos: 1
|
||||
topic: cmd/topic2, qos: 1
|
||||
```
|
||||
|
||||
Add a subscription topic for the specified bridge
|
||||
|
||||
``` bash
|
||||
$ ./bin/emqx_ctl bridges add-subscription emqx cmd/topic3 1
|
||||
Add-subscription topic successfully.
|
||||
```
|
||||
|
||||
Delete the subscription topic for the specified bridge
|
||||
|
||||
``` bash
|
||||
$ ./bin/emqx_ctl bridges del-subscription emqx cmd/topic3
|
||||
Del-subscription topic successfully.
|
||||
```
|
||||
|
||||
Note: In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary.
|
||||
|
||||
286
apps/emqx_bridge_mqtt/docs/guide.rst
Normal file
286
apps/emqx_bridge_mqtt/docs/guide.rst
Normal file
@ -0,0 +1,286 @@
|
||||
|
||||
EMQ Bridge MQTT
|
||||
===============
|
||||
|
||||
The concept of **Bridge** means that EMQ X supports forwarding messages
|
||||
of one of its own topics to another MQTT Broker in some way.
|
||||
|
||||
**Bridge** differs from **Cluster** in that the bridge does not
|
||||
replicate the topic trie and routing tables and only forwards MQTT
|
||||
messages based on bridging rules.
|
||||
|
||||
At present, the bridging methods supported by EMQ X are as follows:
|
||||
|
||||
|
||||
* RPC bridge: RPC Bridge only supports message forwarding and does not
|
||||
support subscribing to the topic of remote nodes to synchronize
|
||||
data;
|
||||
* MQTT Bridge: MQTT Bridge supports both forwarding and data
|
||||
synchronization through subscription topic.
|
||||
|
||||
These concepts are shown below:
|
||||
|
||||
|
||||
.. image:: images/bridge.png
|
||||
:target: images/bridge.png
|
||||
:alt: bridge
|
||||
|
||||
|
||||
In addition, the EMQ X message broker supports multi-node bridge mode interconnection
|
||||
|
||||
.. code-block::
|
||||
|
||||
--------- --------- ---------
|
||||
Publisher --> | Node1 | --Bridge Forward--> | Node2 | --Bridge Forward--> | Node3 | --> Subscriber
|
||||
--------- --------- ---------
|
||||
|
||||
In EMQ X, bridge is configured by modifying ``etc/emqx.conf``. EMQ X distinguishes between different bridges based on different names. E.g
|
||||
|
||||
.. code-block::
|
||||
|
||||
## Bridge address: node name for local bridge, host:port for remote.
|
||||
bridge.mqtt.aws.address = 127.0.0.1:1883
|
||||
|
||||
This configuration declares a bridge named ``aws`` and specifies that it is bridged to the MQTT broker of 127.0.0.1:1883 by MQTT mode.
|
||||
|
||||
In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary (such as bridge.$name.address, where $name refers to the name of bridge)
|
||||
|
||||
The next two sections describe how to create a bridge in RPC and MQTT mode respectively and create a forwarding rule that forwards the messages from sensors. Assuming that two EMQ X nodes are running on two hosts:
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
|
||||
* - Name
|
||||
- Node
|
||||
- MQTT Port
|
||||
* - emqx1
|
||||
- emqx1@192.168.1.1.
|
||||
- 1883
|
||||
* - emqx2
|
||||
- emqx2@192.168.1.2
|
||||
- 1883
|
||||
|
||||
|
||||
EMQ X RPC Bridge Configuration
|
||||
------------------------------
|
||||
|
||||
The following is the basic configuration of RPC bridging. A simplest RPC bridging only requires the following three items
|
||||
|
||||
.. code-block::
|
||||
|
||||
## Bridge Address: Use node name (nodename@host) for rpc bridging, and host:port for mqtt connection
|
||||
bridge.mqtt.emqx2.address = emqx2@192.168.1.2
|
||||
|
||||
## Forwarding topics of the message
|
||||
bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/#
|
||||
|
||||
## bridged mountpoint
|
||||
bridge.mqtt.emqx2.mountpoint = bridge/emqx2/${node}/
|
||||
|
||||
If the messages received by the local node emqx1 matches the topic ``sersor1/#`` or ``sensor2/#``\ , these messages will be forwarded to the ``sensor1/#`` or ``sensor2/#`` topic of the remote node emqx2.
|
||||
|
||||
``forwards`` is used to specify topics. Messages of the in ``forwards`` specified topics on local node are forwarded to the remote node.
|
||||
|
||||
``mountpoint`` is used to add a topic prefix when forwarding a message. To use ``mountpoint``\ , the ``forwards`` directive must be set. In the above example, a message with the topic ``sensor1/hello`` received by the local node will be forwarded to the remote node with the topic ``bridge/emqx2/emqx1@192.168.1.1/sensor1/hello``.
|
||||
|
||||
Limitations of RPC bridging:
|
||||
|
||||
|
||||
#.
|
||||
The RPC bridge of emqx can only forward local messages to the remote node, and cannot synchronize the messages of the remote node to the local node;
|
||||
|
||||
#.
|
||||
RPC bridge can only bridge two EMQ X broker together and cannot bridge EMQ X broker to other MQTT brokers.
|
||||
|
||||
EMQ X MQTT Bridge Configuration
|
||||
-------------------------------
|
||||
|
||||
EMQ X 3.0 officially introduced MQTT bridge, so that EMQ X can bridge any MQTT broker. Because of the characteristics of the MQTT protocol, EMQ X can subscribe to the remote mqtt broker's topic through MQTT bridge, and then synchronize the remote MQTT broker's message to the local.
|
||||
|
||||
EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and connect this MQTT client to the remote MQTT broker. Therefore, in the MQTT bridge configuration, following fields may be set for the EMQ X to connect to the remote broker as an mqtt client
|
||||
|
||||
.. code-block::
|
||||
|
||||
## Bridge Address: Use node name for rpc bridging, use host:port for mqtt connection
|
||||
bridge.mqtt.emqx2.address = 192.168.1.2:1883
|
||||
|
||||
## Bridged Protocol Version
|
||||
## Enumeration value: mqttv3 | mqttv4 | mqttv5
|
||||
bridge.mqtt.emqx2.proto_ver = mqttv4
|
||||
|
||||
## mqtt client's clientid
|
||||
bridge.mqtt.emqx2.clientid = bridge_emq
|
||||
|
||||
## mqtt client's clean_start field
|
||||
## Note: Some MQTT Brokers need to set the clean_start value as `true`
|
||||
bridge.mqtt.emqx2.clean_start = true
|
||||
|
||||
## mqtt client's username field
|
||||
bridge.mqtt.emqx2.username = user
|
||||
|
||||
## mqtt client's password field
|
||||
bridge.mqtt.emqx2.password = passwd
|
||||
|
||||
## Whether the mqtt client uses ssl to connect to a remote serve or not
|
||||
bridge.mqtt.emqx2.ssl = off
|
||||
|
||||
## CA Certificate of Client SSL Connection (PEM format)
|
||||
bridge.mqtt.emqx2.cacertfile = etc/certs/cacert.pem
|
||||
|
||||
## SSL certificate of Client SSL connection
|
||||
bridge.mqtt.emqx2.certfile = etc/certs/client-cert.pem
|
||||
|
||||
## Key file of Client SSL connection
|
||||
bridge.mqtt.emqx2.keyfile = etc/certs/client-key.pem
|
||||
|
||||
## TTLS PSK password
|
||||
## Note 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot be configured at the same time
|
||||
##
|
||||
## See 'https://tools.ietf.org/html/rfc4279#section-2'.
|
||||
## bridge.mqtt.emqx2.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA
|
||||
|
||||
## Client's heartbeat interval
|
||||
bridge.mqtt.emqx2.keepalive = 60s
|
||||
|
||||
## Supported TLS version
|
||||
bridge.mqtt.emqx2.tls_versions = tlsv1.2
|
||||
|
||||
## SSL encryption
|
||||
bridge.mqtt.emqx2.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384
|
||||
|
||||
## Forwarding topics of the message
|
||||
bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/#
|
||||
|
||||
## Bridged mountpoint
|
||||
bridge.mqtt.emqx2.mountpoint = bridge/emqx2/${node}/
|
||||
|
||||
## Subscription topic for bridging
|
||||
bridge.mqtt.emqx2.subscription.1.topic = cmd/topic1
|
||||
|
||||
## Subscription qos for bridging
|
||||
bridge.mqtt.emqx2.subscription.1.qos = 1
|
||||
|
||||
## Subscription topic for bridging
|
||||
bridge.mqtt.emqx2.subscription.2.topic = cmd/topic2
|
||||
|
||||
## Subscription qos for bridging
|
||||
bridge.mqtt.emqx2.subscription.2.qos = 1
|
||||
|
||||
## Bridging reconnection interval
|
||||
## Default: 30s
|
||||
bridge.mqtt.emqx2.reconnect_interval = 30s
|
||||
|
||||
## QoS1 message retransmission interval
|
||||
bridge.mqtt.emqx2.retry_interval = 20s
|
||||
|
||||
## Inflight Size.
|
||||
bridge.mqtt.emqx2.max_inflight_batches = 32
|
||||
|
||||
Bridge Cache Configuration
|
||||
--------------------------
|
||||
|
||||
The bridge of EMQ X has a message caching mechanism. The caching mechanism is applicable to both RPC bridging and MQTT bridging. When the bridge is disconnected (such as when the network connection is unstable), the messages with a topic specified in ``forwards`` can be cached to the local message queue. Until the bridge is restored, these messages are re-forwarded to the remote node. The configuration of the cache queue is as follows
|
||||
|
||||
.. code-block::
|
||||
|
||||
## emqx_bridge internal number of messages used for batch
|
||||
bridge.mqtt.emqx2.queue.batch_count_limit = 32
|
||||
|
||||
## emqx_bridge internal number of message bytes used for batch
|
||||
bridge.mqtt.emqx2.queue.batch_bytes_limit = 1000MB
|
||||
|
||||
## The path for placing replayq queue. If it is not specified, then replayq will run in `mem-only` mode and messages will not be cached on disk.
|
||||
bridge.mqtt.emqx2.queue.replayq_dir = data/emqx_emqx2_bridge/
|
||||
|
||||
## Replayq data segment size
|
||||
bridge.mqtt.emqx2.queue.replayq_seg_bytes = 10MB
|
||||
|
||||
``bridge.mqtt.emqx2.queue.replayq_dir`` is a configuration parameter for specifying the path of the bridge storage queue.
|
||||
|
||||
``bridge.mqtt.emqx2.queue.replayq_seg_bytes`` is used to specify the size of the largest single file of the message queue that is cached on disk. If the message queue size exceeds the specified value, a new file is created to store the message queue.
|
||||
|
||||
CLI for EMQ X Bridge MQTT
|
||||
-------------------------
|
||||
|
||||
CLI for EMQ X Bridge MQTT:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ cd emqx1/ && ./bin/emqx_ctl bridges
|
||||
bridges list # List bridges
|
||||
bridges start <Name> # Start a bridge
|
||||
bridges stop <Name> # Stop a bridge
|
||||
bridges forwards <Name> # Show a bridge forward topic
|
||||
bridges add-forward <Name> <Topic> # Add bridge forward topic
|
||||
bridges del-forward <Name> <Topic> # Delete bridge forward topic
|
||||
bridges subscriptions <Name> # Show a bridge subscriptions topic
|
||||
bridges add-subscription <Name> <Topic> <Qos> # Add bridge subscriptions topic
|
||||
|
||||
List all bridge states
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ ./bin/emqx_ctl bridges list
|
||||
name: emqx status: Stopped $ ./bin/emqx_ctl bridges list
|
||||
name: emqx status: Stopped
|
||||
|
||||
Start the specified bridge
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ ./bin/emqx_ctl bridges start emqx
|
||||
Start bridge successfully.
|
||||
|
||||
Stop the specified bridge
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ ./bin/emqx_ctl bridges stop emqx
|
||||
Stop bridge successfully.
|
||||
|
||||
List the forwarding topics for the specified bridge
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ ./bin/emqx_ctl bridges forwards emqx
|
||||
topic: topic1/#
|
||||
topic: topic2/#
|
||||
|
||||
Add a forwarding topic for the specified bridge
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ ./bin/emqx_ctl bridges add-forwards emqx topic3/#
|
||||
Add-forward topic successfully.
|
||||
|
||||
Delete the forwarding topic for the specified bridge
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ ./bin/emqx_ctl bridges del-forwards emqx topic3/#
|
||||
Del-forward topic successfully.
|
||||
|
||||
List subscriptions for the specified bridge
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ ./bin/emqx_ctl bridges subscriptions emqx
|
||||
topic: cmd/topic1, qos: 1
|
||||
topic: cmd/topic2, qos: 1
|
||||
|
||||
Add a subscription topic for the specified bridge
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ ./bin/emqx_ctl bridges add-subscription emqx cmd/topic3 1
|
||||
Add-subscription topic successfully.
|
||||
|
||||
Delete the subscription topic for the specified bridge
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ ./bin/emqx_ctl bridges del-subscription emqx cmd/topic3
|
||||
Del-subscription topic successfully.
|
||||
|
||||
Note: In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary.
|
||||
|
||||
BIN
apps/emqx_bridge_mqtt/docs/images/bridge.png
Normal file
BIN
apps/emqx_bridge_mqtt/docs/images/bridge.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 144 KiB |
174
apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf
Normal file
174
apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf
Normal file
@ -0,0 +1,174 @@
|
||||
##====================================================================
|
||||
## Configuration for EMQ X MQTT Broker Bridge
|
||||
##====================================================================
|
||||
|
||||
##--------------------------------------------------------------------
|
||||
## Bridges to aws
|
||||
##--------------------------------------------------------------------
|
||||
|
||||
## Bridge address: node name for local bridge, host:port for remote.
|
||||
##
|
||||
## Value: String
|
||||
## Example: emqx@127.0.0.1, 127.0.0.1:1883
|
||||
bridge.mqtt.aws.address = 127.0.0.1:1883
|
||||
|
||||
## Protocol version of the bridge.
|
||||
##
|
||||
## Value: Enum
|
||||
## - mqttv5
|
||||
## - mqttv4
|
||||
## - mqttv3
|
||||
bridge.mqtt.aws.proto_ver = mqttv4
|
||||
|
||||
## Start type of the bridge.
|
||||
##
|
||||
## Value: enum
|
||||
## manual
|
||||
## auto
|
||||
bridge.mqtt.aws.start_type = manual
|
||||
|
||||
## Whether to enable bridge mode for mqtt bridge
|
||||
##
|
||||
## This option is prepared for the mqtt broker which does not
|
||||
## support bridge_mode such as the mqtt-plugin of the rabbitmq
|
||||
##
|
||||
## Value: boolean
|
||||
#bridge.mqtt.aws.bridge_mode = false
|
||||
|
||||
## The ClientId of a remote bridge.
|
||||
##
|
||||
## Placeholders:
|
||||
## ${node}: Node name
|
||||
##
|
||||
## Value: String
|
||||
bridge.mqtt.aws.clientid = bridge_aws
|
||||
|
||||
## The Clean start flag of a remote bridge.
|
||||
##
|
||||
## Value: boolean
|
||||
## Default: true
|
||||
##
|
||||
## NOTE: Some IoT platforms require clean_start
|
||||
## must be set to 'true'
|
||||
bridge.mqtt.aws.clean_start = true
|
||||
|
||||
## The username for a remote bridge.
|
||||
##
|
||||
## Value: String
|
||||
bridge.mqtt.aws.username = user
|
||||
|
||||
## The password for a remote bridge.
|
||||
##
|
||||
## Value: String
|
||||
bridge.mqtt.aws.password = passwd
|
||||
|
||||
## Topics that need to be forward to AWS IoTHUB
|
||||
##
|
||||
## Value: String
|
||||
## Example: topic1/#,topic2/#
|
||||
bridge.mqtt.aws.forwards = topic1/#,topic2/#
|
||||
|
||||
## Forward messages to the mountpoint of an AWS IoTHUB
|
||||
##
|
||||
## Value: String
|
||||
bridge.mqtt.aws.forward_mountpoint = bridge/aws/${node}/
|
||||
|
||||
## Need to subscribe to AWS topics
|
||||
##
|
||||
## Value: String
|
||||
## bridge.mqtt.aws.subscription.1.topic = cmd/topic1
|
||||
|
||||
## Need to subscribe to AWS topics QoS.
|
||||
##
|
||||
## Value: Number
|
||||
## bridge.mqtt.aws.subscription.1.qos = 1
|
||||
|
||||
## A mountpoint that receives messages from AWS IoTHUB
|
||||
##
|
||||
## Value: String
|
||||
## bridge.mqtt.aws.receive_mountpoint = receive/aws/
|
||||
|
||||
|
||||
## Bribge to remote server via SSL.
|
||||
##
|
||||
## Value: on | off
|
||||
bridge.mqtt.aws.ssl = off
|
||||
|
||||
## PEM-encoded CA certificates of the bridge.
|
||||
##
|
||||
## Value: File
|
||||
bridge.mqtt.aws.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem
|
||||
|
||||
## Client SSL Certfile of the bridge.
|
||||
##
|
||||
## Value: File
|
||||
bridge.mqtt.aws.certfile = {{ platform_etc_dir }}/certs/client-cert.pem
|
||||
|
||||
## Client SSL Keyfile of the bridge.
|
||||
##
|
||||
## Value: File
|
||||
bridge.mqtt.aws.keyfile = {{ platform_etc_dir }}/certs/client-key.pem
|
||||
|
||||
## SSL Ciphers used by the bridge.
|
||||
##
|
||||
## Value: String
|
||||
bridge.mqtt.aws.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA
|
||||
|
||||
## Ciphers for TLS PSK.
|
||||
## Note that 'bridge.${BridgeName}.ciphers' and 'bridge.${BridgeName}.psk_ciphers' cannot
|
||||
## be configured at the same time.
|
||||
## See 'https://tools.ietf.org/html/rfc4279#section-2'.
|
||||
#bridge.mqtt.aws.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA
|
||||
|
||||
## Ping interval of a down bridge.
|
||||
##
|
||||
## Value: Duration
|
||||
## Default: 10 seconds
|
||||
bridge.mqtt.aws.keepalive = 60s
|
||||
|
||||
## TLS versions used by the bridge.
|
||||
##
|
||||
## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier
|
||||
## Value: String
|
||||
bridge.mqtt.aws.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1
|
||||
|
||||
## Bridge reconnect time.
|
||||
##
|
||||
## Value: Duration
|
||||
## Default: 30 seconds
|
||||
bridge.mqtt.aws.reconnect_interval = 30s
|
||||
|
||||
## Retry interval for bridge QoS1 message delivering.
|
||||
##
|
||||
## Value: Duration
|
||||
bridge.mqtt.aws.retry_interval = 20s
|
||||
|
||||
## Publish messages in batches, only RPC Bridge supports
|
||||
##
|
||||
## Value: Integer
|
||||
## default: 32
|
||||
bridge.mqtt.aws.batch_size = 32
|
||||
|
||||
## Inflight size.
|
||||
## 0 means infinity (no limit on the inflight window)
|
||||
##
|
||||
## Value: Integer
|
||||
bridge.mqtt.aws.max_inflight_size = 32
|
||||
|
||||
## Base directory for replayq to store messages on disk
|
||||
## If this config entry is missing or set to undefined,
|
||||
## replayq works in a mem-only manner.
|
||||
##
|
||||
## Value: String
|
||||
bridge.mqtt.aws.queue.replayq_dir = {{ platform_data_dir }}/replayq/emqx_aws_bridge/
|
||||
|
||||
## Replayq segment size
|
||||
##
|
||||
## Value: Bytesize
|
||||
bridge.mqtt.aws.queue.replayq_seg_bytes = 10MB
|
||||
|
||||
## Replayq max total size
|
||||
##
|
||||
## Value: Bytesize
|
||||
bridge.mqtt.aws.queue.max_total_size = 5GB
|
||||
|
||||
18
apps/emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl
Normal file
18
apps/emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl
Normal file
@ -0,0 +1,18 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-define(APP, emqx_bridge_mqtt).
|
||||
|
||||
244
apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema
Normal file
244
apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema
Normal file
@ -0,0 +1,244 @@
|
||||
%%-*- mode: erlang -*-
|
||||
%%--------------------------------------------------------------------
|
||||
%% Bridges
|
||||
%%--------------------------------------------------------------------
|
||||
{mapping, "bridge.mqtt.$name.address", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.proto_ver", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, {enum, [mqttv3, mqttv4, mqttv5]}}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.bridge_mode", "emqx_bridge_mqtt.bridges", [
|
||||
{default, false},
|
||||
{datatype, {enum, [true, false]}}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.start_type", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, {enum, [manual, auto]}},
|
||||
{default, auto}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.clientid", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.clean_start", "emqx_bridge_mqtt.bridges", [
|
||||
{default, true},
|
||||
{datatype, {enum, [true, false]}}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.username", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.password", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.forwards", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string},
|
||||
{default, ""}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.forward_mountpoint", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.subscription.$id.topic", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.subscription.$id.qos", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, integer}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.receive_mountpoint", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.ssl", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, flag},
|
||||
{default, off}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.cacertfile", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.certfile", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.keyfile", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.ciphers", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.psk_ciphers", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.keepalive", "emqx_bridge_mqtt.bridges", [
|
||||
{default, "10s"},
|
||||
{datatype, {duration, s}}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.tls_versions", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string},
|
||||
{default, "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1"}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.reconnect_interval", "emqx_bridge_mqtt.bridges", [
|
||||
{default, "30s"},
|
||||
{datatype, {duration, ms}}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.retry_interval", "emqx_bridge_mqtt.bridges", [
|
||||
{default, "20s"},
|
||||
{datatype, {duration, s}}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.max_inflight_size", "emqx_bridge_mqtt.bridges", [
|
||||
{default, 0},
|
||||
{datatype, integer}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.batch_size", "emqx_bridge_mqtt.bridges", [
|
||||
{default, 0},
|
||||
{datatype, integer}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.queue.replayq_dir", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.queue.replayq_seg_bytes", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, bytesize}
|
||||
]}.
|
||||
|
||||
{mapping, "bridge.mqtt.$name.queue.max_total_size", "emqx_bridge_mqtt.bridges", [
|
||||
{datatype, bytesize}
|
||||
]}.
|
||||
|
||||
{translation, "emqx_bridge_mqtt.bridges", fun(Conf) ->
|
||||
|
||||
MapPSKCiphers = fun(PSKCiphers) ->
|
||||
lists:map(
|
||||
fun("PSK-AES128-CBC-SHA") -> {psk, aes_128_cbc, sha};
|
||||
("PSK-AES256-CBC-SHA") -> {psk, aes_256_cbc, sha};
|
||||
("PSK-3DES-EDE-CBC-SHA") -> {psk, '3des_ede_cbc', sha};
|
||||
("PSK-RC4-SHA") -> {psk, rc4_128, sha}
|
||||
end, PSKCiphers)
|
||||
end,
|
||||
|
||||
Split = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end,
|
||||
|
||||
IsSsl = fun(cacertfile) -> true;
|
||||
(certfile) -> true;
|
||||
(keyfile) -> true;
|
||||
(ciphers) -> true;
|
||||
(psk_ciphers) -> true;
|
||||
(tls_versions) -> true;
|
||||
(_Opt) -> false
|
||||
end,
|
||||
|
||||
Parse = fun(tls_versions, Vers) ->
|
||||
[{versions, [list_to_atom(S) || S <- Split(Vers)]}];
|
||||
(ciphers, Ciphers) ->
|
||||
[{ciphers, Split(Ciphers)}];
|
||||
(psk_ciphers, Ciphers) ->
|
||||
[{ciphers, MapPSKCiphers(Split(Ciphers))}, {user_lookup_fun, {fun emqx_psk:lookup/3, <<>>}}];
|
||||
(Opt, Val) ->
|
||||
[{Opt, Val}]
|
||||
end,
|
||||
|
||||
Merge = fun(forwards, Val, Opts) ->
|
||||
[{forwards, string:tokens(Val, ",")}|Opts];
|
||||
(Opt, Val, Opts) ->
|
||||
case IsSsl(Opt) of
|
||||
true ->
|
||||
SslOpts = Parse(Opt, Val) ++ proplists:get_value(ssl_opts, Opts, []),
|
||||
lists:ukeymerge(1, [{ssl_opts, SslOpts}], lists:usort(Opts));
|
||||
false ->
|
||||
[{Opt, Val}|Opts]
|
||||
end
|
||||
end,
|
||||
Queue = fun(Name) ->
|
||||
Configs = cuttlefish_variable:filter_by_prefix("bridge.mqtt." ++ Name ++ ".queue", Conf),
|
||||
|
||||
QOpts = [{list_to_atom(QOpt), QValue}|| {[_, _, _, "queue", QOpt], QValue} <- Configs],
|
||||
maps:from_list(QOpts)
|
||||
end,
|
||||
Subscriptions = fun(Name) ->
|
||||
Configs = cuttlefish_variable:filter_by_prefix("bridge.mqtt." ++ Name ++ ".subscription", Conf),
|
||||
lists:zip([Topic || {_, Topic} <- lists:sort([{I, Topic} || {[_, _, _, "subscription", I, "topic"], Topic} <- Configs])],
|
||||
[QoS || {_, QoS} <- lists:sort([{I, QoS} || {[_, _, _, "subscription", I, "qos"], QoS} <- Configs])])
|
||||
end,
|
||||
IsNodeAddr = fun(Addr) ->
|
||||
case string:tokens(Addr, "@") of
|
||||
[_NodeName, _Hostname] -> true;
|
||||
_ -> false
|
||||
end
|
||||
end,
|
||||
ConnMod = fun(Name) ->
|
||||
|
||||
[AddrConfig] = cuttlefish_variable:filter_by_prefix("bridge.mqtt." ++ Name ++ ".address", Conf),
|
||||
{_, Addr} = AddrConfig,
|
||||
|
||||
Subs = Subscriptions(Name),
|
||||
case IsNodeAddr(Addr) of
|
||||
true when Subs =/= [] ->
|
||||
error({"subscriptions are not supported when bridging between emqx nodes", Name, Subs});
|
||||
true ->
|
||||
emqx_bridge_rpc;
|
||||
false ->
|
||||
emqx_bridge_mqtt
|
||||
end
|
||||
end,
|
||||
|
||||
%% to be backward compatible
|
||||
Translate =
|
||||
fun Tr(queue, Q, Cfg) ->
|
||||
NewQ = maps:fold(Tr, #{}, Q),
|
||||
Cfg#{queue => NewQ};
|
||||
Tr(address, Addr0, Cfg) ->
|
||||
Addr = case IsNodeAddr(Addr0) of
|
||||
true -> list_to_atom(Addr0);
|
||||
false -> Addr0
|
||||
end,
|
||||
Cfg#{address => Addr};
|
||||
Tr(reconnect_interval, Ms, Cfg) ->
|
||||
Cfg#{reconnect_delay_ms => Ms};
|
||||
Tr(proto_ver, Ver, Cfg) ->
|
||||
Cfg#{proto_ver =>
|
||||
case Ver of
|
||||
mqttv3 -> v3;
|
||||
mqttv4 -> v4;
|
||||
mqttv5 -> v5;
|
||||
_ -> v4
|
||||
end};
|
||||
Tr(max_inflight_size, Size, Cfg) ->
|
||||
Cfg#{max_inflight => Size};
|
||||
Tr(Key, Value, Cfg) ->
|
||||
Cfg#{Key => Value}
|
||||
end,
|
||||
C = lists:foldl(
|
||||
fun({["bridge", "mqtt", Name, Opt], Val}, Acc) ->
|
||||
%% e.g #{aws => [{OptKey, OptVal}]}
|
||||
Init = [{list_to_atom(Opt), Val},
|
||||
{connect_module, ConnMod(Name)},
|
||||
{subscriptions, Subscriptions(Name)},
|
||||
{queue, Queue(Name)}],
|
||||
maps:update_with(list_to_atom(Name), fun(Opts) -> Merge(list_to_atom(Opt), Val, Opts) end, Init, Acc);
|
||||
(_, Acc) -> Acc
|
||||
end, #{}, lists:usort(cuttlefish_variable:filter_by_prefix("bridge.mqtt", Conf))),
|
||||
C1 = maps:map(fun(Bn, Bc) ->
|
||||
maps:to_list(maps:fold(Translate, #{}, maps:from_list(Bc)))
|
||||
end, C),
|
||||
maps:to_list(C1)
|
||||
end}.
|
||||
19
apps/emqx_bridge_mqtt/rebar.config
Normal file
19
apps/emqx_bridge_mqtt/rebar.config
Normal file
@ -0,0 +1,19 @@
|
||||
{deps, []}.
|
||||
{edoc_opts, [{preprocess, true}]}.
|
||||
{erl_opts, [warn_unused_vars,
|
||||
warn_shadow_vars,
|
||||
warn_unused_import,
|
||||
warn_obsolete_guard,
|
||||
debug_info]}.
|
||||
|
||||
{xref_checks, [undefined_function_calls, undefined_functions,
|
||||
locals_not_used, deprecated_function_calls,
|
||||
warnings_as_errors, deprecated_functions]}.
|
||||
{cover_enabled, true}.
|
||||
{cover_opts, [verbose]}.
|
||||
{cover_export_enabled, true}.
|
||||
|
||||
{shell, [
|
||||
% {config, "config/sys.config"},
|
||||
{apps, [emqx, emqx_bridge_mqtt]}
|
||||
]}.
|
||||
74
apps/emqx_bridge_mqtt/src/emqx_bridge_connect.erl
Normal file
74
apps/emqx_bridge_mqtt/src/emqx_bridge_connect.erl
Normal file
@ -0,0 +1,74 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-module(emqx_bridge_connect).
|
||||
|
||||
-export([start/2]).
|
||||
|
||||
-export_type([config/0, connection/0]).
|
||||
|
||||
-optional_callbacks([ensure_subscribed/3, ensure_unsubscribed/2]).
|
||||
|
||||
%% map fields depend on implementation
|
||||
-type(config() :: map()).
|
||||
-type(connection() :: term()).
|
||||
-type(batch() :: emqx_protal:batch()).
|
||||
-type(ack_ref() :: emqx_bridge_worker:ack_ref()).
|
||||
-type(topic() :: emqx_topic:topic()).
|
||||
-type(qos() :: emqx_mqtt_types:qos()).
|
||||
|
||||
-include_lib("emqx/include/logger.hrl").
|
||||
|
||||
-logger_header("[Bridge Connect]").
|
||||
|
||||
%% establish the connection to remote node/cluster
|
||||
%% protal worker (the caller process) should be expecting
|
||||
%% a message {disconnected, conn_ref()} when disconnected.
|
||||
-callback start(config()) -> {ok, connection()} | {error, any()}.
|
||||
|
||||
%% send to remote node/cluster
|
||||
%% bridge worker (the caller process) should be expecting
|
||||
%% a message {batch_ack, reference()} when batch is acknowledged by remote node/cluster
|
||||
-callback send(connection(), batch()) -> {ok, ack_ref()} | {ok, integer()} | {error, any()}.
|
||||
|
||||
%% called when owner is shutting down.
|
||||
-callback stop(connection()) -> ok.
|
||||
|
||||
-callback ensure_subscribed(connection(), topic(), qos()) -> ok.
|
||||
|
||||
-callback ensure_unsubscribed(connection(), topic()) -> ok.
|
||||
|
||||
start(Module, Config) ->
|
||||
case Module:start(Config) of
|
||||
{ok, Conn} ->
|
||||
{ok, Conn};
|
||||
{error, Reason} ->
|
||||
Config1 = obfuscate(Config),
|
||||
?LOG(error, "Failed to connect with module=~p\n"
|
||||
"config=~p\nreason:~p", [Module, Config1, Reason]),
|
||||
{error, Reason}
|
||||
end.
|
||||
|
||||
obfuscate(Map) ->
|
||||
maps:fold(fun(K, V, Acc) ->
|
||||
case is_sensitive(K) of
|
||||
true -> [{K, '***'} | Acc];
|
||||
false -> [{K, V} | Acc]
|
||||
end
|
||||
end, [], Map).
|
||||
|
||||
is_sensitive(password) -> true;
|
||||
is_sensitive(_) -> false.
|
||||
14
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src
Normal file
14
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src
Normal file
@ -0,0 +1,14 @@
|
||||
{application, emqx_bridge_mqtt,
|
||||
[{description, "EMQ X Bridge to MQTT Broker"},
|
||||
{vsn, "4.3.0"}, % strict semver, bump manually!
|
||||
{modules, []},
|
||||
{registered, []},
|
||||
{applications, [kernel,stdlib,replayq,emqtt]},
|
||||
{mod, {emqx_bridge_mqtt_app, []}},
|
||||
{env, []},
|
||||
{licenses, ["Apache-2.0"]},
|
||||
{maintainers, ["EMQ X Team <contact@emqx.io>"]},
|
||||
{links, [{"Homepage", "https://emqx.io/"},
|
||||
{"Github", "https://hub.fastgit.org/emqx/emqx-bridge-mqtt"}
|
||||
]}
|
||||
]}.
|
||||
10
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src
Normal file
10
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src
Normal file
@ -0,0 +1,10 @@
|
||||
%% -*-: erlang -*-
|
||||
|
||||
{VSN,
|
||||
[
|
||||
{<<".*">>, []}
|
||||
],
|
||||
[
|
||||
{<<".*">>, []}
|
||||
]
|
||||
}.
|
||||
204
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl
Normal file
204
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl
Normal file
@ -0,0 +1,204 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
%% @doc This module implements EMQX Bridge transport layer on top of MQTT protocol
|
||||
|
||||
-module(emqx_bridge_mqtt).
|
||||
|
||||
-behaviour(emqx_bridge_connect).
|
||||
|
||||
%% behaviour callbacks
|
||||
-export([ start/1
|
||||
, send/2
|
||||
, stop/1
|
||||
]).
|
||||
|
||||
%% optional behaviour callbacks
|
||||
-export([ ensure_subscribed/3
|
||||
, ensure_unsubscribed/2
|
||||
]).
|
||||
|
||||
%% callbacks for emqtt
|
||||
-export([ handle_puback/2
|
||||
, handle_publish/2
|
||||
, handle_disconnected/2
|
||||
]).
|
||||
|
||||
-include_lib("emqx/include/logger.hrl").
|
||||
-include_lib("emqx/include/emqx_mqtt.hrl").
|
||||
|
||||
-define(ACK_REF(ClientPid, PktId), {ClientPid, PktId}).
|
||||
|
||||
%% Messages towards ack collector process
|
||||
-define(REF_IDS(Ref, Ids), {Ref, Ids}).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% emqx_bridge_connect callbacks
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
start(Config = #{address := Address}) ->
|
||||
Parent = self(),
|
||||
Mountpoint = maps:get(receive_mountpoint, Config, undefined),
|
||||
Handlers = make_hdlr(Parent, Mountpoint),
|
||||
{Host, Port} = case string:tokens(Address, ":") of
|
||||
[H] -> {H, 1883};
|
||||
[H, P] -> {H, list_to_integer(P)}
|
||||
end,
|
||||
ClientConfig = Config#{msg_handler => Handlers,
|
||||
host => Host,
|
||||
port => Port,
|
||||
force_ping => true
|
||||
},
|
||||
case emqtt:start_link(replvar(ClientConfig)) of
|
||||
{ok, Pid} ->
|
||||
case emqtt:connect(Pid) of
|
||||
{ok, _} ->
|
||||
try
|
||||
subscribe_remote_topics(Pid, maps:get(subscriptions, Config, [])),
|
||||
{ok, #{client_pid => Pid}}
|
||||
catch
|
||||
throw : Reason ->
|
||||
ok = stop(#{client_pid => Pid}),
|
||||
{error, Reason}
|
||||
end;
|
||||
{error, Reason} ->
|
||||
ok = stop(#{client_pid => Pid}),
|
||||
{error, Reason}
|
||||
end;
|
||||
{error, Reason} ->
|
||||
{error, Reason}
|
||||
end.
|
||||
|
||||
stop(#{client_pid := Pid}) ->
|
||||
safe_stop(Pid, fun() -> emqtt:stop(Pid) end, 1000),
|
||||
ok.
|
||||
|
||||
ensure_subscribed(#{client_pid := Pid}, Topic, QoS) when is_pid(Pid) ->
|
||||
case emqtt:subscribe(Pid, Topic, QoS) of
|
||||
{ok, _, _} -> ok;
|
||||
Error -> Error
|
||||
end;
|
||||
ensure_subscribed(_Conn, _Topic, _QoS) ->
|
||||
%% return ok for now
|
||||
%% next re-connect should should call start with new topic added to config
|
||||
ok.
|
||||
|
||||
ensure_unsubscribed(#{client_pid := Pid}, Topic) when is_pid(Pid) ->
|
||||
case emqtt:unsubscribe(Pid, Topic) of
|
||||
{ok, _, _} -> ok;
|
||||
Error -> Error
|
||||
end;
|
||||
ensure_unsubscribed(_, _) ->
|
||||
%% return ok for now
|
||||
%% next re-connect should should call start with this topic deleted from config
|
||||
ok.
|
||||
|
||||
safe_stop(Pid, StopF, Timeout) ->
|
||||
MRef = monitor(process, Pid),
|
||||
unlink(Pid),
|
||||
try
|
||||
StopF()
|
||||
catch
|
||||
_ : _ ->
|
||||
ok
|
||||
end,
|
||||
receive
|
||||
{'DOWN', MRef, _, _, _} ->
|
||||
ok
|
||||
after
|
||||
Timeout ->
|
||||
exit(Pid, kill)
|
||||
end.
|
||||
|
||||
send(Conn, Msgs) ->
|
||||
send(Conn, Msgs, []).
|
||||
|
||||
send(_Conn, [], []) ->
|
||||
%% all messages in the batch are QoS-0
|
||||
Ref = make_ref(),
|
||||
%% QoS-0 messages do not have packet ID
|
||||
%% the batch ack is simulated with a loop-back message
|
||||
self() ! {batch_ack, Ref},
|
||||
{ok, Ref};
|
||||
send(_Conn, [], PktIds) ->
|
||||
%% PktIds is not an empty list if there is any non-QoS-0 message in the batch,
|
||||
%% And the worker should wait for all acks
|
||||
{ok, PktIds};
|
||||
send(#{client_pid := ClientPid} = Conn, [Msg | Rest], PktIds) ->
|
||||
case emqtt:publish(ClientPid, Msg) of
|
||||
ok ->
|
||||
send(Conn, Rest, PktIds);
|
||||
{ok, PktId} ->
|
||||
send(Conn, Rest, [PktId | PktIds]);
|
||||
{error, Reason} ->
|
||||
%% NOTE: There is no partial sucess of a batch and recover from the middle
|
||||
%% only to retry all messages in one batch
|
||||
{error, Reason}
|
||||
end.
|
||||
|
||||
handle_puback(#{packet_id := PktId, reason_code := RC}, Parent)
|
||||
when RC =:= ?RC_SUCCESS;
|
||||
RC =:= ?RC_NO_MATCHING_SUBSCRIBERS ->
|
||||
Parent ! {batch_ack, PktId}, ok;
|
||||
handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) ->
|
||||
?LOG(warning, "Publish ~p to remote node falied, reason_code: ~p", [PktId, RC]).
|
||||
|
||||
handle_publish(Msg, Mountpoint) ->
|
||||
emqx_broker:publish(emqx_bridge_msg:to_broker_msg(Msg, Mountpoint)).
|
||||
|
||||
handle_disconnected(Reason, Parent) ->
|
||||
Parent ! {disconnected, self(), Reason}.
|
||||
|
||||
make_hdlr(Parent, Mountpoint) ->
|
||||
#{puback => {fun ?MODULE:handle_puback/2, [Parent]},
|
||||
publish => {fun ?MODULE:handle_publish/2, [Mountpoint]},
|
||||
disconnected => {fun ?MODULE:handle_disconnected/2, [Parent]}
|
||||
}.
|
||||
|
||||
subscribe_remote_topics(ClientPid, Subscriptions) ->
|
||||
lists:foreach(fun({Topic, Qos}) ->
|
||||
case emqtt:subscribe(ClientPid, Topic, Qos) of
|
||||
{ok, _, _} -> ok;
|
||||
Error -> throw(Error)
|
||||
end
|
||||
end, Subscriptions).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% Internal funcs
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
replvar(Options) ->
|
||||
replvar([clientid, max_inflight], Options).
|
||||
|
||||
replvar([], Options) ->
|
||||
Options;
|
||||
replvar([Key|More], Options) ->
|
||||
case maps:get(Key, Options, undefined) of
|
||||
undefined ->
|
||||
replvar(More, Options);
|
||||
Val ->
|
||||
replvar(More, maps:put(Key, feedvar(Key, Val, Options), Options))
|
||||
end.
|
||||
|
||||
%% ${node} => node()
|
||||
feedvar(clientid, ClientId, _) ->
|
||||
iolist_to_binary(re:replace(ClientId, "\\${node}", atom_to_list(node())));
|
||||
|
||||
feedvar(max_inflight, 0, _) ->
|
||||
infinity;
|
||||
|
||||
feedvar(max_inflight, Size, _) ->
|
||||
Size.
|
||||
578
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl
Normal file
578
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl
Normal file
@ -0,0 +1,578 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
%% @doc This module implements EMQX Bridge transport layer on top of MQTT protocol
|
||||
|
||||
-module(emqx_bridge_mqtt_actions).
|
||||
|
||||
-include_lib("emqx/include/emqx.hrl").
|
||||
-include_lib("emqx/include/logger.hrl").
|
||||
-include_lib("emqx_rule_engine/include/rule_actions.hrl").
|
||||
|
||||
-import(emqx_rule_utils, [str/1]).
|
||||
|
||||
-export([ on_resource_create/2
|
||||
, on_get_resource_status/2
|
||||
, on_resource_destroy/2
|
||||
]).
|
||||
|
||||
%% Callbacks of ecpool Worker
|
||||
-export([connect/1]).
|
||||
|
||||
-export([subscriptions/1]).
|
||||
|
||||
-export([ on_action_create_data_to_mqtt_broker/2
|
||||
, on_action_data_to_mqtt_broker/2
|
||||
]).
|
||||
|
||||
-define(RESOURCE_TYPE_MQTT, 'bridge_mqtt').
|
||||
-define(RESOURCE_TYPE_RPC, 'bridge_rpc').
|
||||
|
||||
-define(RESOURCE_CONFIG_SPEC_MQTT, #{
|
||||
address => #{
|
||||
order => 1,
|
||||
type => string,
|
||||
required => true,
|
||||
default => <<"127.0.0.1:1883">>,
|
||||
title => #{en => <<" Broker Address">>,
|
||||
zh => <<"远程 broker 地址"/utf8>>},
|
||||
description => #{en => <<"The MQTT Remote Address">>,
|
||||
zh => <<"远程 MQTT Broker 的地址"/utf8>>}
|
||||
},
|
||||
pool_size => #{
|
||||
order => 2,
|
||||
type => number,
|
||||
required => true,
|
||||
default => 8,
|
||||
title => #{en => <<"Pool Size">>,
|
||||
zh => <<"连接池大小"/utf8>>},
|
||||
description => #{en => <<"MQTT Connection Pool Size">>,
|
||||
zh => <<"连接池大小"/utf8>>}
|
||||
},
|
||||
clientid => #{
|
||||
order => 3,
|
||||
type => string,
|
||||
required => true,
|
||||
default => <<"client">>,
|
||||
title => #{en => <<"ClientId">>,
|
||||
zh => <<"客户端 Id"/utf8>>},
|
||||
description => #{en => <<"ClientId for connecting to remote MQTT broker">>,
|
||||
zh => <<"连接远程 Broker 的 ClientId"/utf8>>}
|
||||
},
|
||||
append => #{
|
||||
order => 4,
|
||||
type => boolean,
|
||||
required => false,
|
||||
default => true,
|
||||
title => #{en => <<"Append GUID">>,
|
||||
zh => <<"附加 GUID"/utf8>>},
|
||||
description => #{en => <<"Append GUID to MQTT ClientId?">>,
|
||||
zh => <<"是否将GUID附加到 MQTT ClientId 后"/utf8>>}
|
||||
},
|
||||
username => #{
|
||||
order => 5,
|
||||
type => string,
|
||||
required => false,
|
||||
default => <<"">>,
|
||||
title => #{en => <<"Username">>, zh => <<"用户名"/utf8>>},
|
||||
description => #{en => <<"Username for connecting to remote MQTT Broker">>,
|
||||
zh => <<"连接远程 Broker 的用户名"/utf8>>}
|
||||
},
|
||||
password => #{
|
||||
order => 6,
|
||||
type => password,
|
||||
required => false,
|
||||
default => <<"">>,
|
||||
title => #{en => <<"Password">>,
|
||||
zh => <<"密码"/utf8>>},
|
||||
description => #{en => <<"Password for connecting to remote MQTT Broker">>,
|
||||
zh => <<"连接远程 Broker 的密码"/utf8>>}
|
||||
},
|
||||
mountpoint => #{
|
||||
order => 7,
|
||||
type => string,
|
||||
required => false,
|
||||
default => <<"bridge/aws/${node}/">>,
|
||||
title => #{en => <<"Bridge MountPoint">>,
|
||||
zh => <<"桥接挂载点"/utf8>>},
|
||||
description => #{
|
||||
en => <<"MountPoint for bridge topic:<br/>"
|
||||
"Example: The topic of messages sent to `topic1` on local node "
|
||||
"will be transformed to `bridge/aws/${node}/topic1`">>,
|
||||
zh => <<"桥接主题的挂载点:<br/>"
|
||||
"示例: 本地节点向 `topic1` 发消息，远程桥接节点的主题"
|
||||
"会变换为 `bridge/aws/${node}/topic1`"/utf8>>
|
||||
}
|
||||
},
|
||||
disk_cache => #{
|
||||
order => 8,
|
||||
type => string,
|
||||
required => false,
|
||||
default => <<"off">>,
|
||||
enum => [<<"on">>, <<"off">>],
|
||||
title => #{en => <<"Disk Cache">>,
|
||||
zh => <<"磁盘缓存"/utf8>>},
|
||||
description => #{en => <<"The flag which determines whether messages "
|
||||
"can be cached on local disk when bridge is "
|
||||
"disconnected">>,
|
||||
zh => <<"当桥接断开时用于控制是否将消息缓存到本地磁"
|
||||
"盘队列上"/utf8>>}
|
||||
},
|
||||
proto_ver => #{
|
||||
order => 9,
|
||||
type => string,
|
||||
required => false,
|
||||
default => <<"mqttv4">>,
|
||||
enum => [<<"mqttv3">>, <<"mqttv4">>, <<"mqttv5">>],
|
||||
title => #{en => <<"Protocol Version">>,
|
||||
zh => <<"协议版本"/utf8>>},
|
||||
description => #{en => <<"MQTTT Protocol version">>,
|
||||
zh => <<"MQTT 协议版本"/utf8>>}
|
||||
},
|
||||
keepalive => #{
|
||||
order => 10,
|
||||
type => string,
|
||||
required => false,
|
||||
default => <<"60s">> ,
|
||||
title => #{en => <<"Keepalive">>,
|
||||
zh => <<"心跳间隔"/utf8>>},
|
||||
description => #{en => <<"Keepalive">>,
|
||||
zh => <<"心跳间隔"/utf8>>}
|
||||
},
|
||||
reconnect_interval => #{
|
||||
order => 11,
|
||||
type => string,
|
||||
required => false,
|
||||
default => <<"30s">>,
|
||||
title => #{en => <<"Reconnect Interval">>,
|
||||
zh => <<"重连间隔"/utf8>>},
|
||||
description => #{en => <<"Reconnect interval of bridge:<br/>">>,
|
||||
zh => <<"重连间隔"/utf8>>}
|
||||
},
|
||||
retry_interval => #{
|
||||
order => 12,
|
||||
type => string,
|
||||
required => false,
|
||||
default => <<"20s">>,
|
||||
title => #{en => <<"Retry interval">>,
|
||||
zh => <<"重传间隔"/utf8>>},
|
||||
description => #{en => <<"Retry interval for bridge QoS1 message delivering">>,
|
||||
zh => <<"消息重传间隔"/utf8>>}
|
||||
},
|
||||
bridge_mode => #{
|
||||
order => 13,
|
||||
type => boolean,
|
||||
required => false,
|
||||
default => false,
|
||||
title => #{en => <<"Bridge Mode">>,
|
||||
zh => <<"桥接模式"/utf8>>},
|
||||
description => #{en => <<"Bridge mode for MQTT bridge connection">>,
|
||||
zh => <<"MQTT 连接是否为桥接模式"/utf8>>}
|
||||
},
|
||||
ssl => #{
|
||||
order => 14,
|
||||
type => boolean,
|
||||
default => false,
|
||||
title => #{en => <<"Enable SSL">>,
|
||||
zh => <<"开启SSL链接"/utf8>>},
|
||||
description => #{en => <<"Enable SSL or not">>,
|
||||
zh => <<"是否开启 SSL"/utf8>>}
|
||||
},
|
||||
cacertfile => #{
|
||||
order => 15,
|
||||
type => file,
|
||||
required => false,
|
||||
default => <<"etc/certs/cacert.pem">>,
|
||||
title => #{en => <<"CA certificates">>,
|
||||
zh => <<"CA 证书"/utf8>>},
|
||||
description => #{en => <<"The file path of the CA certificates">>,
|
||||
zh => <<"CA 证书路径"/utf8>>}
|
||||
},
|
||||
certfile => #{
|
||||
order => 16,
|
||||
type => file,
|
||||
required => false,
|
||||
default => <<"etc/certs/client-cert.pem">>,
|
||||
title => #{en => <<"SSL Certfile">>,
|
||||
zh => <<"SSL 客户端证书"/utf8>>},
|
||||
description => #{en => <<"The file path of the client certfile">>,
|
||||
zh => <<"客户端证书路径"/utf8>>}
|
||||
},
|
||||
keyfile => #{
|
||||
order => 17,
|
||||
type => file,
|
||||
required => false,
|
||||
default => <<"etc/certs/client-key.pem">>,
|
||||
title => #{en => <<"SSL Keyfile">>,
|
||||
zh => <<"SSL 密钥文件"/utf8>>},
|
||||
description => #{en => <<"The file path of the client keyfile">>,
|
||||
zh => <<"客户端密钥路径"/utf8>>}
|
||||
},
|
||||
ciphers => #{
|
||||
order => 18,
|
||||
type => string,
|
||||
required => false,
|
||||
default => <<"ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,",
|
||||
"ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,",
|
||||
"ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,",
|
||||
"ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,",
|
||||
"AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,",
|
||||
"ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,",
|
||||
"ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,",
|
||||
"DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,",
|
||||
"ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,",
|
||||
"ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,",
|
||||
"DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA">>,
|
||||
title => #{en => <<"SSL Ciphers">>,
|
||||
zh => <<"SSL 加密算法"/utf8>>},
|
||||
description => #{en => <<"SSL Ciphers">>,
|
||||
zh => <<"SSL 加密算法"/utf8>>}
|
||||
}
|
||||
}).
|
||||
|
||||
-define(RESOURCE_CONFIG_SPEC_RPC, #{
|
||||
address => #{
|
||||
order => 1,
|
||||
type => string,
|
||||
required => true,
|
||||
default => <<"emqx2@127.0.0.1">>,
|
||||
title => #{en => <<"EMQ X Node Name">>,
|
||||
zh => <<"EMQ X 节点名称"/utf8>>},
|
||||
description => #{en => <<"EMQ X Remote Node Name">>,
|
||||
zh => <<"远程 EMQ X 节点名称 "/utf8>>}
|
||||
},
|
||||
mountpoint => #{
|
||||
order => 2,
|
||||
type => string,
|
||||
required => false,
|
||||
default => <<"bridge/emqx/${node}/">>,
|
||||
title => #{en => <<"Bridge MountPoint">>,
|
||||
zh => <<"桥接挂载点"/utf8>>},
|
||||
description => #{en => <<"MountPoint for bridge topic<br/>"
|
||||
"Example: The topic of messages sent to `topic1` on local node "
|
||||
"will be transformed to `bridge/aws/${node}/topic1`">>,
|
||||
zh => <<"桥接主题的挂载点<br/>"
|
||||
"示例: 本地节点向 `topic1` 发消息，远程桥接节点的主题"
|
||||
"会变换为 `bridge/aws/${node}/topic1`"/utf8>>}
|
||||
},
|
||||
pool_size => #{
|
||||
order => 3,
|
||||
type => number,
|
||||
required => true,
|
||||
default => 8,
|
||||
title => #{en => <<"Pool Size">>,
|
||||
zh => <<"连接池大小"/utf8>>},
|
||||
description => #{en => <<"MQTT/RPC Connection Pool Size">>,
|
||||
zh => <<"连接池大小"/utf8>>}
|
||||
},
|
||||
reconnect_interval => #{
|
||||
order => 4,
|
||||
type => string,
|
||||
required => false,
|
||||
default => <<"30s">>,
|
||||
title => #{en => <<"Reconnect Interval">>,
|
||||
zh => <<"重连间隔"/utf8>>},
|
||||
description => #{en => <<"Reconnect Interval of bridge">>,
|
||||
zh => <<"重连间隔"/utf8>>}
|
||||
},
|
||||
batch_size => #{
|
||||
order => 5,
|
||||
type => number,
|
||||
required => false,
|
||||
default => 32,
|
||||
title => #{en => <<"Batch Size">>,
|
||||
zh => <<"批处理大小"/utf8>>},
|
||||
description => #{en => <<"Batch Size">>,
|
||||
zh => <<"批处理大小"/utf8>>}
|
||||
},
|
||||
disk_cache => #{
|
||||
order => 6,
|
||||
type => string,
|
||||
required => false,
|
||||
default => <<"off">>,
|
||||
enum => [<<"on">>, <<"off">>],
|
||||
title => #{en => <<"Disk Cache">>,
|
||||
zh => <<"磁盘缓存"/utf8>>},
|
||||
description => #{en => <<"The flag which determines whether messages "
|
||||
"can be cached on local disk when bridge is "
|
||||
"disconnected">>,
|
||||
zh => <<"当桥接断开时用于控制是否将消息缓存到本地磁"
|
||||
"盘队列上"/utf8>>}
|
||||
}
|
||||
}).
|
||||
|
||||
-define(ACTION_PARAM_RESOURCE, #{
|
||||
type => string,
|
||||
required => true,
|
||||
title => #{en => <<"Resource ID">>, zh => <<"资源 ID"/utf8>>},
|
||||
description => #{en => <<"Bind a resource to this action">>,
|
||||
zh => <<"给动作绑定一个资源"/utf8>>}
|
||||
}).
|
||||
|
||||
-resource_type(#{
|
||||
name => ?RESOURCE_TYPE_MQTT,
|
||||
create => on_resource_create,
|
||||
status => on_get_resource_status,
|
||||
destroy => on_resource_destroy,
|
||||
params => ?RESOURCE_CONFIG_SPEC_MQTT,
|
||||
title => #{en => <<"MQTT Bridge">>, zh => <<"MQTT Bridge"/utf8>>},
|
||||
description => #{en => <<"MQTT Message Bridge">>, zh => <<"MQTT 消息桥接"/utf8>>}
|
||||
}).
|
||||
|
||||
|
||||
-resource_type(#{
|
||||
name => ?RESOURCE_TYPE_RPC,
|
||||
create => on_resource_create,
|
||||
status => on_get_resource_status,
|
||||
destroy => on_resource_destroy,
|
||||
params => ?RESOURCE_CONFIG_SPEC_RPC,
|
||||
title => #{en => <<"EMQX Bridge">>, zh => <<"EMQX Bridge"/utf8>>},
|
||||
description => #{en => <<"EMQ X RPC Bridge">>, zh => <<"EMQ X RPC 消息桥接"/utf8>>}
|
||||
}).
|
||||
|
||||
-rule_action(#{
|
||||
name => data_to_mqtt_broker,
|
||||
category => data_forward,
|
||||
for => 'message.publish',
|
||||
types => [?RESOURCE_TYPE_MQTT, ?RESOURCE_TYPE_RPC],
|
||||
create => on_action_create_data_to_mqtt_broker,
|
||||
params => #{'$resource' => ?ACTION_PARAM_RESOURCE,
|
||||
forward_topic => #{
|
||||
order => 1,
|
||||
type => string,
|
||||
required => false,
|
||||
default => <<"">>,
|
||||
title => #{en => <<"Forward Topic">>,
|
||||
zh => <<"转发消息主题"/utf8>>},
|
||||
description => #{en => <<"The topic used when forwarding the message. "
|
||||
"Defaults to the topic of the bridge message if not provided.">>,
|
||||
zh => <<"转发消息时使用的主题。如果未提供，则默认为桥接消息的主题。"/utf8>>}
|
||||
},
|
||||
payload_tmpl => #{
|
||||
order => 2,
|
||||
type => string,
|
||||
input => textarea,
|
||||
required => false,
|
||||
default => <<"">>,
|
||||
title => #{en => <<"Payload Template">>,
|
||||
zh => <<"消息内容模板"/utf8>>},
|
||||
description => #{en => <<"The payload template, variable interpolation is supported. "
|
||||
"If using empty template (default), then the payload will be "
|
||||
"all the available vars in JSON format">>,
|
||||
zh => <<"消息内容模板，支持变量。"
|
||||
"若使用空模板（默认），消息内容为 JSON 格式的所有字段"/utf8>>}
|
||||
}
|
||||
},
|
||||
title => #{en => <<"Data bridge to MQTT Broker">>,
|
||||
zh => <<"桥接数据到 MQTT Broker"/utf8>>},
|
||||
description => #{en => <<"Bridge Data to MQTT Broker">>,
|
||||
zh => <<"桥接数据到 MQTT Broker"/utf8>>}
|
||||
}).
|
||||
|
||||
on_resource_create(ResId, Params) ->
|
||||
?LOG(info, "Initiating Resource ~p, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]),
|
||||
{ok, _} = application:ensure_all_started(ecpool),
|
||||
PoolName = pool_name(ResId),
|
||||
Options = options(Params, PoolName, ResId),
|
||||
start_resource(ResId, PoolName, Options),
|
||||
case test_resource_status(PoolName) of
|
||||
true -> ok;
|
||||
false ->
|
||||
on_resource_destroy(ResId, #{<<"pool">> => PoolName}),
|
||||
error({{?RESOURCE_TYPE_MQTT, ResId}, connection_failed})
|
||||
end,
|
||||
#{<<"pool">> => PoolName}.
|
||||
|
||||
start_resource(ResId, PoolName, Options) ->
|
||||
case ecpool:start_sup_pool(PoolName, ?MODULE, Options) of
|
||||
{ok, _} ->
|
||||
?LOG(info, "Initiated Resource ~p Successfully, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]);
|
||||
{error, {already_started, _Pid}} ->
|
||||
on_resource_destroy(ResId, #{<<"pool">> => PoolName}),
|
||||
start_resource(ResId, PoolName, Options);
|
||||
{error, Reason} ->
|
||||
?LOG(error, "Initiate Resource ~p failed, ResId: ~p, ~p", [?RESOURCE_TYPE_MQTT, ResId, Reason]),
|
||||
on_resource_destroy(ResId, #{<<"pool">> => PoolName}),
|
||||
error({{?RESOURCE_TYPE_MQTT, ResId}, create_failed})
|
||||
end.
|
||||
|
||||
test_resource_status(PoolName) ->
|
||||
IsConnected = fun(Worker) ->
|
||||
case ecpool_worker:client(Worker) of
|
||||
{ok, Bridge} ->
|
||||
try emqx_bridge_worker:status(Bridge) of
|
||||
connected -> true;
|
||||
_ -> false
|
||||
catch _Error:_Reason ->
|
||||
false
|
||||
end;
|
||||
{error, _} ->
|
||||
false
|
||||
end
|
||||
end,
|
||||
Status = [IsConnected(Worker) || {_WorkerName, Worker} <- ecpool:workers(PoolName)],
|
||||
lists:any(fun(St) -> St =:= true end, Status).
|
||||
|
||||
-spec(on_get_resource_status(ResId::binary(), Params::map()) -> Status::map()).
|
||||
on_get_resource_status(_ResId, #{<<"pool">> := PoolName}) ->
|
||||
IsAlive = test_resource_status(PoolName),
|
||||
#{is_alive => IsAlive}.
|
||||
|
||||
on_resource_destroy(ResId, #{<<"pool">> := PoolName}) ->
|
||||
?LOG(info, "Destroying Resource ~p, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]),
|
||||
case ecpool:stop_sup_pool(PoolName) of
|
||||
ok ->
|
||||
?LOG(info, "Destroyed Resource ~p Successfully, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]);
|
||||
{error, Reason} ->
|
||||
?LOG(error, "Destroy Resource ~p failed, ResId: ~p, ~p", [?RESOURCE_TYPE_MQTT, ResId, Reason]),
|
||||
error({{?RESOURCE_TYPE_MQTT, ResId}, destroy_failed})
|
||||
end.
|
||||
|
||||
on_action_create_data_to_mqtt_broker(ActId, Opts = #{<<"pool">> := PoolName,
|
||||
<<"forward_topic">> := ForwardTopic,
|
||||
<<"payload_tmpl">> := PayloadTmpl}) ->
|
||||
?LOG(info, "Initiating Action ~p.", [?FUNCTION_NAME]),
|
||||
PayloadTks = emqx_rule_utils:preproc_tmpl(PayloadTmpl),
|
||||
TopicTks = case ForwardTopic == <<"">> of
|
||||
true -> undefined;
|
||||
false -> emqx_rule_utils:preproc_tmpl(ForwardTopic)
|
||||
end,
|
||||
Opts.
|
||||
|
||||
on_action_data_to_mqtt_broker(Msg, _Env =
|
||||
#{id := Id, clientid := From, flags := Flags,
|
||||
topic := Topic, timestamp := TimeStamp, qos := QoS,
|
||||
?BINDING_KEYS := #{
|
||||
'ActId' := ActId,
|
||||
'PoolName' := PoolName,
|
||||
'TopicTks' := TopicTks,
|
||||
'PayloadTks' := PayloadTks
|
||||
}}) ->
|
||||
Topic1 = case TopicTks =:= undefined of
|
||||
true -> Topic;
|
||||
false -> emqx_rule_utils:proc_tmpl(TopicTks, Msg)
|
||||
end,
|
||||
BrokerMsg = #message{id = Id,
|
||||
qos = QoS,
|
||||
from = From,
|
||||
flags = Flags,
|
||||
topic = Topic1,
|
||||
payload = format_data(PayloadTks, Msg),
|
||||
timestamp = TimeStamp},
|
||||
ecpool:with_client(PoolName,
|
||||
fun(BridgePid) ->
|
||||
BridgePid ! {deliver, rule_engine, BrokerMsg}
|
||||
end),
|
||||
emqx_rule_metrics:inc_actions_success(ActId).
|
||||
|
||||
format_data([], Msg) ->
|
||||
emqx_json:encode(Msg);
|
||||
|
||||
format_data(Tokens, Msg) ->
|
||||
emqx_rule_utils:proc_tmpl(Tokens, Msg).
|
||||
|
||||
subscriptions(Subscriptions) ->
|
||||
scan_binary(<<"[", Subscriptions/binary, "].">>).
|
||||
|
||||
is_node_addr(Addr0) ->
|
||||
Addr = binary_to_list(Addr0),
|
||||
case string:tokens(Addr, "@") of
|
||||
[_NodeName, _Hostname] -> true;
|
||||
_ -> false
|
||||
end.
|
||||
|
||||
scan_binary(Bin) ->
|
||||
TermString = binary_to_list(Bin),
|
||||
scan_string(TermString).
|
||||
|
||||
scan_string(TermString) ->
|
||||
{ok, Tokens, _} = erl_scan:string(TermString),
|
||||
{ok, Term} = erl_parse:parse_term(Tokens),
|
||||
Term.
|
||||
|
||||
connect(Options) when is_list(Options) ->
|
||||
connect(maps:from_list(Options));
|
||||
connect(Options = #{disk_cache := DiskCache, ecpool_worker_id := Id, pool_name := Pool}) ->
|
||||
Options0 = case DiskCache of
|
||||
true ->
|
||||
DataDir = filename:join([emqx:get_env(data_dir), replayq, Pool, integer_to_list(Id)]),
|
||||
QueueOption = #{replayq_dir => DataDir},
|
||||
Options#{queue => QueueOption};
|
||||
false ->
|
||||
Options
|
||||
end,
|
||||
Options1 = case maps:is_key(append, Options0) of
|
||||
false -> Options0;
|
||||
true ->
|
||||
case maps:get(append, Options0, false) of
|
||||
true ->
|
||||
ClientId = lists:concat([str(maps:get(clientid, Options0)), "_", str(emqx_guid:to_hexstr(emqx_guid:gen()))]),
|
||||
Options0#{clientid => ClientId};
|
||||
false ->
|
||||
Options0
|
||||
end
|
||||
end,
|
||||
Options2 = maps:without([ecpool_worker_id, pool_name, append], Options1),
|
||||
emqx_bridge_worker:start_link(name(Pool, Id), Options2).
|
||||
name(Pool, Id) ->
|
||||
list_to_atom(atom_to_list(Pool) ++ ":" ++ integer_to_list(Id)).
|
||||
pool_name(ResId) ->
|
||||
list_to_atom("bridge_mqtt:" ++ str(ResId)).
|
||||
|
||||
options(Options, PoolName, ResId) ->
|
||||
GetD = fun(Key, Default) -> maps:get(Key, Options, Default) end,
|
||||
Get = fun(Key) -> GetD(Key, undefined) end,
|
||||
Address = Get(<<"address">>),
|
||||
[{max_inflight_batches, 32},
|
||||
{forward_mountpoint, str(Get(<<"mountpoint">>))},
|
||||
{disk_cache, cuttlefish_flag:parse(str(GetD(<<"disk_cache">>, "off")))},
|
||||
{start_type, auto},
|
||||
{reconnect_delay_ms, cuttlefish_duration:parse(str(Get(<<"reconnect_interval">>)), ms)},
|
||||
{if_record_metrics, false},
|
||||
{pool_size, GetD(<<"pool_size">>, 1)},
|
||||
{pool_name, PoolName}
|
||||
] ++ case is_node_addr(Address) of
|
||||
true ->
|
||||
[{address, binary_to_atom(Get(<<"address">>), utf8)},
|
||||
{connect_module, emqx_bridge_rpc},
|
||||
{batch_size, Get(<<"batch_size">>)}];
|
||||
false ->
|
||||
[{address, binary_to_list(Address)},
|
||||
{bridge_mode, GetD(<<"bridge_mode">>, true)},
|
||||
{clean_start, true},
|
||||
{clientid, str(Get(<<"clientid">>))},
|
||||
{append, Get(<<"append">>)},
|
||||
{connect_module, emqx_bridge_mqtt},
|
||||
{keepalive, cuttlefish_duration:parse(str(Get(<<"keepalive">>)), s)},
|
||||
{username, str(Get(<<"username">>))},
|
||||
{password, str(Get(<<"password">>))},
|
||||
{proto_ver, mqtt_ver(Get(<<"proto_ver">>))},
|
||||
{retry_interval, cuttlefish_duration:parse(str(GetD(<<"retry_interval">>, "30s")), s)}
|
||||
| maybe_ssl(Options, Get(<<"ssl">>), ResId)]
|
||||
end.
|
||||
|
||||
maybe_ssl(_Options, false, _ResId) ->
|
||||
[];
|
||||
maybe_ssl(Options, true, ResId) ->
|
||||
[{ssl, true}, {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Options, "rules", ResId)}].
|
||||
|
||||
mqtt_ver(ProtoVer) ->
|
||||
case ProtoVer of
|
||||
<<"mqttv3">> -> v3;
|
||||
<<"mqttv4">> -> v4;
|
||||
<<"mqttv5">> -> v5;
|
||||
_ -> v4
|
||||
end.
|
||||
33
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl
Normal file
33
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl
Normal file
@ -0,0 +1,33 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-module(emqx_bridge_mqtt_app).
|
||||
|
||||
-emqx_plugin(bridge).
|
||||
|
||||
-behaviour(application).
|
||||
|
||||
-export([start/2, stop/1]).
|
||||
|
||||
start(_StartType, _StartArgs) ->
|
||||
emqx_ctl:register_command(bridges, {emqx_bridge_mqtt_cli, cli}, []),
|
||||
emqx_bridge_worker:register_metrics(),
|
||||
emqx_bridge_mqtt_sup:start_link().
|
||||
|
||||
stop(_State) ->
|
||||
emqx_ctl:unregister_command(bridges),
|
||||
ok.
|
||||
|
||||
92
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_cli.erl
Normal file
92
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_cli.erl
Normal file
@ -0,0 +1,92 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-module(emqx_bridge_mqtt_cli).
|
||||
|
||||
-include("emqx_bridge_mqtt.hrl").
|
||||
|
||||
-import(lists, [foreach/2]).
|
||||
|
||||
-export([cli/1]).
|
||||
|
||||
cli(["list"]) ->
|
||||
foreach(fun({Name, State0}) ->
|
||||
State = case State0 of
|
||||
connected -> <<"Running">>;
|
||||
_ -> <<"Stopped">>
|
||||
end,
|
||||
emqx_ctl:print("name: ~s status: ~s~n", [Name, State])
|
||||
end, emqx_bridge_mqtt_sup:bridges());
|
||||
|
||||
cli(["start", Name]) ->
|
||||
emqx_ctl:print("~s.~n", [try emqx_bridge_worker:ensure_started(Name) of
|
||||
ok -> <<"Start bridge successfully">>;
|
||||
connected -> <<"Bridge already started">>;
|
||||
_ -> <<"Start bridge failed">>
|
||||
catch
|
||||
_Error:_Reason ->
|
||||
<<"Start bridge failed">>
|
||||
end]);
|
||||
|
||||
cli(["stop", Name]) ->
|
||||
emqx_ctl:print("~s.~n", [try emqx_bridge_worker:ensure_stopped(Name) of
|
||||
ok -> <<"Stop bridge successfully">>;
|
||||
_ -> <<"Stop bridge failed">>
|
||||
catch
|
||||
_Error:_Reason ->
|
||||
<<"Stop bridge failed">>
|
||||
end]);
|
||||
|
||||
cli(["forwards", Name]) ->
|
||||
foreach(fun(Topic) ->
|
||||
emqx_ctl:print("topic: ~s~n", [Topic])
|
||||
end, emqx_bridge_worker:get_forwards(Name));
|
||||
|
||||
cli(["add-forward", Name, Topic]) ->
|
||||
ok = emqx_bridge_worker:ensure_forward_present(Name, iolist_to_binary(Topic)),
|
||||
emqx_ctl:print("Add-forward topic successfully.~n");
|
||||
|
||||
cli(["del-forward", Name, Topic]) ->
|
||||
ok = emqx_bridge_worker:ensure_forward_absent(Name, iolist_to_binary(Topic)),
|
||||
emqx_ctl:print("Del-forward topic successfully.~n");
|
||||
|
||||
cli(["subscriptions", Name]) ->
|
||||
foreach(fun({Topic, Qos}) ->
|
||||
emqx_ctl:print("topic: ~s, qos: ~p~n", [Topic, Qos])
|
||||
end, emqx_bridge_worker:get_subscriptions(Name));
|
||||
|
||||
cli(["add-subscription", Name, Topic, Qos]) ->
|
||||
case emqx_bridge_worker:ensure_subscription_present(Name, Topic, list_to_integer(Qos)) of
|
||||
ok -> emqx_ctl:print("Add-subscription topic successfully.~n");
|
||||
{error, Reason} -> emqx_ctl:print("Add-subscription failed reason: ~p.~n", [Reason])
|
||||
end;
|
||||
|
||||
cli(["del-subscription", Name, Topic]) ->
|
||||
ok = emqx_bridge_worker:ensure_subscription_absent(Name, Topic),
|
||||
emqx_ctl:print("Del-subscription topic successfully.~n");
|
||||
|
||||
cli(_) ->
|
||||
emqx_ctl:usage([{"bridges list", "List bridges"},
|
||||
{"bridges start <Name>", "Start a bridge"},
|
||||
{"bridges stop <Name>", "Stop a bridge"},
|
||||
{"bridges forwards <Name>", "Show a bridge forward topic"},
|
||||
{"bridges add-forward <Name> <Topic>", "Add bridge forward topic"},
|
||||
{"bridges del-forward <Name> <Topic>", "Delete bridge forward topic"},
|
||||
{"bridges subscriptions <Name>", "Show a bridge subscriptions topic"},
|
||||
{"bridges add-subscription <Name> <Topic> <Qos>", "Add bridge subscriptions topic"},
|
||||
{"bridges del-subscription <Name> <Topic>", "Delete bridge subscriptions topic"}]).
|
||||
|
||||
|
||||
84
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl
Normal file
84
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl
Normal file
@ -0,0 +1,84 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-module(emqx_bridge_mqtt_sup).
|
||||
-behaviour(supervisor).
|
||||
|
||||
-include("emqx_bridge_mqtt.hrl").
|
||||
-include_lib("emqx/include/logger.hrl").
|
||||
|
||||
-logger_header("[Bridge]").
|
||||
|
||||
%% APIs
|
||||
-export([ start_link/0
|
||||
, start_link/1
|
||||
]).
|
||||
|
||||
-export([ create_bridge/2
|
||||
, drop_bridge/1
|
||||
, bridges/0
|
||||
, is_bridge_exist/1
|
||||
]).
|
||||
|
||||
%% supervisor callbacks
|
||||
-export([init/1]).
|
||||
|
||||
-define(SUP, ?MODULE).
|
||||
-define(WORKER_SUP, emqx_bridge_worker_sup).
|
||||
|
||||
start_link() -> start_link(?SUP).
|
||||
|
||||
start_link(Name) ->
|
||||
supervisor:start_link({local, Name}, ?MODULE, Name).
|
||||
|
||||
init(?SUP) ->
|
||||
BridgesConf = application:get_env(?APP, bridges, []),
|
||||
BridgeSpec = lists:map(fun bridge_spec/1, BridgesConf),
|
||||
SupFlag = #{strategy => one_for_one,
|
||||
intensity => 100,
|
||||
period => 10},
|
||||
{ok, {SupFlag, BridgeSpec}}.
|
||||
|
||||
bridge_spec({Name, Config}) ->
|
||||
#{id => Name,
|
||||
start => {emqx_bridge_worker, start_link, [Name, Config]},
|
||||
restart => permanent,
|
||||
shutdown => 5000,
|
||||
type => worker,
|
||||
modules => [emqx_bridge_worker]}.
|
||||
|
||||
-spec(bridges() -> [{node(), map()}]).
|
||||
bridges() ->
|
||||
[{Name, emqx_bridge_worker:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?SUP)].
|
||||
|
||||
-spec(is_bridge_exist(atom() | pid()) -> boolean()).
|
||||
is_bridge_exist(Id) ->
|
||||
case supervisor:get_childspec(?SUP, Id) of
|
||||
{ok, _ChildSpec} -> true;
|
||||
{error, _Error} -> false
|
||||
end.
|
||||
|
||||
create_bridge(Id, Config) ->
|
||||
supervisor:start_child(?SUP, bridge_spec({Id, Config})).
|
||||
|
||||
drop_bridge(Id) ->
|
||||
case supervisor:terminate_child(?SUP, Id) of
|
||||
ok ->
|
||||
supervisor:delete_child(?SUP, Id);
|
||||
{error, Error} ->
|
||||
?LOG(error, "Delete bridge failed, error : ~p", [Error]),
|
||||
{error, Error}
|
||||
end.
|
||||
99
apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl
Normal file
99
apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl
Normal file
@ -0,0 +1,99 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-module(emqx_bridge_msg).
|
||||
|
||||
-export([ to_binary/1
|
||||
, from_binary/1
|
||||
, to_export/3
|
||||
, to_broker_msgs/1
|
||||
, to_broker_msg/1
|
||||
, to_broker_msg/2
|
||||
, estimate_size/1
|
||||
]).
|
||||
|
||||
-export_type([msg/0]).
|
||||
|
||||
-include_lib("emqx/include/emqx.hrl").
|
||||
|
||||
-include_lib("emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl").
|
||||
-include_lib("emqtt/include/emqtt.hrl").
|
||||
|
||||
|
||||
-type msg() :: emqx_types:message().
|
||||
-type exp_msg() :: emqx_types:message() | #mqtt_msg{}.
|
||||
|
||||
%% @doc Make export format:
|
||||
%% 1. Mount topic to a prefix
|
||||
%% 2. Fix QoS to 1
|
||||
%% @end
|
||||
%% Shame that we have to know the callback module here
|
||||
%% would be great if we can get rid of #mqtt_msg{} record
|
||||
%% and use #message{} in all places.
|
||||
-spec to_export(emqx_bridge_rpc | emqx_bridge_worker,
|
||||
undefined | binary(), msg()) -> exp_msg().
|
||||
to_export(emqx_bridge_mqtt, Mountpoint,
|
||||
#message{topic = Topic,
|
||||
payload = Payload,
|
||||
flags = Flags,
|
||||
qos = QoS
|
||||
}) ->
|
||||
Retain = maps:get(retain, Flags, false),
|
||||
#mqtt_msg{qos = QoS,
|
||||
retain = Retain,
|
||||
topic = topic(Mountpoint, Topic),
|
||||
props = #{},
|
||||
payload = Payload};
|
||||
to_export(_Module, Mountpoint,
|
||||
#message{topic = Topic} = Msg) ->
|
||||
Msg#message{topic = topic(Mountpoint, Topic)}.
|
||||
|
||||
%% @doc Make `binary()' in order to make iodata to be persisted on disk.
|
||||
-spec to_binary(msg()) -> binary().
|
||||
to_binary(Msg) -> term_to_binary(Msg).
|
||||
|
||||
%% @doc Unmarshal binary into `msg()'.
|
||||
-spec from_binary(binary()) -> msg().
|
||||
from_binary(Bin) -> binary_to_term(Bin).
|
||||
|
||||
%% @doc Estimate the size of a message.
|
||||
%% Count only the topic length + payload size
|
||||
-spec estimate_size(msg()) -> integer().
|
||||
estimate_size(#message{topic = Topic, payload = Payload}) ->
|
||||
size(Topic) + size(Payload).
|
||||
|
||||
%% @doc By message/batch receiver, transform received batch into
|
||||
%% messages to deliver to local brokers.
|
||||
to_broker_msgs(Batch) -> lists:map(fun to_broker_msg/1, Batch).
|
||||
|
||||
to_broker_msg(#message{} = Msg) ->
|
||||
%% internal format from another EMQX node via rpc
|
||||
Msg;
|
||||
to_broker_msg(Msg) ->
|
||||
to_broker_msg(Msg, undefined).
|
||||
to_broker_msg(#{qos := QoS, dup := Dup, retain := Retain, topic := Topic,
|
||||
properties := Props, payload := Payload}, Mountpoint) ->
|
||||
%% published from remote node over a MQTT connection
|
||||
set_headers(Props,
|
||||
emqx_message:set_flags(#{dup => Dup, retain => Retain},
|
||||
emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))).
|
||||
|
||||
set_headers(undefined, Msg) ->
|
||||
Msg;
|
||||
set_headers(Val, Msg) ->
|
||||
emqx_message:set_headers(Val, Msg).
|
||||
topic(undefined, Topic) -> Topic;
|
||||
topic(Prefix, Topic) -> emqx_topic:prepend(Prefix, Topic).
|
||||
100
apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl
Normal file
100
apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl
Normal file
@ -0,0 +1,100 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
%% @doc This module implements EMQX Bridge transport layer based on gen_rpc.
|
||||
|
||||
-module(emqx_bridge_rpc).
|
||||
|
||||
-behaviour(emqx_bridge_connect).
|
||||
|
||||
%% behaviour callbacks
|
||||
-export([ start/1
|
||||
, send/2
|
||||
, stop/1
|
||||
]).
|
||||
|
||||
%% Internal exports
|
||||
-export([ handle_send/1
|
||||
, heartbeat/2
|
||||
]).
|
||||
|
||||
-type ack_ref() :: emqx_bridge_worker:ack_ref().
|
||||
-type batch() :: emqx_bridge_worker:batch().
|
||||
-type node_or_tuple() :: atom() | {atom(), term()}.
|
||||
|
||||
-define(HEARTBEAT_INTERVAL, timer:seconds(1)).
|
||||
|
||||
-define(RPC, emqx_rpc).
|
||||
|
||||
start(#{address := Remote}) ->
|
||||
case poke(Remote) of
|
||||
ok ->
|
||||
Pid = proc_lib:spawn_link(?MODULE, heartbeat, [self(), Remote]),
|
||||
{ok, #{client_pid => Pid, address => Remote}};
|
||||
Error ->
|
||||
Error
|
||||
end.
|
||||
|
||||
stop(#{client_pid := Pid}) when is_pid(Pid) ->
|
||||
Ref = erlang:monitor(process, Pid),
|
||||
unlink(Pid),
|
||||
Pid ! stop,
|
||||
receive
|
||||
{'DOWN', Ref, process, Pid, _Reason} ->
|
||||
ok
|
||||
after
|
||||
1000 ->
|
||||
exit(Pid, kill)
|
||||
end,
|
||||
ok.
|
||||
|
||||
%% @doc Callback for `emqx_bridge_connect' behaviour
|
||||
-spec send(#{address := node_or_tuple(), _ => _}, batch()) -> {ok, ack_ref()} | {error, any()}.
|
||||
send(#{address := Remote}, Batch) ->
|
||||
case ?RPC:call(Remote, ?MODULE, handle_send, [Batch]) of
|
||||
ok ->
|
||||
Ref = make_ref(),
|
||||
self() ! {batch_ack, Ref},
|
||||
{ok, Ref};
|
||||
{badrpc, Reason} -> {error, Reason}
|
||||
end.
|
||||
|
||||
%% @doc Handle send on receiver side.
|
||||
-spec handle_send(batch()) -> ok.
|
||||
handle_send(Batch) ->
|
||||
lists:foreach(fun(Msg) -> emqx_broker:publish(Msg) end, Batch).
|
||||
|
||||
%% @hidden Heartbeat loop
|
||||
heartbeat(Parent, RemoteNode) ->
|
||||
Interval = ?HEARTBEAT_INTERVAL,
|
||||
receive
|
||||
stop -> exit(normal)
|
||||
after
|
||||
Interval ->
|
||||
case poke(RemoteNode) of
|
||||
ok ->
|
||||
?MODULE:heartbeat(Parent, RemoteNode);
|
||||
{error, Reason} ->
|
||||
Parent ! {disconnected, self(), Reason},
|
||||
exit(normal)
|
||||
end
|
||||
end.
|
||||
|
||||
poke(Node) ->
|
||||
case ?RPC:call(Node, erlang, node, []) of
|
||||
Node -> ok;
|
||||
{badrpc, Reason} -> {error, Reason}
|
||||
end.
|
||||
641
apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl
Normal file
641
apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl
Normal file
@ -0,0 +1,641 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
%% @doc Bridge works in two layers (1) batching layer (2) transport layer
|
||||
%% The `bridge' batching layer collects local messages in batches and sends over
|
||||
%% to remote MQTT node/cluster via `connection' transport layer.
|
||||
%% In case `REMOTE' is also an EMQX node, `connection' is recommended to be
|
||||
%% the `gen_rpc' based implementation `emqx_bridge_rpc'. Otherwise `connection'
|
||||
%% has to be `emqx_bridge_mqtt'.
|
||||
%%
|
||||
%% ```
|
||||
%% +------+ +--------+
|
||||
%% | EMQX | | REMOTE |
|
||||
%% | | | |
|
||||
%% | (bridge) <==(connection)==> | |
|
||||
%% | | | |
|
||||
%% | | | |
|
||||
%% +------+ +--------+
|
||||
%% '''
|
||||
%%
|
||||
%%
|
||||
%% This module implements 2 kinds of APIs with regards to batching and
|
||||
%% messaging protocol. (1) A `gen_statem' based local batch collector;
|
||||
%% (2) APIs for incoming remote batches/messages.
|
||||
%%
|
||||
%% Batch collector state diagram
|
||||
%%
|
||||
%% [idle] --(0) --> [connecting] --(2)--> [connected]
|
||||
%% | ^ |
|
||||
%% | | |
|
||||
%% '--(1)---'--------(3)------'
|
||||
%%
|
||||
%% (0): auto or manual start
|
||||
%% (1): retry timeout
|
||||
%% (2): successfuly connected to remote node/cluster
|
||||
%% (3): received {disconnected, Reason} OR
|
||||
%% failed to send to remote node/cluster.
|
||||
%%
|
||||
%% NOTE: A bridge worker may subscribe to multiple (including wildcard)
|
||||
%% local topics, and the underlying `emqx_bridge_connect' may subscribe to
|
||||
%% multiple remote topics, however, worker/connections are not designed
|
||||
%% to support automatic load-balancing, i.e. in case it can not keep up
|
||||
%% with the amount of messages comming in, administrator should split and
|
||||
%% balance topics between worker/connections manually.
|
||||
%%
|
||||
%% NOTES:
|
||||
%% * Local messages are all normalised to QoS-1 when exporting to remote
|
||||
|
||||
-module(emqx_bridge_worker).
|
||||
-behaviour(gen_statem).
|
||||
|
||||
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
|
||||
|
||||
%% APIs
|
||||
-export([ start_link/1
|
||||
, start_link/2
|
||||
, register_metrics/0
|
||||
, stop/1
|
||||
]).
|
||||
|
||||
%% gen_statem callbacks
|
||||
-export([ terminate/3
|
||||
, code_change/4
|
||||
, init/1
|
||||
, callback_mode/0
|
||||
]).
|
||||
|
||||
%% state functions
|
||||
-export([ idle/3
|
||||
, connected/3
|
||||
]).
|
||||
|
||||
%% management APIs
|
||||
-export([ ensure_started/1
|
||||
, ensure_stopped/1
|
||||
, ensure_stopped/2
|
||||
, status/1
|
||||
]).
|
||||
|
||||
-export([ get_forwards/1
|
||||
, ensure_forward_present/2
|
||||
, ensure_forward_absent/2
|
||||
]).
|
||||
|
||||
-export([ get_subscriptions/1
|
||||
, ensure_subscription_present/3
|
||||
, ensure_subscription_absent/2
|
||||
]).
|
||||
|
||||
%% Internal
|
||||
-export([msg_marshaller/1]).
|
||||
|
||||
-export_type([ config/0
|
||||
, batch/0
|
||||
, ack_ref/0
|
||||
]).
|
||||
|
||||
-type id() :: atom() | string() | pid().
|
||||
-type qos() :: emqx_mqtt_types:qos().
|
||||
-type config() :: map().
|
||||
-type batch() :: [emqx_bridge_msg:exp_msg()].
|
||||
-type ack_ref() :: term().
|
||||
-type topic() :: emqx_topic:topic().
|
||||
|
||||
-include_lib("emqx/include/logger.hrl").
|
||||
-include_lib("emqx/include/emqx_mqtt.hrl").
|
||||
|
||||
-logger_header("[Bridge]").
|
||||
|
||||
%% same as default in-flight limit for emqtt
|
||||
-define(DEFAULT_BATCH_SIZE, 32).
|
||||
-define(DEFAULT_RECONNECT_DELAY_MS, timer:seconds(5)).
|
||||
-define(DEFAULT_SEG_BYTES, (1 bsl 20)).
|
||||
-define(DEFAULT_MAX_TOTAL_SIZE, (1 bsl 31)).
|
||||
-define(NO_BRIDGE_HANDLER, undefined).
|
||||
|
||||
%% @doc Start a bridge worker. Supported configs:
|
||||
%% start_type: 'manual' (default) or 'auto', when manual, bridge will stay
|
||||
%% at 'idle' state until a manual call to start it.
|
||||
%% connect_module: The module which implements emqx_bridge_connect behaviour
|
||||
%% and work as message batch transport layer
|
||||
%% reconnect_delay_ms: Delay in milli-seconds for the bridge worker to retry
|
||||
%% in case of transportation failure.
|
||||
%% max_inflight: Max number of batches allowed to send-ahead before receiving
|
||||
%% confirmation from remote node/cluster
|
||||
%% mountpoint: The topic mount point for messages sent to remote node/cluster
|
||||
%% `undefined', `<<>>' or `""' to disable
|
||||
%% forwards: Local topics to subscribe.
|
||||
%% queue.batch_bytes_limit: Max number of bytes to collect in a batch for each
|
||||
%% send call towards emqx_bridge_connect
|
||||
%% queue.batch_count_limit: Max number of messages to collect in a batch for
|
||||
%% each send call towards emqx_bridge_connect
|
||||
%% queue.replayq_dir: Directory where replayq should persist messages
|
||||
%% queue.replayq_seg_bytes: Size in bytes for each replayq segment file
|
||||
%%
|
||||
%% Find more connection specific configs in the callback modules
|
||||
%% of emqx_bridge_connect behaviour.
|
||||
start_link(Config) when is_list(Config) ->
|
||||
start_link(maps:from_list(Config));
|
||||
start_link(Config) ->
|
||||
gen_statem:start_link(?MODULE, Config, []).
|
||||
|
||||
start_link(Name, Config) when is_list(Config) ->
|
||||
start_link(Name, maps:from_list(Config));
|
||||
start_link(Name, Config) ->
|
||||
Name1 = name(Name),
|
||||
gen_statem:start_link({local, Name1}, ?MODULE, Config#{name => Name1}, []).
|
||||
|
||||
ensure_started(Name) ->
|
||||
gen_statem:call(name(Name), ensure_started).
|
||||
|
||||
%% @doc Manually stop bridge worker. State idempotency ensured.
|
||||
ensure_stopped(Id) ->
|
||||
ensure_stopped(Id, 1000).
|
||||
|
||||
ensure_stopped(Id, Timeout) ->
|
||||
Pid = case id(Id) of
|
||||
P when is_pid(P) -> P;
|
||||
N -> whereis(N)
|
||||
end,
|
||||
case Pid of
|
||||
undefined ->
|
||||
ok;
|
||||
_ ->
|
||||
MRef = monitor(process, Pid),
|
||||
unlink(Pid),
|
||||
_ = gen_statem:call(id(Id), ensure_stopped, Timeout),
|
||||
receive
|
||||
{'DOWN', MRef, _, _, _} ->
|
||||
ok
|
||||
after
|
||||
Timeout ->
|
||||
exit(Pid, kill)
|
||||
end
|
||||
end.
|
||||
|
||||
stop(Pid) -> gen_statem:stop(Pid).
|
||||
|
||||
status(Pid) when is_pid(Pid) ->
|
||||
gen_statem:call(Pid, status);
|
||||
status(Id) ->
|
||||
gen_statem:call(name(Id), status).
|
||||
|
||||
%% @doc Return all forwards (local subscriptions).
|
||||
-spec get_forwards(id()) -> [topic()].
|
||||
get_forwards(Id) -> gen_statem:call(id(Id), get_forwards, timer:seconds(1000)).
|
||||
|
||||
%% @doc Return all subscriptions (subscription over mqtt connection to remote broker).
|
||||
-spec get_subscriptions(id()) -> [{emqx_topic:topic(), qos()}].
|
||||
get_subscriptions(Id) -> gen_statem:call(id(Id), get_subscriptions).
|
||||
|
||||
%% @doc Add a new forward (local topic subscription).
|
||||
-spec ensure_forward_present(id(), topic()) -> ok.
|
||||
ensure_forward_present(Id, Topic) ->
|
||||
gen_statem:call(id(Id), {ensure_present, forwards, topic(Topic)}).
|
||||
|
||||
%% @doc Ensure a forward topic is deleted.
|
||||
-spec ensure_forward_absent(id(), topic()) -> ok.
|
||||
ensure_forward_absent(Id, Topic) ->
|
||||
gen_statem:call(id(Id), {ensure_absent, forwards, topic(Topic)}).
|
||||
|
||||
%% @doc Ensure subscribed to remote topic.
|
||||
%% NOTE: only applicable when connection module is emqx_bridge_mqtt
|
||||
%% return `{error, no_remote_subscription_support}' otherwise.
|
||||
-spec ensure_subscription_present(id(), topic(), qos()) -> ok | {error, any()}.
|
||||
ensure_subscription_present(Id, Topic, QoS) ->
|
||||
gen_statem:call(id(Id), {ensure_present, subscriptions, {topic(Topic), QoS}}).
|
||||
|
||||
%% @doc Ensure unsubscribed from remote topic.
|
||||
%% NOTE: only applicable when connection module is emqx_bridge_mqtt
|
||||
-spec ensure_subscription_absent(id(), topic()) -> ok.
|
||||
ensure_subscription_absent(Id, Topic) ->
|
||||
gen_statem:call(id(Id), {ensure_absent, subscriptions, topic(Topic)}).
|
||||
|
||||
callback_mode() -> [state_functions].
|
||||
|
||||
%% @doc Config should be a map().
|
||||
init(Config) ->
|
||||
erlang:process_flag(trap_exit, true),
|
||||
ConnectModule = maps:get(connect_module, Config),
|
||||
Subscriptions = maps:get(subscriptions, Config, []),
|
||||
Forwards = maps:get(forwards, Config, []),
|
||||
Queue = open_replayq(Config),
|
||||
State = init_opts(Config),
|
||||
Topics = [iolist_to_binary(T) || T <- Forwards],
|
||||
Subs = check_subscriptions(Subscriptions),
|
||||
ConnectCfg = get_conn_cfg(Config),
|
||||
self() ! idle,
|
||||
{ok, idle, State#{connect_module => ConnectModule,
|
||||
connect_cfg => ConnectCfg,
|
||||
forwards => Topics,
|
||||
subscriptions => Subs,
|
||||
replayq => Queue
|
||||
}}.
|
||||
|
||||
init_opts(Config) ->
|
||||
IfRecordMetrics = maps:get(if_record_metrics, Config, true),
|
||||
ReconnDelayMs = maps:get(reconnect_delay_ms, Config, ?DEFAULT_RECONNECT_DELAY_MS),
|
||||
StartType = maps:get(start_type, Config, manual),
|
||||
BridgeHandler = maps:get(bridge_handler, Config, ?NO_BRIDGE_HANDLER),
|
||||
Mountpoint = maps:get(forward_mountpoint, Config, undefined),
|
||||
ReceiveMountpoint = maps:get(receive_mountpoint, Config, undefined),
|
||||
MaxInflightSize = maps:get(max_inflight, Config, ?DEFAULT_BATCH_SIZE),
|
||||
BatchSize = maps:get(batch_size, Config, ?DEFAULT_BATCH_SIZE),
|
||||
Name = maps:get(name, Config, undefined),
|
||||
#{start_type => StartType,
|
||||
reconnect_delay_ms => ReconnDelayMs,
|
||||
batch_size => BatchSize,
|
||||
mountpoint => format_mountpoint(Mountpoint),
|
||||
receive_mountpoint => ReceiveMountpoint,
|
||||
inflight => [],
|
||||
max_inflight => MaxInflightSize,
|
||||
connection => undefined,
|
||||
bridge_handler => BridgeHandler,
|
||||
if_record_metrics => IfRecordMetrics,
|
||||
name => Name}.
|
||||
|
||||
open_replayq(Config) ->
|
||||
QCfg = maps:get(queue, Config, #{}),
|
||||
Dir = maps:get(replayq_dir, QCfg, undefined),
|
||||
SegBytes = maps:get(replayq_seg_bytes, QCfg, ?DEFAULT_SEG_BYTES),
|
||||
MaxTotalSize = maps:get(max_total_size, QCfg, ?DEFAULT_MAX_TOTAL_SIZE),
|
||||
QueueConfig = case Dir =:= undefined orelse Dir =:= "" of
|
||||
true -> #{mem_only => true};
|
||||
false -> #{dir => Dir, seg_bytes => SegBytes, max_total_size => MaxTotalSize}
|
||||
end,
|
||||
replayq:open(QueueConfig#{sizer => fun emqx_bridge_msg:estimate_size/1,
|
||||
marshaller => fun ?MODULE:msg_marshaller/1}).
|
||||
|
||||
check_subscriptions(Subscriptions) ->
|
||||
lists:map(fun({Topic, QoS}) ->
|
||||
Topic1 = iolist_to_binary(Topic),
|
||||
true = emqx_topic:validate({filter, Topic1}),
|
||||
{Topic1, QoS}
|
||||
end, Subscriptions).
|
||||
|
||||
get_conn_cfg(Config) ->
|
||||
maps:without([connect_module,
|
||||
queue,
|
||||
reconnect_delay_ms,
|
||||
forwards,
|
||||
mountpoint,
|
||||
name
|
||||
], Config).
|
||||
|
||||
code_change(_Vsn, State, Data, _Extra) ->
|
||||
{ok, State, Data}.
|
||||
|
||||
terminate(_Reason, _StateName, #{replayq := Q} = State) ->
|
||||
_ = disconnect(State),
|
||||
_ = replayq:close(Q),
|
||||
ok.
|
||||
|
||||
%% ensure_started will be deprecated in the future
|
||||
idle({call, From}, ensure_started, State) ->
|
||||
case do_connect(State) of
|
||||
{ok, State1} ->
|
||||
{next_state, connected, State1, [{reply, From, ok}, {state_timeout, 0, connected}]};
|
||||
{error, Reason, _State} ->
|
||||
{keep_state_and_data, [{reply, From, {error, Reason}}]}
|
||||
end;
|
||||
%% @doc Standing by for manual start.
|
||||
idle(info, idle, #{start_type := manual}) ->
|
||||
keep_state_and_data;
|
||||
%% @doc Standing by for auto start.
|
||||
idle(info, idle, #{start_type := auto} = State) ->
|
||||
connecting(State);
|
||||
idle(state_timeout, reconnect, State) ->
|
||||
connecting(State);
|
||||
|
||||
idle(info, {batch_ack, Ref}, State) ->
|
||||
NewState = handle_batch_ack(State, Ref),
|
||||
{keep_state, NewState};
|
||||
|
||||
idle(Type, Content, State) ->
|
||||
common(idle, Type, Content, State).
|
||||
|
||||
connecting(#{reconnect_delay_ms := ReconnectDelayMs} = State) ->
|
||||
case do_connect(State) of
|
||||
{ok, State1} ->
|
||||
{next_state, connected, State1, {state_timeout, 0, connected}};
|
||||
_ ->
|
||||
{keep_state_and_data, {state_timeout, ReconnectDelayMs, reconnect}}
|
||||
end.
|
||||
|
||||
connected(state_timeout, connected, #{inflight := Inflight} = State) ->
|
||||
case retry_inflight(State#{inflight := []}, Inflight) of
|
||||
{ok, NewState} ->
|
||||
{keep_state, NewState, {next_event, internal, maybe_send}};
|
||||
{error, NewState} ->
|
||||
{keep_state, NewState}
|
||||
end;
|
||||
connected(internal, maybe_send, State) ->
|
||||
{_, NewState} = pop_and_send(State),
|
||||
{keep_state, NewState};
|
||||
|
||||
connected(info, {disconnected, Conn, Reason},
|
||||
#{connection := Connection, name := Name, reconnect_delay_ms := ReconnectDelayMs} = State) ->
|
||||
?tp(info, disconnected, #{name => Name, reason => Reason}),
|
||||
case Conn =:= maps:get(client_pid, Connection, undefined) of
|
||||
true ->
|
||||
{next_state, idle, State#{connection => undefined}, {state_timeout, ReconnectDelayMs, reconnect}};
|
||||
false ->
|
||||
keep_state_and_data
|
||||
end;
|
||||
connected(info, {batch_ack, Ref}, State) ->
|
||||
NewState = handle_batch_ack(State, Ref),
|
||||
{keep_state, NewState, {next_event, internal, maybe_send}};
|
||||
connected(Type, Content, State) ->
|
||||
common(connected, Type, Content, State).
|
||||
|
||||
%% Common handlers
|
||||
common(StateName, {call, From}, status, _State) ->
|
||||
{keep_state_and_data, [{reply, From, StateName}]};
|
||||
common(_StateName, {call, From}, ensure_started, _State) ->
|
||||
{keep_state_and_data, [{reply, From, connected}]};
|
||||
common(_StateName, {call, From}, ensure_stopped, _State) ->
|
||||
{stop_and_reply, {shutdown, manual}, [{reply, From, ok}]};
|
||||
common(_StateName, {call, From}, get_forwards, #{forwards := Forwards}) ->
|
||||
{keep_state_and_data, [{reply, From, Forwards}]};
|
||||
common(_StateName, {call, From}, get_subscriptions, #{subscriptions := Subs}) ->
|
||||
{keep_state_and_data, [{reply, From, Subs}]};
|
||||
common(_StateName, {call, From}, {ensure_present, What, Topic}, State) ->
|
||||
{Result, NewState} = ensure_present(What, Topic, State),
|
||||
{keep_state, NewState, [{reply, From, Result}]};
|
||||
common(_StateName, {call, From}, {ensure_absent, What, Topic}, State) ->
|
||||
{Result, NewState} = ensure_absent(What, Topic, State),
|
||||
{keep_state, NewState, [{reply, From, Result}]};
|
||||
common(_StateName, info, {deliver, _, Msg},
|
||||
State = #{replayq := Q, if_record_metrics := IfRecordMetric}) ->
|
||||
Msgs = collect([Msg]),
|
||||
bridges_metrics_inc(IfRecordMetric,
|
||||
'bridge.mqtt.message_received',
|
||||
length(Msgs)
|
||||
),
|
||||
NewQ = replayq:append(Q, Msgs),
|
||||
{keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}};
|
||||
common(_StateName, info, {'EXIT', _, _}, State) ->
|
||||
{keep_state, State};
|
||||
common(StateName, Type, Content, #{name := Name} = State) ->
|
||||
?LOG(notice, "Bridge ~p discarded ~p type event at state ~p:~p",
|
||||
[Name, Type, StateName, Content]),
|
||||
{keep_state, State}.
|
||||
|
||||
eval_bridge_handler(State = #{bridge_handler := ?NO_BRIDGE_HANDLER}, _Msg) ->
|
||||
State;
|
||||
eval_bridge_handler(State = #{bridge_handler := Handler}, Msg) ->
|
||||
Handler(Msg),
|
||||
State.
|
||||
|
||||
ensure_present(Key, Topic, State) ->
|
||||
Topics = maps:get(Key, State),
|
||||
case is_topic_present(Topic, Topics) of
|
||||
true ->
|
||||
{ok, State};
|
||||
false ->
|
||||
R = do_ensure_present(Key, Topic, State),
|
||||
{R, State#{Key := lists:usort([Topic | Topics])}}
|
||||
end.
|
||||
|
||||
ensure_absent(Key, Topic, State) ->
|
||||
Topics = maps:get(Key, State),
|
||||
case is_topic_present(Topic, Topics) of
|
||||
true ->
|
||||
R = do_ensure_absent(Key, Topic, State),
|
||||
{R, State#{Key := ensure_topic_absent(Topic, Topics)}};
|
||||
false ->
|
||||
{ok, State}
|
||||
end.
|
||||
|
||||
ensure_topic_absent(_Topic, []) -> [];
|
||||
ensure_topic_absent(Topic, [{_, _} | _] = L) -> lists:keydelete(Topic, 1, L);
|
||||
ensure_topic_absent(Topic, L) -> lists:delete(Topic, L).
|
||||
|
||||
is_topic_present({Topic, _QoS}, Topics) ->
|
||||
is_topic_present(Topic, Topics);
|
||||
is_topic_present(Topic, Topics) ->
|
||||
lists:member(Topic, Topics) orelse false =/= lists:keyfind(Topic, 1, Topics).
|
||||
|
||||
do_connect(#{forwards := Forwards,
|
||||
subscriptions := Subs,
|
||||
connect_module := ConnectModule,
|
||||
connect_cfg := ConnectCfg,
|
||||
inflight := Inflight,
|
||||
name := Name} = State) ->
|
||||
ok = subscribe_local_topics(Forwards, Name),
|
||||
case emqx_bridge_connect:start(ConnectModule, ConnectCfg#{subscriptions => Subs}) of
|
||||
{ok, Conn} ->
|
||||
Res = eval_bridge_handler(State#{connection => Conn}, connected),
|
||||
?tp(info, connected, #{name => Name, inflight => length(Inflight)}),
|
||||
{ok, Res};
|
||||
{error, Reason} ->
|
||||
{error, Reason, State}
|
||||
end.
|
||||
|
||||
do_ensure_present(forwards, Topic, #{name := Name}) ->
|
||||
subscribe_local_topic(Topic, Name);
|
||||
do_ensure_present(subscriptions, _Topic, #{connection := undefined}) ->
|
||||
{error, no_connection};
|
||||
do_ensure_present(subscriptions, _Topic, #{connect_module := emqx_bridge_rpc}) ->
|
||||
{error, no_remote_subscription_support};
|
||||
do_ensure_present(subscriptions, {Topic, QoS}, #{connect_module := ConnectModule,
|
||||
connection := Conn}) ->
|
||||
ConnectModule:ensure_subscribed(Conn, Topic, QoS).
|
||||
|
||||
do_ensure_absent(forwards, Topic, _) ->
|
||||
do_unsubscribe(Topic);
|
||||
do_ensure_absent(subscriptions, _Topic, #{connection := undefined}) ->
|
||||
{error, no_connection};
|
||||
do_ensure_absent(subscriptions, _Topic, #{connect_module := emqx_bridge_rpc}) ->
|
||||
{error, no_remote_subscription_support};
|
||||
do_ensure_absent(subscriptions, Topic, #{connect_module := ConnectModule,
|
||||
connection := Conn}) ->
|
||||
ConnectModule:ensure_unsubscribed(Conn, Topic).
|
||||
|
||||
collect(Acc) ->
|
||||
receive
|
||||
{deliver, _, Msg} ->
|
||||
collect([Msg | Acc])
|
||||
after
|
||||
0 ->
|
||||
lists:reverse(Acc)
|
||||
end.
|
||||
|
||||
%% Retry all inflight (previously sent but not acked) batches.
|
||||
retry_inflight(State, []) -> {ok, State};
|
||||
retry_inflight(State, [#{q_ack_ref := QAckRef, batch := Batch} | Rest] = OldInf) ->
|
||||
case do_send(State, QAckRef, Batch) of
|
||||
{ok, State1} ->
|
||||
retry_inflight(State1, Rest);
|
||||
{error, #{inflight := NewInf} = State1} ->
|
||||
{error, State1#{inflight := NewInf ++ OldInf}}
|
||||
end.
|
||||
|
||||
pop_and_send(#{inflight := Inflight, max_inflight := Max } = State) ->
|
||||
pop_and_send_loop(State, Max - length(Inflight)).
|
||||
|
||||
pop_and_send_loop(State, 0) ->
|
||||
?tp(debug, inflight_full, #{}),
|
||||
{ok, State};
|
||||
pop_and_send_loop(#{replayq := Q, connect_module := Module} = State, N) ->
|
||||
case replayq:is_empty(Q) of
|
||||
true ->
|
||||
?tp(debug, replayq_drained, #{}),
|
||||
{ok, State};
|
||||
false ->
|
||||
BatchSize = case Module of
|
||||
emqx_bridge_rpc -> maps:get(batch_size, State);
|
||||
_ -> 1
|
||||
end,
|
||||
Opts = #{count_limit => BatchSize, bytes_limit => 999999999},
|
||||
{Q1, QAckRef, Batch} = replayq:pop(Q, Opts),
|
||||
case do_send(State#{replayq := Q1}, QAckRef, Batch) of
|
||||
{ok, NewState} -> pop_and_send_loop(NewState, N - 1);
|
||||
{error, NewState} -> {error, NewState}
|
||||
end
|
||||
end.
|
||||
|
||||
%% Assert non-empty batch because we have a is_empty check earlier.
|
||||
do_send(#{inflight := Inflight,
|
||||
connect_module := Module,
|
||||
connection := Connection,
|
||||
mountpoint := Mountpoint,
|
||||
if_record_metrics := IfRecordMetrics} = State, QAckRef, [_ | _] = Batch) ->
|
||||
ExportMsg = fun(Message) ->
|
||||
bridges_metrics_inc(IfRecordMetrics, 'bridge.mqtt.message_sent'),
|
||||
emqx_bridge_msg:to_export(Module, Mountpoint, Message)
|
||||
end,
|
||||
case Module:send(Connection, [ExportMsg(M) || M <- Batch]) of
|
||||
{ok, Refs} ->
|
||||
{ok, State#{inflight := Inflight ++ [#{q_ack_ref => QAckRef,
|
||||
send_ack_ref => map_set(Refs),
|
||||
batch => Batch}]}};
|
||||
{error, Reason} ->
|
||||
?LOG(info, "mqtt_bridge_produce_failed ~p", [Reason]),
|
||||
{error, State}
|
||||
end.
|
||||
|
||||
%% map as set, ack-reference -> 1
|
||||
map_set(Ref) when is_reference(Ref) ->
|
||||
%% QoS-0 or RPC call returns a reference
|
||||
map_set([Ref]);
|
||||
map_set(List) ->
|
||||
map_set(List, #{}).
|
||||
|
||||
map_set([], Set) -> Set;
|
||||
map_set([H | T], Set) -> map_set(T, Set#{H => 1}).
|
||||
|
||||
handle_batch_ack(#{inflight := Inflight0, replayq := Q} = State, Ref) ->
|
||||
Inflight1 = do_ack(Inflight0, Ref),
|
||||
Inflight = drop_acked_batches(Q, Inflight1),
|
||||
State#{inflight := Inflight}.
|
||||
|
||||
do_ack([], Ref) ->
|
||||
?LOG(debug, "stale_batch_ack_reference ~p", [Ref]),
|
||||
[];
|
||||
do_ack([#{send_ack_ref := Refs} = First | Rest], Ref) ->
|
||||
case maps:is_key(Ref, Refs) of
|
||||
true ->
|
||||
NewRefs = maps:without([Ref], Refs),
|
||||
[First#{send_ack_ref := NewRefs} | Rest];
|
||||
false ->
|
||||
[First | do_ack(Rest, Ref)]
|
||||
end.
|
||||
|
||||
%% Drop the consecutive header of the inflight list having empty send_ack_ref
|
||||
drop_acked_batches(_Q, []) ->
|
||||
?tp(debug, inflight_drained, #{}),
|
||||
[];
|
||||
drop_acked_batches(Q, [#{send_ack_ref := Refs,
|
||||
q_ack_ref := QAckRef} | Rest] = All) ->
|
||||
case maps:size(Refs) of
|
||||
0 ->
|
||||
%% all messages are acked by bridge target
|
||||
%% now it's safe to ack replayq (delete from disk)
|
||||
ok = replayq:ack(Q, QAckRef),
|
||||
%% continue to check more sent batches
|
||||
drop_acked_batches(Q, Rest);
|
||||
_ ->
|
||||
%% the head (oldest) inflight batch is not acked, keep waiting
|
||||
All
|
||||
end.
|
||||
|
||||
subscribe_local_topics(Topics, Name) ->
|
||||
lists:foreach(fun(Topic) -> subscribe_local_topic(Topic, Name) end, Topics).
|
||||
|
||||
subscribe_local_topic(Topic, Name) ->
|
||||
do_subscribe(Topic, Name).
|
||||
|
||||
topic(T) -> iolist_to_binary(T).
|
||||
|
||||
validate(RawTopic) ->
|
||||
Topic = topic(RawTopic),
|
||||
try emqx_topic:validate(Topic) of
|
||||
_Success -> Topic
|
||||
catch
|
||||
error:Reason ->
|
||||
error({bad_topic, Topic, Reason})
|
||||
end.
|
||||
|
||||
do_subscribe(RawTopic, Name) ->
|
||||
TopicFilter = validate(RawTopic),
|
||||
{Topic, SubOpts} = emqx_topic:parse(TopicFilter, #{qos => ?QOS_1}),
|
||||
emqx_broker:subscribe(Topic, Name, SubOpts).
|
||||
|
||||
do_unsubscribe(RawTopic) ->
|
||||
TopicFilter = validate(RawTopic),
|
||||
{Topic, _SubOpts} = emqx_topic:parse(TopicFilter),
|
||||
emqx_broker:unsubscribe(Topic).
|
||||
|
||||
disconnect(#{connection := Conn,
|
||||
connect_module := Module
|
||||
} = State) when Conn =/= undefined ->
|
||||
Module:stop(Conn),
|
||||
State0 = State#{connection => undefined},
|
||||
eval_bridge_handler(State0, disconnected);
|
||||
disconnect(State) ->
|
||||
eval_bridge_handler(State, disconnected).
|
||||
|
||||
%% Called only when replayq needs to dump it to disk.
|
||||
msg_marshaller(Bin) when is_binary(Bin) -> emqx_bridge_msg:from_binary(Bin);
|
||||
msg_marshaller(Msg) -> emqx_bridge_msg:to_binary(Msg).
|
||||
|
||||
format_mountpoint(undefined) ->
|
||||
undefined;
|
||||
format_mountpoint(Prefix) ->
|
||||
binary:replace(iolist_to_binary(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)).
|
||||
|
||||
name(Id) -> list_to_atom(lists:concat([?MODULE, "_", Id])).
|
||||
|
||||
id(Pid) when is_pid(Pid) -> Pid;
|
||||
id(Name) -> name(Name).
|
||||
|
||||
register_metrics() ->
|
||||
lists:foreach(fun emqx_metrics:ensure/1,
|
||||
['bridge.mqtt.message_sent',
|
||||
'bridge.mqtt.message_received'
|
||||
]).
|
||||
|
||||
bridges_metrics_inc(true, Metric) ->
|
||||
emqx_metrics:inc(Metric);
|
||||
bridges_metrics_inc(_IsRecordMetric, _Metric) ->
|
||||
ok.
|
||||
|
||||
bridges_metrics_inc(true, Metric, Value) ->
|
||||
emqx_metrics:inc(Metric, Value);
|
||||
bridges_metrics_inc(_IsRecordMetric, _Metric, _Value) ->
|
||||
ok.
|
||||
47
apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl
Normal file
47
apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl
Normal file
@ -0,0 +1,47 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-module(emqx_bridge_mqtt_tests).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-include_lib("emqx/include/emqx_mqtt.hrl").
|
||||
|
||||
send_and_ack_test() ->
|
||||
%% delegate from gen_rpc to rpc for unit test
|
||||
meck:new(emqtt, [passthrough, no_history]),
|
||||
meck:expect(emqtt, start_link, 1,
|
||||
fun(_) ->
|
||||
{ok, spawn_link(fun() -> ok end)}
|
||||
end),
|
||||
meck:expect(emqtt, connect, 1, {ok, dummy}),
|
||||
meck:expect(emqtt, stop, 1,
|
||||
fun(Pid) -> Pid ! stop end),
|
||||
meck:expect(emqtt, publish, 2,
|
||||
fun(Client, Msg) ->
|
||||
Client ! {publish, Msg},
|
||||
{ok, Msg} %% as packet id
|
||||
end),
|
||||
try
|
||||
Max = 1,
|
||||
Batch = lists:seq(1, Max),
|
||||
{ok, Conn} = emqx_bridge_mqtt:start(#{address => "127.0.0.1:1883"}),
|
||||
% %% return last packet id as batch reference
|
||||
{ok, _AckRef} = emqx_bridge_mqtt:send(Conn, Batch),
|
||||
|
||||
ok = emqx_bridge_mqtt:stop(Conn)
|
||||
after
|
||||
meck:unload(emqtt)
|
||||
end.
|
||||
42
apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl
Normal file
42
apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl
Normal file
@ -0,0 +1,42 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-module(emqx_bridge_rpc_tests).
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
send_and_ack_test() ->
|
||||
%% delegate from emqx_rpc to rpc for unit test
|
||||
meck:new(emqx_rpc, [passthrough, no_history]),
|
||||
meck:expect(emqx_rpc, call, 4,
|
||||
fun(Node, Module, Fun, Args) ->
|
||||
rpc:call(Node, Module, Fun, Args)
|
||||
end),
|
||||
meck:expect(emqx_rpc, cast, 4,
|
||||
fun(Node, Module, Fun, Args) ->
|
||||
rpc:cast(Node, Module, Fun, Args)
|
||||
end),
|
||||
meck:new(emqx_bridge_worker, [passthrough, no_history]),
|
||||
try
|
||||
{ok, #{client_pid := Pid, address := Node}} = emqx_bridge_rpc:start(#{address => node()}),
|
||||
{ok, Ref} = emqx_bridge_rpc:send(#{address => Node}, []),
|
||||
receive
|
||||
{batch_ack, Ref} ->
|
||||
ok
|
||||
end,
|
||||
ok = emqx_bridge_rpc:stop( #{client_pid => Pid})
|
||||
after
|
||||
meck:unload(emqx_rpc)
|
||||
end.
|
||||
41
apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl
Normal file
41
apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl
Normal file
@ -0,0 +1,41 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-module(emqx_bridge_stub_conn).
|
||||
|
||||
-behaviour(emqx_bridge_connect).
|
||||
|
||||
%% behaviour callbacks
|
||||
-export([ start/1
|
||||
, send/2
|
||||
, stop/1
|
||||
]).
|
||||
|
||||
-type ack_ref() :: emqx_bridge_worker:ack_ref().
|
||||
-type batch() :: emqx_bridge_worker:batch().
|
||||
|
||||
start(#{client_pid := Pid} = Cfg) ->
|
||||
Pid ! {self(), ?MODULE, ready},
|
||||
{ok, Cfg}.
|
||||
|
||||
stop(_) -> ok.
|
||||
|
||||
%% @doc Callback for `emqx_bridge_connect' behaviour
|
||||
-spec send(_, batch()) -> {ok, ack_ref()} | {error, any()}.
|
||||
send(#{client_pid := Pid}, Batch) ->
|
||||
Ref = make_ref(),
|
||||
Pid ! {stub_message, self(), Ref, Batch},
|
||||
{ok, Ref}.
|
||||
343
apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl
Normal file
343
apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl
Normal file
@ -0,0 +1,343 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-module(emqx_bridge_worker_SUITE).
|
||||
|
||||
-compile(export_all).
|
||||
-compile(nowarn_export_all).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-include_lib("common_test/include/ct.hrl").
|
||||
-include_lib("emqx/include/emqx_mqtt.hrl").
|
||||
-include_lib("emqx/include/emqx.hrl").
|
||||
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
|
||||
|
||||
-define(wait(For, Timeout), emqx_ct_helpers:wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)).
|
||||
|
||||
-define(SNK_WAIT(WHAT), ?assertMatch({ok, _}, ?block_until(#{?snk_kind := WHAT}, 2000, 1000))).
|
||||
|
||||
receive_messages(Count) ->
|
||||
receive_messages(Count, []).
|
||||
|
||||
receive_messages(0, Msgs) ->
|
||||
Msgs;
|
||||
receive_messages(Count, Msgs) ->
|
||||
receive
|
||||
{publish, Msg} ->
|
||||
receive_messages(Count-1, [Msg|Msgs]);
|
||||
_Other ->
|
||||
receive_messages(Count, Msgs)
|
||||
after 1000 ->
|
||||
Msgs
|
||||
end.
|
||||
|
||||
all() ->
|
||||
lists:filtermap(
|
||||
fun({FunName, _Arity}) ->
|
||||
case atom_to_list(FunName) of
|
||||
"t_" ++ _ -> {true, FunName};
|
||||
_ -> false
|
||||
end
|
||||
end,
|
||||
?MODULE:module_info(exports)).
|
||||
|
||||
init_per_suite(Config) ->
|
||||
case node() of
|
||||
nonode@nohost -> net_kernel:start(['emqx@127.0.0.1', longnames]);
|
||||
_ -> ok
|
||||
end,
|
||||
ok = application:set_env(gen_rpc, tcp_client_num, 1),
|
||||
emqx_ct_helpers:start_apps([emqx_modules, emqx_bridge_mqtt]),
|
||||
emqx_logger:set_log_level(error),
|
||||
[{log_level, error} | Config].
|
||||
|
||||
end_per_suite(_Config) ->
|
||||
emqx_ct_helpers:stop_apps([emqx_bridge_mqtt, emqx_modules]).
|
||||
|
||||
init_per_testcase(_TestCase, Config) ->
|
||||
ok = snabbkaffe:start_trace(),
|
||||
Config.
|
||||
|
||||
end_per_testcase(_TestCase, _Config) ->
|
||||
ok = snabbkaffe:stop().
|
||||
|
||||
t_mngr(Config) when is_list(Config) ->
|
||||
Subs = [{<<"a">>, 1}, {<<"b">>, 2}],
|
||||
Cfg = #{address => node(),
|
||||
forwards => [<<"mngr">>],
|
||||
connect_module => emqx_bridge_rpc,
|
||||
mountpoint => <<"forwarded">>,
|
||||
subscriptions => Subs,
|
||||
start_type => auto},
|
||||
Name = ?FUNCTION_NAME,
|
||||
{ok, Pid} = emqx_bridge_worker:start_link(Name, Cfg),
|
||||
try
|
||||
?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)),
|
||||
?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr")),
|
||||
?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr2")),
|
||||
?assertEqual([<<"mngr">>, <<"mngr2">>], emqx_bridge_worker:get_forwards(Pid)),
|
||||
?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr2")),
|
||||
?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr3")),
|
||||
?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Pid)),
|
||||
?assertEqual({error, no_remote_subscription_support},
|
||||
emqx_bridge_worker:ensure_subscription_present(Pid, <<"t">>, 0)),
|
||||
?assertEqual({error, no_remote_subscription_support},
|
||||
emqx_bridge_worker:ensure_subscription_absent(Pid, <<"t">>)),
|
||||
?assertEqual(Subs, emqx_bridge_worker:get_subscriptions(Pid))
|
||||
after
|
||||
ok = emqx_bridge_worker:stop(Pid)
|
||||
end.
|
||||
|
||||
%% A loopback RPC to local node
|
||||
t_rpc(Config) when is_list(Config) ->
|
||||
Cfg = #{address => node(),
|
||||
forwards => [<<"t_rpc/#">>],
|
||||
connect_module => emqx_bridge_rpc,
|
||||
forward_mountpoint => <<"forwarded">>,
|
||||
start_type => auto},
|
||||
{ok, Pid} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg),
|
||||
ClientId = <<"ClientId">>,
|
||||
try
|
||||
{ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]),
|
||||
{ok, _Props} = emqtt:connect(ConnPid),
|
||||
{ok, _Props, [1]} = emqtt:subscribe(ConnPid, {<<"forwarded/t_rpc/one">>, ?QOS_1}),
|
||||
timer:sleep(100),
|
||||
{ok, _PacketId} = emqtt:publish(ConnPid, <<"t_rpc/one">>, <<"hello">>, ?QOS_1),
|
||||
timer:sleep(100),
|
||||
?assertEqual(1, length(receive_messages(1))),
|
||||
emqtt:disconnect(ConnPid)
|
||||
after
|
||||
ok = emqx_bridge_worker:stop(Pid)
|
||||
end.
|
||||
|
||||
%% Full data loopback flow explained:
|
||||
%% mqtt-client ----> local-broker ---(local-subscription)--->
|
||||
%% bridge(export) --- (mqtt-connection)--> local-broker ---(remote-subscription) -->
|
||||
%% bridge(import) --> mqtt-client
|
||||
t_mqtt(Config) when is_list(Config) ->
|
||||
SendToTopic = <<"t_mqtt/one">>,
|
||||
SendToTopic2 = <<"t_mqtt/two">>,
|
||||
SendToTopic3 = <<"t_mqtt/three">>,
|
||||
Mountpoint = <<"forwarded/${node}/">>,
|
||||
Cfg = #{address => "127.0.0.1:1883",
|
||||
forwards => [SendToTopic],
|
||||
connect_module => emqx_bridge_mqtt,
|
||||
forward_mountpoint => Mountpoint,
|
||||
username => "user",
|
||||
clean_start => true,
|
||||
clientid => "bridge_aws",
|
||||
keepalive => 60000,
|
||||
password => "passwd",
|
||||
proto_ver => mqttv4,
|
||||
queue => #{replayq_dir => "data/t_mqtt/",
|
||||
replayq_seg_bytes => 10000,
|
||||
batch_bytes_limit => 1000,
|
||||
batch_count_limit => 10
|
||||
},
|
||||
reconnect_delay_ms => 1000,
|
||||
ssl => false,
|
||||
%% Consume back to forwarded message for verification
|
||||
%% NOTE: this is a indefenite loopback without mocking emqx_bridge_worker:import_batch/1
|
||||
subscriptions => [{SendToTopic2, _QoS = 1}],
|
||||
receive_mountpoint => <<"receive/aws/">>,
|
||||
start_type => auto},
|
||||
{ok, Pid} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg),
|
||||
ClientId = <<"client-1">>,
|
||||
try
|
||||
?assertEqual([{SendToTopic2, 1}], emqx_bridge_worker:get_subscriptions(Pid)),
|
||||
ok = emqx_bridge_worker:ensure_subscription_present(Pid, SendToTopic3, _QoS = 1),
|
||||
?assertEqual([{SendToTopic3, 1},{SendToTopic2, 1}],
|
||||
emqx_bridge_worker:get_subscriptions(Pid)),
|
||||
{ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]),
|
||||
{ok, _Props} = emqtt:connect(ConnPid),
|
||||
emqtt:subscribe(ConnPid, <<"forwarded/+/t_mqtt/one">>, 1),
|
||||
%% message from a different client, to avoid getting terminated by no-local
|
||||
Max = 10,
|
||||
Msgs = lists:seq(1, Max),
|
||||
lists:foreach(fun(I) ->
|
||||
{ok, _PacketId} = emqtt:publish(ConnPid, SendToTopic, integer_to_binary(I), ?QOS_1)
|
||||
end, Msgs),
|
||||
?assertEqual(10, length(receive_messages(200))),
|
||||
|
||||
emqtt:subscribe(ConnPid, <<"receive/aws/t_mqtt/two">>, 1),
|
||||
%% message from a different client, to avoid getting terminated by no-local
|
||||
Max = 10,
|
||||
Msgs = lists:seq(1, Max),
|
||||
lists:foreach(fun(I) ->
|
||||
{ok, _PacketId} = emqtt:publish(ConnPid, SendToTopic2, integer_to_binary(I), ?QOS_1)
|
||||
end, Msgs),
|
||||
?assertEqual(10, length(receive_messages(200))),
|
||||
|
||||
emqtt:disconnect(ConnPid)
|
||||
after
|
||||
ok = emqx_bridge_worker:stop(Pid)
|
||||
end.
|
||||
|
||||
t_stub_normal(Config) when is_list(Config) ->
|
||||
Cfg = #{forwards => [<<"t_stub_normal/#">>],
|
||||
connect_module => emqx_bridge_stub_conn,
|
||||
forward_mountpoint => <<"forwarded">>,
|
||||
start_type => auto,
|
||||
client_pid => self()
|
||||
},
|
||||
{ok, Pid} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg),
|
||||
receive
|
||||
{Pid, emqx_bridge_stub_conn, ready} -> ok
|
||||
after
|
||||
5000 ->
|
||||
error(timeout)
|
||||
end,
|
||||
ClientId = <<"ClientId">>,
|
||||
try
|
||||
{ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]),
|
||||
{ok, _} = emqtt:connect(ConnPid),
|
||||
{ok, _PacketId} = emqtt:publish(ConnPid, <<"t_stub_normal/one">>, <<"hello">>, ?QOS_1),
|
||||
receive
|
||||
{stub_message, WorkerPid, BatchRef, _Batch} ->
|
||||
WorkerPid ! {batch_ack, BatchRef},
|
||||
ok
|
||||
after
|
||||
5000 ->
|
||||
error(timeout)
|
||||
end,
|
||||
?SNK_WAIT(inflight_drained),
|
||||
?SNK_WAIT(replayq_drained),
|
||||
emqtt:disconnect(ConnPid)
|
||||
after
|
||||
ok = emqx_bridge_worker:stop(Pid)
|
||||
end.
|
||||
|
||||
t_stub_overflow(Config) when is_list(Config) ->
|
||||
Topic = <<"t_stub_overflow/one">>,
|
||||
MaxInflight = 20,
|
||||
Cfg = #{forwards => [Topic],
|
||||
connect_module => emqx_bridge_stub_conn,
|
||||
forward_mountpoint => <<"forwarded">>,
|
||||
start_type => auto,
|
||||
client_pid => self(),
|
||||
max_inflight => MaxInflight
|
||||
},
|
||||
{ok, Worker} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg),
|
||||
ClientId = <<"ClientId">>,
|
||||
try
|
||||
{ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]),
|
||||
{ok, _} = emqtt:connect(ConnPid),
|
||||
lists:foreach(
|
||||
fun(I) ->
|
||||
Data = integer_to_binary(I),
|
||||
_ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1)
|
||||
end, lists:seq(1, MaxInflight * 2)),
|
||||
?SNK_WAIT(inflight_full),
|
||||
Acks = stub_receive(MaxInflight),
|
||||
lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, Acks),
|
||||
Acks2 = stub_receive(MaxInflight),
|
||||
lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, Acks2),
|
||||
?SNK_WAIT(inflight_drained),
|
||||
?SNK_WAIT(replayq_drained),
|
||||
emqtt:disconnect(ConnPid)
|
||||
after
|
||||
ok = emqx_bridge_worker:stop(Worker)
|
||||
end.
|
||||
|
||||
t_stub_random_order(Config) when is_list(Config) ->
|
||||
Topic = <<"t_stub_random_order/a">>,
|
||||
MaxInflight = 10,
|
||||
Cfg = #{forwards => [Topic],
|
||||
connect_module => emqx_bridge_stub_conn,
|
||||
forward_mountpoint => <<"forwarded">>,
|
||||
start_type => auto,
|
||||
client_pid => self(),
|
||||
max_inflight => MaxInflight
|
||||
},
|
||||
{ok, Worker} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg),
|
||||
ClientId = <<"ClientId">>,
|
||||
try
|
||||
{ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]),
|
||||
{ok, _} = emqtt:connect(ConnPid),
|
||||
lists:foreach(
|
||||
fun(I) ->
|
||||
Data = integer_to_binary(I),
|
||||
_ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1)
|
||||
end, lists:seq(1, MaxInflight)),
|
||||
Acks = stub_receive(MaxInflight),
|
||||
lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end,
|
||||
lists:reverse(Acks)),
|
||||
?SNK_WAIT(inflight_drained),
|
||||
?SNK_WAIT(replayq_drained),
|
||||
emqtt:disconnect(ConnPid)
|
||||
after
|
||||
ok = emqx_bridge_worker:stop(Worker)
|
||||
end.
|
||||
|
||||
t_stub_retry_inflight(Config) when is_list(Config) ->
|
||||
Topic = <<"to_stub_retry_inflight/a">>,
|
||||
MaxInflight = 10,
|
||||
Cfg = #{forwards => [Topic],
|
||||
connect_module => emqx_bridge_stub_conn,
|
||||
forward_mountpoint => <<"forwarded">>,
|
||||
reconnect_delay_ms => 10,
|
||||
start_type => auto,
|
||||
client_pid => self(),
|
||||
max_inflight => MaxInflight
|
||||
},
|
||||
{ok, Worker} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg),
|
||||
ClientId = <<"ClientId2">>,
|
||||
try
|
||||
case ?block_until(#{?snk_kind := connected, inflight := 0}, 2000, 1000) of
|
||||
{ok, #{inflight := 0}} -> ok;
|
||||
Other -> ct:fail("~p", [Other])
|
||||
end,
|
||||
{ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]),
|
||||
{ok, _} = emqtt:connect(ConnPid),
|
||||
lists:foreach(
|
||||
fun(I) ->
|
||||
Data = integer_to_binary(I),
|
||||
_ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1)
|
||||
end, lists:seq(1, MaxInflight)),
|
||||
%% receive acks but do not ack
|
||||
Acks1 = stub_receive(MaxInflight),
|
||||
?assertEqual(MaxInflight, length(Acks1)),
|
||||
%% simulate a disconnect
|
||||
Worker ! {disconnected, self(), test},
|
||||
?SNK_WAIT(disconnected),
|
||||
case ?block_until(#{?snk_kind := connected, inflight := MaxInflight}, 2000, 20) of
|
||||
{ok, _} -> ok;
|
||||
Error -> ct:fail("~p", [Error])
|
||||
end,
|
||||
%% expect worker to retry inflight, so to receive acks again
|
||||
Acks2 = stub_receive(MaxInflight),
|
||||
?assertEqual(MaxInflight, length(Acks2)),
|
||||
lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end,
|
||||
lists:reverse(Acks2)),
|
||||
?SNK_WAIT(inflight_drained),
|
||||
?SNK_WAIT(replayq_drained),
|
||||
emqtt:disconnect(ConnPid)
|
||||
after
|
||||
ok = emqx_bridge_worker:stop(Worker)
|
||||
end.
|
||||
|
||||
stub_receive(N) ->
|
||||
stub_receive(N, []).
|
||||
|
||||
stub_receive(0, Acc) -> lists:reverse(Acc);
|
||||
stub_receive(N, Acc) ->
|
||||
receive
|
||||
{stub_message, WorkerPid, BatchRef, _Batch} ->
|
||||
stub_receive(N - 1, [{WorkerPid, BatchRef} | Acc])
|
||||
after
|
||||
5000 ->
|
||||
lists:reverse(Acc)
|
||||
end.
|
||||
134
apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl
Normal file
134
apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl
Normal file
@ -0,0 +1,134 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-module(emqx_bridge_worker_tests).
|
||||
-behaviour(emqx_bridge_connect).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-include_lib("emqx/include/emqx.hrl").
|
||||
-include_lib("emqx/include/emqx_mqtt.hrl").
|
||||
|
||||
-define(BRIDGE_NAME, test).
|
||||
-define(BRIDGE_REG_NAME, emqx_bridge_worker_test).
|
||||
-define(WAIT(PATTERN, TIMEOUT),
|
||||
receive
|
||||
PATTERN ->
|
||||
ok
|
||||
after
|
||||
TIMEOUT ->
|
||||
error(timeout)
|
||||
end).
|
||||
|
||||
%% stub callbacks
|
||||
-export([start/1, send/2, stop/1]).
|
||||
|
||||
start(#{connect_result := Result, test_pid := Pid, test_ref := Ref}) ->
|
||||
case is_pid(Pid) of
|
||||
true -> Pid ! {connection_start_attempt, Ref};
|
||||
false -> ok
|
||||
end,
|
||||
Result.
|
||||
|
||||
send(SendFun, Batch) when is_function(SendFun, 2) ->
|
||||
SendFun(Batch).
|
||||
|
||||
stop(_Pid) -> ok.
|
||||
|
||||
%% bridge worker should retry connecting remote node indefinitely
|
||||
% reconnect_test() ->
|
||||
% emqx_metrics:start_link(),
|
||||
% emqx_bridge_worker:register_metrics(),
|
||||
% Ref = make_ref(),
|
||||
% Config = make_config(Ref, self(), {error, test}),
|
||||
% {ok, Pid} = emqx_bridge_worker:start_link(?BRIDGE_NAME, Config),
|
||||
% %% assert name registered
|
||||
% ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)),
|
||||
% ?WAIT({connection_start_attempt, Ref}, 1000),
|
||||
% %% expect same message again
|
||||
% ?WAIT({connection_start_attempt, Ref}, 1000),
|
||||
% ok = emqx_bridge_worker:stop(?BRIDGE_REG_NAME),
|
||||
% emqx_metrics:stop(),
|
||||
% ok.
|
||||
|
||||
%% connect first, disconnect, then connect again
|
||||
disturbance_test() ->
|
||||
emqx_metrics:start_link(),
|
||||
emqx_bridge_worker:register_metrics(),
|
||||
Ref = make_ref(),
|
||||
TestPid = self(),
|
||||
Config = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}),
|
||||
{ok, Pid} = emqx_bridge_worker:start_link(?BRIDGE_NAME, Config),
|
||||
?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)),
|
||||
?WAIT({connection_start_attempt, Ref}, 1000),
|
||||
Pid ! {disconnected, TestPid, test},
|
||||
?WAIT({connection_start_attempt, Ref}, 1000),
|
||||
emqx_metrics:stop(),
|
||||
ok = emqx_bridge_worker:stop(?BRIDGE_REG_NAME).
|
||||
|
||||
% % %% buffer should continue taking in messages when disconnected
|
||||
% buffer_when_disconnected_test_() ->
|
||||
% {timeout, 10000, fun test_buffer_when_disconnected/0}.
|
||||
|
||||
% test_buffer_when_disconnected() ->
|
||||
% Ref = make_ref(),
|
||||
% Nums = lists:seq(1, 100),
|
||||
% Sender = spawn_link(fun() -> receive {bridge, Pid} -> sender_loop(Pid, Nums, _Interval = 5) end end),
|
||||
% SenderMref = monitor(process, Sender),
|
||||
% Receiver = spawn_link(fun() -> receive {bridge, Pid} -> receiver_loop(Pid, Nums, _Interval = 1) end end),
|
||||
% ReceiverMref = monitor(process, Receiver),
|
||||
% SendFun = fun(Batch) ->
|
||||
% BatchRef = make_ref(),
|
||||
% Receiver ! {batch, BatchRef, Batch},
|
||||
% {ok, BatchRef}
|
||||
% end,
|
||||
% Config0 = make_config(Ref, false, {ok, #{client_pid => undefined}}),
|
||||
% Config = Config0#{reconnect_delay_ms => 100},
|
||||
% emqx_metrics:start_link(),
|
||||
% emqx_bridge_worker:register_metrics(),
|
||||
% {ok, Pid} = emqx_bridge_worker:start_link(?BRIDGE_NAME, Config),
|
||||
% Sender ! {bridge, Pid},
|
||||
% Receiver ! {bridge, Pid},
|
||||
% ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)),
|
||||
% Pid ! {disconnected, Ref, test},
|
||||
% ?WAIT({'DOWN', SenderMref, process, Sender, normal}, 5000),
|
||||
% ?WAIT({'DOWN', ReceiverMref, process, Receiver, normal}, 1000),
|
||||
% ok = emqx_bridge_worker:stop(?BRIDGE_REG_NAME),
|
||||
% emqx_metrics:stop().
|
||||
|
||||
manual_start_stop_test() ->
|
||||
emqx_metrics:start_link(),
|
||||
emqx_bridge_worker:register_metrics(),
|
||||
Ref = make_ref(),
|
||||
TestPid = self(),
|
||||
Config0 = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}),
|
||||
Config = Config0#{start_type := manual},
|
||||
{ok, Pid} = emqx_bridge_worker:start_link(?BRIDGE_NAME, Config),
|
||||
%% call ensure_started again should yeld the same result
|
||||
ok = emqx_bridge_worker:ensure_started(?BRIDGE_NAME),
|
||||
?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)),
|
||||
emqx_bridge_worker:ensure_stopped(unknown),
|
||||
emqx_bridge_worker:ensure_stopped(Pid),
|
||||
emqx_bridge_worker:ensure_stopped(?BRIDGE_REG_NAME),
|
||||
emqx_metrics:stop().
|
||||
|
||||
make_config(Ref, TestPid, Result) ->
|
||||
#{test_pid => TestPid,
|
||||
test_ref => Ref,
|
||||
connect_module => ?MODULE,
|
||||
reconnect_delay_ms => 50,
|
||||
connect_result => Result,
|
||||
start_type => auto
|
||||
}.
|
||||
29
apps/emqx_exhook/.gitignore
vendored
Normal file
29
apps/emqx_exhook/.gitignore
vendored
Normal file
@ -0,0 +1,29 @@
|
||||
.rebar3
|
||||
_*
|
||||
.eunit
|
||||
*.o
|
||||
*.beam
|
||||
*.plt
|
||||
*.swp
|
||||
*.swo
|
||||
.erlang.cookie
|
||||
ebin
|
||||
log
|
||||
erl_crash.dump
|
||||
.rebar
|
||||
logs
|
||||
_build
|
||||
.idea
|
||||
*.iml
|
||||
rebar3.crashdump
|
||||
*~
|
||||
rebar.lock
|
||||
data/
|
||||
*.conf.rendered
|
||||
*.pyc
|
||||
.DS_Store
|
||||
*.class
|
||||
Mnesia.nonode@nohost/
|
||||
src/emqx_exhook_pb.erl
|
||||
src/emqx_exhook_v_1_hook_provider_client.erl
|
||||
src/emqx_exhook_v_1_hook_provider_bhvr.erl
|
||||
39
apps/emqx_exhook/README.md
Normal file
39
apps/emqx_exhook/README.md
Normal file
@ -0,0 +1,39 @@
|
||||
# emqx_exhook
|
||||
|
||||
The `emqx_exhook` extremly enhance the extensibility for EMQ X. It allow using an others programming language to mount the hooks intead of erlang.
|
||||
|
||||
## Feature
|
||||
|
||||
- [x] Based on gRPC, it brings a very wide range of applicability
|
||||
- [x] Allows you to use the return value to extend emqx behavior.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
EMQ X Third-party Runtime
|
||||
+========================+ +========+==========+
|
||||
| ExHook | | | |
|
||||
| +----------------+ | gRPC | gRPC | User's |
|
||||
| | gPRC Client | ------------------> | Server | Codes |
|
||||
| +----------------+ | (HTTP/2) | | |
|
||||
| | | | |
|
||||
+========================+ +========+==========+
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### gRPC service
|
||||
|
||||
See: `priv/protos/exhook.proto`
|
||||
|
||||
### CLI
|
||||
|
||||
## Example
|
||||
|
||||
## Recommended gRPC Framework
|
||||
|
||||
See: https://hub.fastgit.org/grpc-ecosystem/awesome-grpc
|
||||
|
||||
## Thanks
|
||||
|
||||
- [grpcbox](https://hub.fastgit.org/tsloughter/grpcbox)
|
||||
116
apps/emqx_exhook/docs/design-cn.md
Normal file
116
apps/emqx_exhook/docs/design-cn.md
Normal file
@ -0,0 +1,116 @@
|
||||
# 设计
|
||||
|
||||
## 动机
|
||||
|
||||
在 EMQ X Broker v4.1-v4.2 中，我们发布了 2 个插件来扩展 emqx 的编程能力：
|
||||
|
||||
1. `emqx-extension-hook` 提供了使用 Java, Python 向 Broker 挂载钩子的功能
|
||||
2. `emqx-exproto` 提供了使用 Java，Python 编写用户自定义协议接入插件的功能
|
||||
|
||||
但在后续的支持中发现许多难以处理的问题：
|
||||
|
||||
1. 有大量的编程语言需要支持，需要编写和维护如 Go, JavaScript, Lua.. 等语言的驱动。
|
||||
2. `erlport` 使用的操作系统的管道进行通信，这让用户代码只能部署在和 emqx 同一个操作系统上。部署方式受到了极大的限制。
|
||||
3. 用户程序的启动参数直接打包到 Broker 中，导致用户开发无法实时的进行调试，单步跟踪等。
|
||||
4. `erlport` 会占用 `stdin` `stdout`。
|
||||
|
||||
因此，我们计划重构这部分的实现，其中主要的内容是：
|
||||
1. 使用 `gRPC` 替换 `erlport`。
|
||||
2. 将 `emqx-extension-hook` 重命名为 `emqx-exhook`
|
||||
|
||||
|
||||
旧版本的设计：[emqx-extension-hook design in v4.2.0](https://hub.fastgit.org/emqx/emqx-exhook/blob/v4.2.0/docs/design.md)
|
||||
|
||||
## 设计
|
||||
|
||||
架构如下：
|
||||
|
||||
```
|
||||
EMQ X
|
||||
+========================+ +========+==========+
|
||||
| ExHook | | | |
|
||||
| +----------------+ | gRPC | gRPC | User's |
|
||||
| | gRPC Client | ------------------> | Server | Codes |
|
||||
| +----------------+ | (HTTP/2) | | |
|
||||
| | | | |
|
||||
+========================+ +========+==========+
|
||||
```
|
||||
|
||||
`emqx-exhook` 通过 gRPC 的方式向用户部署的 gRPC 服务发送钩子的请求，并处理其返回的值。
|
||||
|
||||
|
||||
和 emqx 原生的钩子一致，emqx-exhook 也按照链式的方式执行：
|
||||
|
||||
<img src="https://docs.emqx.net/broker/latest/cn/advanced/assets/chain_of_responsiblity.png" style="zoom:50%;" />
|
||||
|
||||
### gRPC 服务示例
|
||||
|
||||
用户需要实现的方法，和数据类型的定义在 `priv/protos/exhook.proto` 文件中：
|
||||
|
||||
```protobuff
|
||||
syntax = "proto3";
|
||||
|
||||
package emqx.exhook.v1;
|
||||
|
||||
service HookProvider {
|
||||
|
||||
rpc OnProviderLoaded(ProviderLoadedRequest) returns (LoadedResponse) {};
|
||||
|
||||
rpc OnProviderUnloaded(ProviderUnloadedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnClientConnect(ClientConnectRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnClientConnack(ClientConnackRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnClientConnected(ClientConnectedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnClientDisconnected(ClientDisconnectedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnClientAuthenticate(ClientAuthenticateRequest) returns (ValuedResponse) {};
|
||||
|
||||
rpc OnClientCheckAcl(ClientCheckAclRequest) returns (ValuedResponse) {};
|
||||
|
||||
rpc OnClientSubscribe(ClientSubscribeRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnClientUnsubscribe(ClientUnsubscribeRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionCreated(SessionCreatedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionSubscribed(SessionSubscribedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionUnsubscribed(SessionUnsubscribedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionResumed(SessionResumedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionDiscarded(SessionDiscardedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionTakeovered(SessionTakeoveredRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionTerminated(SessionTerminatedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnMessagePublish(MessagePublishRequest) returns (ValuedResponse) {};
|
||||
|
||||
rpc OnMessageDelivered(MessageDeliveredRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnMessageDropped(MessageDroppedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnMessageAcked(MessageAckedRequest) returns (EmptySuccess) {};
|
||||
}
|
||||
```
|
||||
|
||||
### 配置文件示例
|
||||
|
||||
```
|
||||
## 配置 gRPC 服务地址 (HTTP)
|
||||
##
|
||||
## s1 为服务器的名称
|
||||
exhook.server.s1.url = http://127.0.0.1:9001
|
||||
|
||||
## 配置 gRPC 服务地址 (HTTPS)
|
||||
##
|
||||
## s2 为服务器名称
|
||||
exhook.server.s2.url = https://127.0.0.1:9002
|
||||
exhook.server.s2.cacertfile = ca.pem
|
||||
exhook.server.s2.certfile = cert.pem
|
||||
exhook.server.s2.keyfile = key.pem
|
||||
```
|
||||
15
apps/emqx_exhook/etc/emqx_exhook.conf
Normal file
15
apps/emqx_exhook/etc/emqx_exhook.conf
Normal file
@ -0,0 +1,15 @@
|
||||
##====================================================================
|
||||
## EMQ X Hooks
|
||||
##====================================================================
|
||||
|
||||
##--------------------------------------------------------------------
|
||||
## Server Address
|
||||
|
||||
## The gRPC server url
|
||||
##
|
||||
## exhook.server.$name.url = url()
|
||||
exhook.server.default.url = http://127.0.0.1:9000
|
||||
|
||||
#exhook.server.default.ssl.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem
|
||||
#exhook.server.default.ssl.certfile = {{ platform_etc_dir }}/certs/cert.pem
|
||||
#exhook.server.default.ssl.keyfile = {{ platform_etc_dir }}/certs/key.pem
|
||||
44
apps/emqx_exhook/include/emqx_exhook.hrl
Normal file
44
apps/emqx_exhook/include/emqx_exhook.hrl
Normal file
@ -0,0 +1,44 @@
|
||||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-ifndef(EMQX_EXHOOK_HRL).
|
||||
-define(EMQX_EXHOOK_HRL, true).
|
||||
|
||||
-define(APP, emqx_exhook).
|
||||
|
||||
-define(ENABLED_HOOKS,
|
||||
[ {'client.connect', {emqx_exhook_handler, on_client_connect, []}}
|
||||
, {'client.connack', {emqx_exhook_handler, on_client_connack, []}}
|
||||
, {'client.connected', {emqx_exhook_handler, on_client_connected, []}}
|
||||
, {'client.disconnected', {emqx_exhook_handler, on_client_disconnected, []}}
|
||||
, {'client.authenticate', {emqx_exhook_handler, on_client_authenticate, []}}
|
||||
, {'client.check_acl', {emqx_exhook_handler, on_client_check_acl, []}}
|
||||
, {'client.subscribe', {emqx_exhook_handler, on_client_subscribe, []}}
|
||||
, {'client.unsubscribe', {emqx_exhook_handler, on_client_unsubscribe, []}}
|
||||
, {'session.created', {emqx_exhook_handler, on_session_created, []}}
|
||||
, {'session.subscribed', {emqx_exhook_handler, on_session_subscribed, []}}
|
||||
, {'session.unsubscribed',{emqx_exhook_handler, on_session_unsubscribed, []}}
|
||||
, {'session.resumed', {emqx_exhook_handler, on_session_resumed, []}}
|
||||
, {'session.discarded', {emqx_exhook_handler, on_session_discarded, []}}
|
||||
, {'session.takeovered', {emqx_exhook_handler, on_session_takeovered, []}}
|
||||
, {'session.terminated', {emqx_exhook_handler, on_session_terminated, []}}
|
||||
, {'message.publish', {emqx_exhook_handler, on_message_publish, []}}
|
||||
, {'message.delivered', {emqx_exhook_handler, on_message_delivered, []}}
|
||||
, {'message.acked', {emqx_exhook_handler, on_message_acked, []}}
|
||||
, {'message.dropped', {emqx_exhook_handler, on_message_dropped, []}}
|
||||
]).
|
||||
|
||||
-endif.
|
||||
38
apps/emqx_exhook/priv/emqx_exhook.schema
Normal file
38
apps/emqx_exhook/priv/emqx_exhook.schema
Normal file
@ -0,0 +1,38 @@
|
||||
%%-*- mode: erlang -*-
|
||||
|
||||
{mapping, "exhook.server.$name.url", "emqx_exhook.servers", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "exhook.server.$name.ssl.cacertfile", "emqx_exhook.servers", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "exhook.server.$name.ssl.certfile", "emqx_exhook.servers", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{mapping, "exhook.server.$name.ssl.keyfile", "emqx_exhook.servers", [
|
||||
{datatype, string}
|
||||
]}.
|
||||
|
||||
{translation, "emqx_exhook.servers", fun(Conf) ->
|
||||
Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end,
|
||||
ServerOptions = fun(Prefix) ->
|
||||
case http_uri:parse(cuttlefish:conf_get(Prefix ++ ".url", Conf)) of
|
||||
{ok, {http, _, Host, Port, _, _}} ->
|
||||
[{scheme, http}, {host, Host}, {port, Port}];
|
||||
{ok, {https, _, Host, Port, _, _}} ->
|
||||
[{scheme, https}, {host, Host}, {port, Port},
|
||||
{ssl_options,
|
||||
Filter([{ssl, true},
|
||||
{certfile, cuttlefish:conf_get(Prefix ++ ".ssl.certfile", Conf, undefined)},
|
||||
{keyfile, cuttlefish:conf_get(Prefix ++ ".ssl.keyfile", Conf, undefined)},
|
||||
{cacertfile, cuttlefish:conf_get(Prefix ++ ".ssl.cacertfile", Conf, undefined)}
|
||||
])}];
|
||||
_ -> error(invalid_server_options)
|
||||
end
|
||||
end,
|
||||
[{list_to_atom(Name), ServerOptions("exhook.server." ++ Name)}
|
||||
|| {["exhook", "server", Name, "url"], _} <- cuttlefish_variable:filter_by_prefix("exhook.server", Conf)]
|
||||
end}.
|
||||
407
apps/emqx_exhook/priv/protos/exhook.proto
Normal file
407
apps/emqx_exhook/priv/protos/exhook.proto
Normal file
@ -0,0 +1,407 @@
|
||||
//------------------------------------------------------------------------------
|
||||
// Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
option csharp_namespace = "Emqx.Exhook.V1";
|
||||
option go_package = "emqx.io/grpc/exhook";
|
||||
option java_multiple_files = true;
|
||||
option java_package = "io.emqx.exhook";
|
||||
option java_outer_classname = "EmqxExHookProto";
|
||||
|
||||
package emqx.exhook.v1;
|
||||
|
||||
service HookProvider {
|
||||
|
||||
rpc OnProviderLoaded(ProviderLoadedRequest) returns (LoadedResponse) {};
|
||||
|
||||
rpc OnProviderUnloaded(ProviderUnloadedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnClientConnect(ClientConnectRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnClientConnack(ClientConnackRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnClientConnected(ClientConnectedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnClientDisconnected(ClientDisconnectedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnClientAuthenticate(ClientAuthenticateRequest) returns (ValuedResponse) {};
|
||||
|
||||
rpc OnClientCheckAcl(ClientCheckAclRequest) returns (ValuedResponse) {};
|
||||
|
||||
rpc OnClientSubscribe(ClientSubscribeRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnClientUnsubscribe(ClientUnsubscribeRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionCreated(SessionCreatedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionSubscribed(SessionSubscribedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionUnsubscribed(SessionUnsubscribedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionResumed(SessionResumedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionDiscarded(SessionDiscardedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionTakeovered(SessionTakeoveredRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnSessionTerminated(SessionTerminatedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnMessagePublish(MessagePublishRequest) returns (ValuedResponse) {};
|
||||
|
||||
rpc OnMessageDelivered(MessageDeliveredRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnMessageDropped(MessageDroppedRequest) returns (EmptySuccess) {};
|
||||
|
||||
rpc OnMessageAcked(MessageAckedRequest) returns (EmptySuccess) {};
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Request & Response
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
message ProviderLoadedRequest {
|
||||
|
||||
BrokerInfo broker = 1;
|
||||
}
|
||||
|
||||
message LoadedResponse {
|
||||
|
||||
repeated HookSpec hooks = 1;
|
||||
}
|
||||
|
||||
message ProviderUnloadedRequest { }
|
||||
|
||||
message ClientConnectRequest {
|
||||
|
||||
ConnInfo conninfo = 1;
|
||||
|
||||
// MQTT CONNECT packet's properties (MQTT v5.0)
|
||||
//
|
||||
// It should be empty on MQTT v3.1.1/v3.1 or others protocol
|
||||
repeated Property props = 2;
|
||||
}
|
||||
|
||||
message ClientConnackRequest {
|
||||
|
||||
ConnInfo conninfo = 1;
|
||||
|
||||
string result_code = 2;
|
||||
|
||||
repeated Property props = 3;
|
||||
}
|
||||
|
||||
message ClientConnectedRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
}
|
||||
|
||||
message ClientDisconnectedRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
|
||||
string reason = 2;
|
||||
}
|
||||
|
||||
message ClientAuthenticateRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
|
||||
bool result = 2;
|
||||
}
|
||||
|
||||
message ClientCheckAclRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
|
||||
enum AclReqType {
|
||||
|
||||
PUBLISH = 0;
|
||||
|
||||
SUBSCRIBE = 1;
|
||||
}
|
||||
|
||||
AclReqType type = 2;
|
||||
|
||||
string topic = 3;
|
||||
|
||||
bool result = 4;
|
||||
}
|
||||
|
||||
message ClientSubscribeRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
|
||||
repeated Property props = 2;
|
||||
|
||||
repeated TopicFilter topic_filters = 3;
|
||||
}
|
||||
|
||||
message ClientUnsubscribeRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
|
||||
repeated Property props = 2;
|
||||
|
||||
repeated TopicFilter topic_filters = 3;
|
||||
}
|
||||
|
||||
message SessionCreatedRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
}
|
||||
|
||||
message SessionSubscribedRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
|
||||
string topic = 2;
|
||||
|
||||
SubOpts subopts = 3;
|
||||
}
|
||||
|
||||
message SessionUnsubscribedRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
|
||||
string topic = 2;
|
||||
}
|
||||
|
||||
message SessionResumedRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
}
|
||||
|
||||
message SessionDiscardedRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
}
|
||||
|
||||
message SessionTakeoveredRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
}
|
||||
|
||||
message SessionTerminatedRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
|
||||
string reason = 2;
|
||||
}
|
||||
|
||||
message MessagePublishRequest {
|
||||
|
||||
Message message = 1;
|
||||
}
|
||||
|
||||
message MessageDeliveredRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
|
||||
Message message = 2;
|
||||
}
|
||||
|
||||
message MessageDroppedRequest {
|
||||
|
||||
Message message = 1;
|
||||
|
||||
string reason = 2;
|
||||
}
|
||||
|
||||
message MessageAckedRequest {
|
||||
|
||||
ClientInfo clientinfo = 1;
|
||||
|
||||
Message message = 2;
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Basic data types
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
message EmptySuccess { }
|
||||
|
||||
message ValuedResponse {
|
||||
|
||||
// The responsed value type
|
||||
// - ignore: Ignore the responsed value
|
||||
// - contiune: Use the responsed value and execute the next hook
|
||||
// - stop_and_return: Use the responsed value and stop the chain executing
|
||||
enum ResponsedType {
|
||||
|
||||
IGNORE = 0;
|
||||
|
||||
CONTINUE = 1;
|
||||
|
||||
STOP_AND_RETURN = 2;
|
||||
}
|
||||
|
||||
ResponsedType type = 1;
|
||||
|
||||
oneof value {
|
||||
|
||||
// Boolean result, used on the 'client.authenticate', 'client.check_acl' hooks
|
||||
bool bool_result = 3;
|
||||
|
||||
// Message result, used on the 'message.*' hooks
|
||||
Message message = 4;
|
||||
}
|
||||
}
|
||||
|
||||
message BrokerInfo {
|
||||
|
||||
string version = 1;
|
||||
|
||||
string sysdescr = 2;
|
||||
|
||||
string uptime = 3;
|
||||
|
||||
string datetime = 4;
|
||||
}
|
||||
|
||||
message HookSpec {
|
||||
|
||||
// The registered hooks name
|
||||
//
|
||||
// Available value:
|
||||
// "client.connect", "client.connack"
|
||||
// "client.connected", "client.disconnected"
|
||||
// "client.authenticate", "client.check_acl"
|
||||
// "client.subscribe", "client.unsubscribe"
|
||||
//
|
||||
// "session.created", "session.subscribed"
|
||||
// "session.unsubscribed", "session.resumed"
|
||||
// "session.discarded", "session.takeovered"
|
||||
// "session.terminated"
|
||||
//
|
||||
// "message.publish", "message.delivered"
|
||||
// "message.acked", "message.dropped"
|
||||
string name = 1;
|
||||
|
||||
// The topic filters for message hooks
|
||||
repeated string topics = 2;
|
||||
}
|
||||
|
||||
message ConnInfo {
|
||||
|
||||
string node = 1;
|
||||
|
||||
string clientid = 2;
|
||||
|
||||
string username = 3;
|
||||
|
||||
string peerhost = 4;
|
||||
|
||||
uint32 sockport = 5;
|
||||
|
||||
string proto_name = 6;
|
||||
|
||||
string proto_ver = 7;
|
||||
|
||||
uint32 keepalive = 8;
|
||||
}
|
||||
|
||||
message ClientInfo {
|
||||
|
||||
string node = 1;
|
||||
|
||||
string clientid = 2;
|
||||
|
||||
string username = 3;
|
||||
|
||||
string password = 4;
|
||||
|
||||
string peerhost = 5;
|
||||
|
||||
uint32 sockport = 6;
|
||||
|
||||
string protocol = 7;
|
||||
|
||||
string mountpoint = 8;
|
||||
|
||||
bool is_superuser = 9;
|
||||
|
||||
bool anonymous = 10;
|
||||
|
||||
// common name of client TLS cert
|
||||
string cn = 11;
|
||||
|
||||
// subject of client TLS cert
|
||||
string dn = 12;
|
||||
}
|
||||
|
||||
message Message {
|
||||
|
||||
string node = 1;
|
||||
|
||||
string id = 2;
|
||||
|
||||
uint32 qos = 3;
|
||||
|
||||
string from = 4;
|
||||
|
||||
string topic = 5;
|
||||
|
||||
bytes payload = 6;
|
||||
|
||||
uint64 timestamp = 7;
|
||||
}
|
||||
|
||||
message Property {
|
||||
|
||||
string name = 1;
|
||||
|
||||
string value = 2;
|
||||
}
|
||||
|
||||
message TopicFilter {
|
||||
|
||||
string name = 1;
|
||||
|
||||
uint32 qos = 2;
|
||||
}
|
||||
|
||||
message SubOpts {
|
||||
|
||||
// The QoS level
|
||||
uint32 qos = 1;
|
||||
|
||||
// The group name for shared subscription
|
||||
string share = 2;
|
||||
|
||||
// The Retain Handling option (MQTT v5.0)
|
||||
//
|
||||
// 0 = Send retained messages at the time of the subscribe
|
||||
// 1 = Send retained messages at subscribe only if the subscription does
|
||||
// not currently exist
|
||||
// 2 = Do not send retained messages at the time of the subscribe
|
||||
uint32 rh = 3;
|
||||
|
||||
// The Retain as Published option (MQTT v5.0)
|
||||
//
|
||||
// If 1, Application Messages forwarded using this subscription keep the
|
||||
// RETAIN flag they were published with.
|
||||
// If 0, Application Messages forwarded using this subscription have the
|
||||
// RETAIN flag set to 0.
|
||||
// Retained messages sent when the subscription is established have the RETAIN flag set to 1.
|
||||
uint32 rap = 4;
|
||||
|
||||
// The No Local option (MQTT v5.0)
|
||||
//
|
||||
// If the value is 1, Application Messages MUST NOT be forwarded to a
|
||||
// connection with a ClientID equal to the ClientID of the publishing
|
||||
uint32 nl = 5;
|
||||
}
|
||||
49
apps/emqx_exhook/rebar.config
Normal file
49
apps/emqx_exhook/rebar.config
Normal file
@ -0,0 +1,49 @@
|
||||
%%-*- mode: erlang -*-
|
||||
{plugins,
|
||||
[rebar3_proper,
|
||||
{grpc_plugin, {git, "https://hub.fastgit.org/HJianBo/grpc_plugin", {tag, "v0.10.2"}}}
|
||||
]}.
|
||||
|
||||
{deps,
|
||||
[{grpc, {git, "https://hub.fastgit.org/emqx/grpc-erl", {tag, "0.6.2"}}}
|
||||
]}.
|
||||
|
||||
{grpc,
|
||||
[{protos, ["priv/protos"]},
|
||||
{gpb_opts, [{module_name_prefix, "emqx_"},
|
||||
{module_name_suffix, "_pb"}]}
|
||||
]}.
|
||||
|
||||
{provider_hooks,
|
||||
[{pre, [{compile, {grpc, gen}},
|
||||
{clean, {grpc, clean}}]}
|
||||
]}.
|
||||
|
||||
{edoc_opts, [{preprocess, true}]}.
|
||||
|
||||
{erl_opts, [warn_unused_vars,
|
||||
warn_shadow_vars,
|
||||
warn_unused_import,
|
||||
warn_obsolete_guard,
|
||||
debug_info,
|
||||
{parse_transform}]}.
|
||||
|
||||
{xref_checks, [undefined_function_calls, undefined_functions,
|
||||
locals_not_used, deprecated_function_calls,
|
||||
warnings_as_errors, deprecated_functions]}.
|
||||
{xref_ignores, [emqx_exhook_pb]}.
|
||||
|
||||
{cover_enabled, true}.
|
||||
{cover_opts, [verbose]}.
|
||||
{cover_export_enabled, true}.
|
||||
{cover_excl_mods, [emqx_exhook_pb,
|
||||
emqx_exhook_v_1_hook_provider_bhvr,
|
||||
emqx_exhook_v_1_hook_provider_client]}.
|
||||
|
||||
{profiles,
|
||||
[{test,
|
||||
[{deps,
|
||||
[{emqx_ct_helper, {git, "https://hub.fastgit.org/emqx/emqx-ct-helpers", {tag, "v1.3.1"}}}
|
||||
]}
|
||||
]}
|
||||
]}.
|
||||
12
apps/emqx_exhook/src/emqx_exhook.app.src
Normal file
12
apps/emqx_exhook/src/emqx_exhook.app.src
Normal file
@ -0,0 +1,12 @@
|
||||
{application, emqx_exhook,
|
||||
[{description, "EMQ X Extension for Hook"},
|
||||
{vsn, "4.3.0"},
|
||||
{modules, []},
|
||||
{registered, []},
|
||||
{mod, {emqx_exhook_app, []}},
|
||||
{applications, [kernel,stdlib,grpc]},
|
||||
{env,[]},
|
||||
{licenses, ["Apache-2.0"]},
|
||||
{maintainers, ["EMQ X Team <contact@emqx.io>"]},
|
||||
{links, [{"Homepage", "https://emqx.io/"}]}
|
||||
]}.
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user