test/dgiot
1
0
Fork 0
mirror of https://gitee.com/dgiiot/dgiot.git synced 2024-11-29 10:48:04 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: v4.4.11

Browse Source
This commit is contained in:
jhonliu 2022-12-29 18:13:09 +08:00
parent 3cf20f0c98
commit 32e6dee548
376 changed files with 17953 additions and 4687 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.ci/acl_migration_test/build.sh Normal file
View File

@ -0,0 +1,14 @@
#!/bin/bash
set -xe
cd "$EMQX_PATH"
rm -rf _build _upgrade_base
mkdir _upgrade_base
pushd _upgrade_base
wget "https://s3-us-west-2.amazonaws.com/packages.emqx/emqx-ce/v${EMQX_BASE}/emqx-ubuntu20.04-${EMQX_BASE}-amd64.zip"
popd
make emqx-zip

15
.ci/acl_migration_test/prepare.sh Normal file
View File

@ -0,0 +1,15 @@
#!/bin/bash
set -xe
mkdir -p "$TEST_PATH"
cd "$TEST_PATH"
cp ../"$EMQX_PATH"/_upgrade_base/*.zip ./
unzip ./*.zip
cp ../"$EMQX_PATH"/_packages/emqx/*.zip ./emqx/releases/
git clone --depth 1 https://github.com/terry-xiaoyu/one_more_emqx.git
./one_more_emqx/one_more_emqx.sh emqx2

17
.ci/acl_migration_test/suite.sh Normal file
View File

@ -0,0 +1,17 @@
#!/bin/bash
set -xe
export EMQX_PATH="$1"
export EMQX_BASE="$2"
export TEST_PATH="emqx_test"
./build.sh
VERSION=$("$EMQX_PATH"/pkg-vsn.sh)
export VERSION
./prepare.sh
./test.sh

121
.ci/acl_migration_test/test.sh Normal file
View File

@ -0,0 +1,121 @@
#!/bin/bash
set -e
EMQX_ENDPOINT="http://localhost:8081/api/v4/acl"
EMQX2_ENDPOINT="http://localhost:8917/api/v4/acl"
function run() {
emqx="$1"
shift
echo "[$emqx]" "$@"
pushd "$TEST_PATH/$emqx"
"$@"
popd
}
function post_rule() {
endpoint="$1"
rule="$2"
echo -n "->($endpoint) "
curl -s -u admin:public -X POST "$endpoint" -d "$rule"
echo
}
function verify_clientid_rule() {
endpoint="$1"
id="$2"
echo -n "<-($endpoint) "
curl -s -u admin:public "$endpoint/clientid/$id" | grep "$id" || (echo "verify rule for client $id failed" && return 1)
}
# Run nodes
run emqx ./bin/emqx start
run emqx2 ./bin/emqx start
run emqx ./bin/emqx_ctl plugins load emqx_auth_mnesia
run emqx2 ./bin/emqx_ctl plugins load emqx_auth_mnesia
run emqx2 ./bin/emqx_ctl cluster join 'emqx@127.0.0.1'
# Add ACL rule to unupgraded EMQX nodes
post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT1_A","topic": "t", "action": "pub", "access": "allow"}'
post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT1_B","topic": "t", "action": "pub", "access": "allow"}'
# Upgrade emqx2 node
run emqx2 ./bin/emqx install "$VERSION"
sleep 60
# Verify upgrade blocked
run emqx2 ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep false || (echo "emqx2 shouldn't have migrated" && exit 1)
# Verify old rules on both nodes
verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_A'
verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_A'
verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_B'
verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_B'
# Add ACL on OLD and NEW node, verify on all nodes
post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT2_A","topic": "t", "action": "pub", "access": "allow"}'
post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT2_B","topic": "t", "action": "pub", "access": "allow"}'
verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_A'
verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_A'
verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_B'
verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_B'
# Upgrade emqx node
run emqx ./bin/emqx install "$VERSION"
# Wait for upgrade
sleep 60
# Verify if upgrade occured
run emqx ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep true || (echo "emqx should have migrated" && exit 1)
run emqx2 ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep true || (echo "emqx2 should have migrated" && exit 1)
# Verify rules are kept
verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_A'
verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_A'
verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_B'
verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_B'
verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_A'
verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_A'
verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_B'
verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_B'
# Add ACL on OLD and NEW node, verify on all nodes
post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT3_A","topic": "t", "action": "pub", "access": "allow"}'
post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT3_B","topic": "t", "action": "pub", "access": "allow"}'
verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT3_A'
verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT3_A'
verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT3_B'
verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT3_B'
# Stop nodes
run emqx ./bin/emqx stop
run emqx2 ./bin/emqx stop
echo "Success!"

16
.ci/build_packages/Dockerfile
View File

@ -1,16 +0,0 @@
ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
FROM ${BUILD_FROM}
ARG EMQX_NAME=emqx
COPY . /emqx
WORKDIR /emqx
RUN rm -rf _build/${EMQX_NAME}/lib _build/${EMQX_NAME}-pkg/lib
RUN make ${EMQX_NAME}-zip || cat rebar3.crashdump
RUN make ${EMQX_NAME}-pkg || cat rebar3.crashdump
RUN /emqx/.ci/build_packages/tests.sh

268
.ci/build_packages/tests.sh
View File

@ -1,28 +1,56 @@
#!/bin/bash
#!/usr/bin/env bash
## This script tests built package start/stop
## Accept 2 args PROFILE and PACKAGE_TYPE
set -x -e -u
if [ -z "${1:-}" ]; then
echo "Usage $0 <PROFILE> e.g. emqx, emqx-edge"
exit 1
fi
if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then
echo "Usage $0 <PACKAGE_NAME> zip|pkg"
exit 1
fi
PROFILE="${1}"
PACKAGE_TYPE="${2}"
export CODE_PATH=${CODE_PATH:-"/emqx"}
export EMQX_NAME=${EMQX_NAME:-"emqx"}
export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}"
export PACKAGE_PATH="${CODE_PATH}/_packages/${PROFILE}"
export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base"
# export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1"
# export EMQX_NODE_COOKIE=$(date +%s%N)
case "$(uname -m)" in
x86_64)
ARCH='amd64'
;;
aarch64)
ARCH='arm64'
;;
arm*)
ARCH=arm
;;
esac
export ARCH
SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)"
if [ "$PACKAGE_TYPE" = 'zip' ]; then
PKG_SUFFIX="zip"
else
case "${SYSTEM:-}" in
ubuntu*|debian*|raspbian*)
PKG_SUFFIX='deb'
;;
*)
PKG_SUFFIX='rpm'
;;
esac
fi
PACKAGE_NAME="${PROFILE}-$("$CODE_PATH"/scripts/pkg-full-vsn.sh)"
OLD_PACKAGE_PATTERN="${PROFILE}-$("$CODE_PATH"/scripts/pkg-full-vsn.sh 'vsn_matcher')"
PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}"
PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}"
if ! [ -f "$PACKAGE_FILE" ]; then
echo "$PACKAGE_FILE is not a file"
exit 1
fi
emqx_prepare(){
mkdir -p "${PACKAGE_PATH}"
if [ ! -d "/paho-mqtt-testing" ]; then
git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho-mqtt-testing
fi
@ -31,82 +59,80 @@ emqx_prepare(){
emqx_test(){
cd "${PACKAGE_PATH}"
local packagename="${PACKAGE_FILE_NAME}"
case "$PKG_SUFFIX" in
"zip")
unzip -q "${PACKAGE_PATH}/${packagename}"
export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \
EMQX_MQTT__MAX_TOPIC_ALIAS=10
sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins
for var in "$PACKAGE_PATH"/"${EMQX_NAME}"-*;do
case ${var##*.} in
"zip")
packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip)
unzip -q "${PACKAGE_PATH}/${packagename}"
export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \
EMQX_MQTT__MAX_TOPIC_ALIAS=10
sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins
echo "running ${packagename} start"
"${PACKAGE_PATH}"/emqx/bin/emqx start || ( tail "${PACKAGE_PATH}"/emqx/log/emqx.log.1 && exit 1 )
IDLE_TIME=0
while ! "${PACKAGE_PATH}"/emqx/bin/emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted'
do
if [ $IDLE_TIME -gt 10 ]
then
echo "emqx running error"
exit 1
fi
sleep 10
IDLE_TIME=$((IDLE_TIME+1))
done
pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
"${PACKAGE_PATH}"/emqx/bin/emqx stop
echo "running ${packagename} stop"
rm -rf "${PACKAGE_PATH}"/emqx
;;
"deb")
packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.deb)
dpkg -i "${PACKAGE_PATH}/${packagename}"
if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ]
echo "running ${packagename} start"
if ! "${PACKAGE_PATH}"/emqx/bin/emqx start; then
cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true
cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true
exit 1
fi
IDLE_TIME=0
while ! "${PACKAGE_PATH}"/emqx/bin/emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted'
do
if [ $IDLE_TIME -gt 10 ]
then
echo "package install error"
echo "emqx running error"
exit 1
fi
sleep 10
IDLE_TIME=$((IDLE_TIME+1))
done
pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
"${PACKAGE_PATH}"/emqx/bin/emqx stop
echo "running ${packagename} stop"
rm -rf "${PACKAGE_PATH}"/emqx
;;
"deb")
dpkg -i "${PACKAGE_PATH}/${packagename}"
if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ]
then
echo "package install error"
exit 1
fi
echo "running ${packagename} start"
running_test
echo "running ${packagename} stop"
echo "running ${packagename} start"
running_test
echo "running ${packagename} stop"
dpkg -r "${EMQX_NAME}"
if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "rc" ]
then
echo "package remove error"
exit 1
fi
dpkg -r "${PROFILE}"
if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "rc" ]
then
echo "package remove error"
exit 1
fi
dpkg -P "${EMQX_NAME}"
if dpkg -l |grep -q emqx
then
echo "package uninstall error"
exit 1
fi
;;
"rpm")
packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm)
rpm -ivh "${PACKAGE_PATH}/${packagename}"
if ! rpm -q emqx | grep -q emqx; then
echo "package install error"
exit 1
fi
dpkg -P "${PROFILE}"
if dpkg -l |grep -q emqx
then
echo "package uninstall error"
exit 1
fi
;;
"rpm")
yum install -y "${PACKAGE_PATH}/${packagename}"
if ! rpm -q "${PROFILE}" | grep -q "${PROFILE}"; then
echo "package install error"
exit 1
fi
echo "running ${packagename} start"
running_test
echo "running ${packagename} stop"
echo "running ${packagename} start"
running_test
echo "running ${packagename} stop"
rpm -e "${EMQX_NAME}"
if [ "$(rpm -q emqx)" != "package emqx is not installed" ];then
echo "package uninstall error"
exit 1
fi
;;
esac
done
rpm -e "${PROFILE}"
if [ "$(rpm -q emqx)" != "package emqx is not installed" ];then
echo "package uninstall error"
exit 1
fi
;;
esac
}
running_test(){
@ -114,7 +140,11 @@ running_test(){
EMQX_MQTT__MAX_TOPIC_ALIAS=10
sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins
emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 )
if ! emqx start; then
cat /var/log/emqx/erlang.log.1 || true
cat /var/log/emqx/emqx.log.1 || true
exit 1
fi
IDLE_TIME=0
while ! emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted'
do
@ -129,45 +159,47 @@ running_test(){
pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
# shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
emqx stop || kill "$(ps -ef | grep -E '\-progname\s.+emqx\s' |awk '{print $2}')"
if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \
|| [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then
service emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 )
IDLE_TIME=0
while ! emqx_ctl status | grep -E 'Node\s.*@.*\sis\sstarted'
do
if [ $IDLE_TIME -gt 10 ]
then
echo "emqx service error"
exit 1
fi
sleep 10
IDLE_TIME=$((IDLE_TIME+1))
done
service emqx stop
fi
}
relup_test(){
TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh)"
if [ -d "${RELUP_PACKAGE_PATH}" ];then
cd "${RELUP_PACKAGE_PATH}"
if [ ! -d "${RELUP_PACKAGE_PATH}" ];then
return 0
fi
cd "${RELUP_PACKAGE_PATH}"
while read -r pkg; do
packagename=$(basename "${pkg}")
unzip -q "$packagename"
if ! ./emqx/bin/emqx start; then
cat emqx/log/erlang.log.1 || true
cat emqx/log/emqx.log.1 || true
exit 1
fi
./emqx/bin/emqx_ctl status
./emqx/bin/emqx versions
OldVsn="$(./emqx/bin/emqx eval 'Versions=[{S, V} || {_,V,_, S} <- release_handler:which_releases()],
Current = proplists:get_value(current, Versions, proplists:get_value(permanent, Versions)),
io:format("~s", [Current])')"
cp "${PACKAGE_PATH}/${PROFILE}-${TARGET_VERSION}"-*.zip ./emqx/releases/
./emqx/bin/emqx install "${TARGET_VERSION}"
[ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
export EMQX_WAIT_FOR_STOP=300
./emqx/bin/emqx_ctl status
find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" |
while read -r pkg; do
packagename=$(basename "${pkg}")
unzip "$packagename"
./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 )
./emqx/bin/emqx_ctl status
./emqx/bin/emqx versions
cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases
./emqx/bin/emqx install "${TARGET_VERSION}"
[ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
./emqx/bin/emqx_ctl status
./emqx/bin/emqx stop
rm -rf emqx
done
fi
# also test remove old rel
./emqx/bin/emqx uninstall "$OldVsn"
# check emqx still runs
./emqx/bin/emqx ping
if ! ./emqx/bin/emqx stop; then
cat emqx/log/erlang.log.1 || true
cat emqx/log/emqx.log.1 || true
echo "failed to stop emqx"
exit 1
fi
rm -rf emqx
done < <(find . -maxdepth 1 -name "${OLD_PACKAGE_PATTERN}.zip")
}
emqx_prepare

2
.ci/docker-compose-file/conf.env
View File

@ -1,5 +1,5 @@
EMQX_AUTH__LDAP__SERVERS=ldap_server
EMQX_AUTH__MONGO__SERVER=mongo_server:27017
EMQX_AUTH__MONGO__SERVER=toxiproxy:27017
EMQX_AUTH__MYSQL__SERVER=mysql_server:3306
EMQX_AUTH__MYSQL__USERNAME=root
EMQX_AUTH__MYSQL__PASSWORD=public

99
.ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml Normal file
View File

@ -0,0 +1,99 @@
version: '3.9'
services:
haproxy:
container_name: haproxy
image: haproxy:2.3
depends_on:
- emqx1
- emqx2
volumes:
- ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
- ../../etc/certs:/usr/local/etc/haproxy/certs
ports:
- "18083:18083"
# - "1883:1883"
# - "8883:8883"
# - "8083:8083"
# - "5683:5683/udp"
# - "9999:9999"
# - "8084:8084"
networks:
- emqx_bridge
working_dir: /usr/local/etc/haproxy
command:
- bash
- -c
- |
cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /usr/local/etc/haproxy/certs/emqx.pem
haproxy -f /usr/local/etc/haproxy/haproxy.cfg
emqx1:
restart: always
container_name: node1.emqx.io
image: $TARGET:$EMQX_TAG
env_file:
- conf.cluster.env
volumes:
- etc:/opt/emqx/etc
environment:
- "EMQX_HOST=node1.emqx.io"
ports:
- "11881:18083"
# - "1883:1883"
command:
- /bin/sh
- -c
- |
sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
/opt/emqx/bin/emqx foreground
healthcheck:
test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"]
interval: 5s
timeout: 25s
retries: 5
networks:
emqx_bridge:
aliases:
- node1.emqx.io
emqx2:
restart: always
container_name: node2.emqx.io
image: $TARGET:$EMQX_TAG
env_file:
- conf.cluster.env
volumes:
- etc:/opt/emqx/etc
environment:
- "EMQX_HOST=node2.emqx.io"
ports:
- "11882:18083"
command:
- /bin/sh
- -c
- |
sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
/opt/emqx/bin/emqx foreground
healthcheck:
test: ["CMD", "/opt/emqx/bin/emqx", "ping"]
interval: 5s
timeout: 25s
retries: 5
networks:
emqx_bridge:
aliases:
- node2.emqx.io
volumes:
etc:
networks:
emqx_bridge:
driver: bridge
name: emqx_bridge
ipam:
driver: default
config:
- subnet: 172.100.239.0/24
gateway: 172.100.239.1

2
.ci/docker-compose-file/docker-compose-emqx-cluster.yaml
View File

@ -27,6 +27,7 @@ services:
haproxy -f /usr/local/etc/haproxy/haproxy.cfg
emqx1:
restart: always
container_name: node1.emqx.io
image: $TARGET:$EMQX_TAG
env_file:
@ -51,6 +52,7 @@ services:
- node1.emqx.io
emqx2:
restart: always
container_name: node2.emqx.io
image: $TARGET:$EMQX_TAG
env_file:

10
.ci/docker-compose-file/docker-compose-enterprise-tomcat-tcp.yaml Normal file
View File

@ -0,0 +1,10 @@
version: '3.9'
services:
web_server:
container_name: Tomcat
build:
context: ./http-service
image: web-server
networks:
- emqx_bridge

16
.ci/docker-compose-file/docker-compose-toxiproxy.yaml Normal file
View File

@ -0,0 +1,16 @@
version: '3.9'
services:
toxiproxy:
container_name: toxiproxy
image: ghcr.io/shopify/toxiproxy:2.5.0
restart: always
networks:
- emqx_bridge
volumes:
- "./toxiproxy.json:/config/toxiproxy.json"
ports:
- 8474:8474
command:
- "-host=0.0.0.0"
- "-config=/config/toxiproxy.json"

2
.ci/docker-compose-file/docker-compose.yaml
View File

@ -3,7 +3,7 @@ version: '3.9'
services:
erlang:
container_name: erlang
image: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
image: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
env_file:
- conf.env
environment:

15
.ci/docker-compose-file/http-service/Dockerfile Normal file
View File

@ -0,0 +1,15 @@
FROM tomcat:10.0.5
RUN wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip \
&& unzip -q apache-maven-3.6.3-bin.zip \
&& mv apache-maven-3.6.3 /opt/apache-maven-3.6.3/ \
&& ln -s /opt/apache-maven-3.6.3/ /opt/maven
ENV M2_HOME=/opt/maven
ENV M2=$M2_HOME/bin
ENV PATH=$M2:$PATH
COPY ./web-server /code
WORKDIR /code
RUN mvn package -Dmaven.skip.test=true
RUN mv ./target/emqx-web-0.0.1.war /usr/local/tomcat/webapps/emqx-web.war
EXPOSE 8080
CMD ["/usr/local/tomcat/bin/catalina.sh","run"]

65
.ci/docker-compose-file/http-service/web-server/pom.xml Normal file
View File

@ -0,0 +1,65 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>emqx-web</groupId>
<artifactId>emqx-web</artifactId>
<version>0.0.1</version>
<packaging>war</packaging>
<dependencies>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.16</version>
</dependency>
<dependency>
<groupId>commons-dbutils</groupId>
<artifactId>commons-dbutils</artifactId>
<version>1.7</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>commons-pool</groupId>
<artifactId>commons-pool</artifactId>
<version>1.6</version>
</dependency>
<dependency>
<groupId>jakarta.servlet</groupId>
<artifactId>jakarta.servlet-api</artifactId>
<version>5.0.0</version>
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/reousrce</directory>
<excludes>
<exclude>**/*.java</exclude>
</excludes>
</resource>
</resources>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-war-plugin</artifactId>
<version>3.2.3</version>
</plugin>
</plugins>
</build>
</project>

54
.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/AuthDAO.java Normal file
View File

@ -0,0 +1,54 @@
package com.emqx.dao;
import java.io.IOException;
import java.sql.SQLException;
import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.handlers.ScalarHandler;
import com.emqx.util.EmqxDatabaseUtil;
public class AuthDAO {
public String getUserName(String userName) throws IOException, SQLException {
QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
String sql = "select password from http_user where username='"+userName+"'";
String password =runner.query(sql, new ScalarHandler<String>());
return password;
}
public String getClient(String clientid) throws IOException, SQLException {
QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
String sql = "select password from http_user where clientid='"+clientid+"'";
String password =runner.query(sql, new ScalarHandler<String>());
return password;
}
public String getUserAccess(String userName) throws IOException, SQLException {
QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
String sql = "select access from http_acl where username='"+userName+"'";
String access =runner.query(sql, new ScalarHandler<String>());
return access;
}
public String getUserTopic(String userName) throws IOException, SQLException {
QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
String sql = "select topic from http_acl where username='"+userName+"'";
String topic =runner.query(sql, new ScalarHandler<String>());
return topic;
}
public String getClientAccess(String clientid) throws IOException, SQLException {
QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
String sql = "select access from http_acl where clientid='"+clientid+"'";
String access =runner.query(sql, new ScalarHandler<String>());
return access;
}
public String getClientTopic(String clientid) throws IOException, SQLException {
QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
String sql = "select topic from http_acl where clientid='"+clientid+"'";
String topic =runner.query(sql, new ScalarHandler<String>());
return topic;
}
}

45
.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/DBUtilsTest.java Normal file
View File

@ -0,0 +1,45 @@
package com.emqx.dao;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Properties;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.handlers.ColumnListHandler;
import org.apache.commons.dbutils.handlers.ScalarHandler;
import org.apache.commons.dbutils.handlers.columns.StringColumnHandler;
public class DBUtilsTest {
public static void main(String args[]) throws FileNotFoundException, IOException, SQLException {
Properties property = new Properties();//流文件
property.load(DBUtilsTest.class.getClassLoader().getResourceAsStream("database.properties"));
BasicDataSource dataSource = new BasicDataSource();
dataSource.setDriverClassName(property.getProperty("jdbc.driver"));
dataSource.setUrl(property.getProperty("jdbc.url"));
dataSource.setUsername(property.getProperty("jdbc.username"));
dataSource.setPassword(property.getProperty("jdbc.password"));
// 初始化连接数 if(initialSize!=null)
//dataSource.setInitialSize(Integer.parseInt(initialSize));
// 最小空闲连接 if(minIdle!=null)
//dataSource.setMinIdle(Integer.parseInt(minIdle));
// 最大空闲连接 if(maxIdle!=null)
//dataSource.setMaxIdle(Integer.parseInt(maxIdle));
QueryRunner runner = new QueryRunner(dataSource);
String sql="select username from mqtt_user where id=1";
String result = runner.query(sql, new ScalarHandler<String>());
System.out.println(result);
}
}

103
.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AclServlet.java Normal file
View File

@ -0,0 +1,103 @@
package com.emqx.servlet;
import java.io.IOException;
import java.sql.SQLException;
import com.emqx.dao.AuthDAO;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServlet;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
public class AclServlet extends HttpServlet {
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
// TODO Auto-generated method stub
doPost(req, resp);
}
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
String clientid = req.getParameter("clientid");
String username = req.getParameter("username");
String access = req.getParameter("access");
String topic = req.getParameter("topic");
//String password = req.getParameter("password");
//step0: password is not null, or not pass.
AuthDAO dao = new AuthDAO();
try {
//step1: check username access&topic
if(username != null) {
String access_1 = dao.getUserAccess(username);
String topic_1 = dao.getUserTopic(username);
if(access.equals(access_1)) {
if(topic.equals(topic_1)) {
resp.setStatus(200);
}
else {
if(clientid != null){
String access_2 = dao.getClientAccess(clientid);
String topic_2 = dao.getClientTopic(clientid);
if(access.equals(access_2)) {
if(topic.equals(topic_2)) {
resp.setStatus(200);
}
else {
resp.setStatus(400);
}
}else {
resp.setStatus(400);
}
}else {
resp.setStatus(400);
}
}
}else {//step2.1: username password is not match, then check clientid password
if(clientid != null){
String access_3 = dao.getClientAccess(clientid);
String topic_3 = dao.getClientTopic(clientid);
if(access.equals(access_3)) {
if(topic.equals(topic_3)) {
resp.setStatus(200);
}
else {
resp.setStatus(400);
}
}else {
resp.setStatus(400);
}
}else {
resp.setStatus(400);
}
}
}else {//step2.2: username is null, then check clientid password
if(clientid != null){
String access_4 = dao.getClientAccess(clientid);
String topic_4 = dao.getClientTopic(clientid);
if(access.equals(access_4)) {
if(topic.equals(topic_4)) {
resp.setStatus(200);
}
else {
resp.setStatus(400);
}
}else {
resp.setStatus(400);
}
}else {
resp.setStatus(400);
}
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}

72
.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AuthServlet.java Normal file
View File

@ -0,0 +1,72 @@
package com.emqx.servlet;
import java.io.IOException;
import java.sql.SQLException;
import com.emqx.dao.AuthDAO;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServlet;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
public class AuthServlet extends HttpServlet {
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
// TODO Auto-generated method stub
doPost(req, resp);
}
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
String clientid = req.getParameter("clientid");
String username =req.getParameter("username");
String password = req.getParameter("password");
//step0: password is not null, or not pass.
if(password == null) {
resp.setStatus(400);
return;
}
AuthDAO dao = new AuthDAO();
try {
//step1: check username password
if(username != null) {
String password_d = dao.getUserName(username);
if(password.equals(password_d)) {
resp.setStatus(200);
//200
}else {//step2.1: username password is not match, then check clientid password
if(clientid != null){
String password_c = dao.getClient(clientid);
if(password.equals(password_c)) {
resp.setStatus(200);
}else {
resp.setStatus(400);
}
}else {
resp.setStatus(400);
}
}
}else {//step2.2: username is null, then check clientid password
if(clientid != null){
String password_c = dao.getClient(clientid);
if(password.equals(password_c)) {
resp.setStatus(200);
}else {
resp.setStatus(400);
}
}else {
resp.setStatus(400);
}
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}

27
.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/util/EmqxDatabaseUtil.java Normal file
View File

@ -0,0 +1,27 @@
package com.emqx.util;
import java.io.IOException;
import java.util.Properties;
import javax.sql.DataSource;
import org.apache.commons.dbcp.BasicDataSource;
import com.emqx.dao.DBUtilsTest;
public class EmqxDatabaseUtil {
public static DataSource getDataSource() throws IOException {
Properties property = new Properties();// 流文件
property.load(EmqxDatabaseUtil.class.getClassLoader().getResourceAsStream("database.properties"));
BasicDataSource dataSource = new BasicDataSource();
dataSource.setDriverClassName(property.getProperty("jdbc.driver"));
dataSource.setUrl(property.getProperty("jdbc.url"));
dataSource.setUsername(property.getProperty("jdbc.username"));
dataSource.setPassword(property.getProperty("jdbc.password"));
return dataSource;
}
}

4
.ci/docker-compose-file/http-service/web-server/src/main/reousrce/database.properties Normal file
View File

@ -0,0 +1,4 @@
jdbc.driver= com.mysql.jdbc.Driver
jdbc.url= jdbc:mysql://mysql_server:3306/mqtt
jdbc.username= root
jdbc.password= public

3
.ci/docker-compose-file/http-service/web-server/src/main/webapp/META-INF/MANIFEST.MF Normal file
View File

@ -0,0 +1,3 @@
Manifest-Version: 1.0
Class-Path:

31
.ci/docker-compose-file/http-service/web-server/src/main/webapp/WEB-INF/web.xml Normal file
View File

@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://JAVA.sun.com/xml/ns/javaee"
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
id="WebApp_ID" version="2.5">
<display-name>emqx-web</display-name>
<servlet>
<servlet-name>Auth</servlet-name>
<servlet-class>com.emqx.servlet.AuthServlet</servlet-class>
</servlet>
<servlet>
<servlet-name>Acl</servlet-name>
<servlet-class>com.emqx.servlet.AclServlet</servlet-class>
</servlet>
<servlet-mapping>
<servlet-name>Auth</servlet-name>
<url-pattern>/auth</url-pattern>
</servlet-mapping>
<servlet-mapping>
<servlet-name>Acl</servlet-name>
<url-pattern>/acl</url-pattern>
</servlet-mapping>
<welcome-file-list>
<welcome-file>index.html</welcome-file>
<welcome-file>index.htm</welcome-file>
<welcome-file>index.jsp</welcome-file>
<welcome-file>default.html</welcome-file>
<welcome-file>default.htm</welcome-file>
<welcome-file>default.jsp</welcome-file>
</welcome-file-list>
</web-app>

10
.ci/docker-compose-file/http-service/web-server/src/main/webapp/index.html Normal file
View File

@ -0,0 +1,10 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>love</title>
</head>
<body>
It's lucky, jiabanxiang.
</body>
</html>

2
.ci/docker-compose-file/python/pytest.sh
View File

@ -10,7 +10,7 @@ LB="haproxy"
apk update && apk add git curl
git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing
pip install pytest
pip install pytest==6.2.5
pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$LB"
RESULT=$?

8
.ci/docker-compose-file/toxiproxy.json Normal file
View File

@ -0,0 +1,8 @@
[
{
"name": "mongo_single",
"listen": "0.0.0.0:27017",
"upstream": "mongo:27017",
"enabled": true
}
]

4
.ci/fvt_tests/http_server/rebar.config
View File

@ -1,7 +1,7 @@
{erl_opts, [debug_info]}.
{deps,
{deps,
[
{minirest, {git, "https://github.com.cnpmjs.org/emqx/minirest.git", {tag, "0.3.1"}}}
{minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.6"}}}
]}.
{shell, [

44
.ci/fvt_tests/local_relup_test_run.sh Normal file
View File

@ -0,0 +1,44 @@
#!/bin/bash
USAGE="$0 profile vsn old_vsn package_path"
EXAMPLE="$0 emqx 4.3.8-b3bb6075 v4.3.2 /home/alice/relup_dubug/downloaded_packages"
if [[ $# -ne 4 ]]; then
echo "$USAGE"
echo "$EXAMPLE"
exit 1
fi
set -ex
PROFILE="$1"
VSN="$2"
OLD_VSN="$3"
PACKAGE_PATH="$4"
FROM_OTP_VSN="${5:-24.3.4.2-1}"
TO_OTP_VSN="${6:-24.3.4.2-1}"
TEMPDIR=$(mktemp -d)
trap '{ rm -rf -- "$TEMPDIR"; }' EXIT
git clone --branch=master "https://github.com/terry-xiaoyu/one_more_emqx.git" "$TEMPDIR/one_more_emqx"
cp -r "$PACKAGE_PATH" "$TEMPDIR/packages"
cp relup.lux "$TEMPDIR/"
cp -r http_server "$TEMPDIR/http_server"
exec docker run \
-v "$TEMPDIR:/relup_test" \
-w "/relup_test" \
-e REBAR_COLOR=none \
-it emqx/relup-test-env:erl23.2.7.2-emqx-3-ubuntu20.04 \
lux \
--progress verbose \
--case_timeout infinity \
--var PROFILE="$PROFILE" \
--var PACKAGE_PATH="/relup_test/packages" \
--var ONE_MORE_EMQX_PATH="/relup_test/one_more_emqx" \
--var VSN="$VSN" \
--var OLD_VSN="$OLD_VSN" \
--var FROM_OTP_VSN="$FROM_OTP_VSN" \
--var TO_OTP_VSN="$TO_OTP_VSN" \
relup.lux

105
.ci/fvt_tests/relup.lux
View File

@ -1,15 +1,14 @@
[config var=PROFILE]
[config var=PACKAGE_PATH]
[config var=BENCH_PATH]
[config var=ONE_MORE_EMQX_PATH]
[config var=VSN]
[config var=OLD_VSNS]
[config var=OLD_VSN]
[config var=FROM_OTP_VSN]
[config var=TO_OTP_VSN]
[config shell_cmd=/bin/bash]
[config timeout=600000]
[loop old_vsn $OLD_VSNS]
[shell http_server]
!cd http_server
!rebar3 shell
@ -21,30 +20,27 @@
?>
[shell emqx]
!OLD_VSN=$(echo $OLD_VSN | sed -r 's/[v|e]//g')
!cd $PACKAGE_PATH
!unzip -q -o $PROFILE-ubuntu20.04-$(echo $old_vsn | sed -r 's/[v|e]//g')-amd64.zip
!unzip -q -o $PROFILE-$(echo $OLD_VSN | sed -r 's/[v|e]//g')-otp${FROM_OTP_VSN}-ubuntu20.04-amd64.zip
?SH-PROMPT
!cd emqx
!export EMQX_LOG__CONSOLE_HANDLER__ENABLE=true
!export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug
!export EMQX_LOG__PRIMARY_LEVEL=debug
!export EMQX_ZONES__DEFAULT__LISTENERS__MQTT_WSS__BIND="0.0.0.0:8085"
!export EMQX_LOG__LEVEL=debug
!./bin/emqx start
?EMQ X .* is started successfully!
?SH-PROMPT
[shell emqx2]
!OLD_VSN=$(echo $OLD_VSN | sed -r 's/[v|e]//g')
!cd $PACKAGE_PATH
!cp -f $ONE_MORE_EMQX_PATH/one_more_$(echo $PROFILE | sed 's/-/_/g').sh .
!./one_more_$(echo $PROFILE | sed 's/-/_/g').sh emqx2
?SH-PROMPT
!cd emqx2
!export EMQX_LOG__CONSOLE_HANDLER__ENABLE=true
!export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug
!export EMQX_LOG__PRIMARY_LEVEL=debug
!export EMQX_LOG__LEVEL=debug
!./bin/emqx start
?EMQ X .* is started successfully!
@ -67,6 +63,8 @@
!./bin/emqx_ctl rules create 'SELECT * FROM "t/#"' '[{"name":"data_to_webserver", "params": {"$$resource": "resource:691c29ba"}}]'
?created
?SH-PROMPT
!sleep 5
?SH-PROMPT
[shell emqx]
!./bin/emqx_ctl resources list
@ -75,19 +73,41 @@
!./bin/emqx_ctl rules list
?691c29ba
?SH-PROMPT
!./bin/emqx_ctl broker metrics | grep "messages.publish"
???SH-PROMPT
[shell bench]
!cd $BENCH_PATH
!./emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
???sent
!emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
# e.g. Start with 20 workers, addrs pool size: 1 and req interval: 200 ms
?^Start
[shell emqx]
!echo "" > log/emqx.log.1
?SH-PROMPT
!cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
!cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/
## upgrade to the new version
!./bin/emqx install $VSN
?Made release permanent: "$VSN"
?SH-PROMPT
!./bin/emqx versions |grep permanent
?(.*)$VSN
?SH-PROMPT
## downgrade to the old version
!./bin/emqx install $${OLD_VSN}
?Made release permanent:.*
?SH-PROMPT
!./bin/emqx versions |grep permanent | grep -qs "$${OLD_VSN}"
?SH-PROMPT:
!echo ==$$?==
?^==0==
?SH-PROMPT:
## again, upgrade to the new version
!./bin/emqx install $VSN
?Made release permanent: "$VSN"
?SH-PROMPT
@ -103,7 +123,7 @@
"""
?SH-PROMPT
!./bin/emqx_ctl plugins list | grep emqx_management
!./bin/emqx_ctl plugins list | grep --color=never emqx_management
?Plugin\(emqx_management.*active=true\)
?SH-PROMPT
@ -111,8 +131,29 @@
!echo "" > log/emqx.log.1
?SH-PROMPT
!cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
!cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/
## upgrade to the new version
!./bin/emqx install $VSN
?Made release permanent: "$VSN"
?SH-PROMPT
!./bin/emqx versions |grep permanent
?(.*)$VSN
?SH-PROMPT
## downgrade to the old version
!./bin/emqx install $${OLD_VSN}
?Made release permanent:.*
?SH-PROMPT
!./bin/emqx versions |grep permanent | grep -qs "$${OLD_VSN}"
?SH-PROMPT:
!echo ==$$?==
?^==0==
?SH-PROMPT:
## again, upgrade to the new version
!./bin/emqx install $VSN
?Made release permanent: "$VSN"
?SH-PROMPT
@ -128,26 +169,34 @@
"""
?SH-PROMPT
!./bin/emqx_ctl plugins list | grep emqx_management
!./bin/emqx_ctl plugins list | grep --color=never emqx_management
?Plugin\(emqx_management.*active=true\)
?SH-PROMPT
[shell bench]
???publish complete
??SH-PROMPT:
?publish complete
?SH-PROMPT
!sleep 30
?SH-PROMPT
!curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq --raw-output ".data[0].metrics[] | select(.node==\"emqx@127.0.0.1\").matched"
?300
[shell emqx]
!./bin/emqx_ctl broker metrics | grep "messages.publish"
???SH-PROMPT
## We don't guarantee not to lose a single message!
## So even if we received 290~300 messages, we consider it as success
[shell bench]
!curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq -M --raw-output ".data[0].metrics[] | select(.node==\"emqx@127.0.0.1\").matched"
?(29[0-9])|(300)
?SH-PROMPT
!curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq --raw-output ".data[0].actions[0].metrics[] | select(.node==\"emqx@127.0.0.1\").success"
?300
!curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq -M --raw-output ".data[0].actions[0].metrics[] | select(.node==\"emqx@127.0.0.1\").success"
?(29[0-9])|(300)
?SH-PROMPT
## The /counter API is provided by .ci/fvt_test/http_server
!curl http://127.0.0.1:8080/counter
???{"data":300,"code":0}
?\{"data":(29[0-9])|(300),"code":0\}
?SH-PROMPT
[shell emqx2]
@ -181,8 +230,6 @@
!halt(3).
?SH-PROMPT:
[endloop]
[cleanup]
!echo ==$$?==
?==0==

1
.gitattributes vendored
View File

@ -1,3 +1,4 @@
build text eol=lf
* text=auto
*.* text eol=lf
*.jpg -text

5
.gitignore vendored
View File

@ -39,8 +39,6 @@ etc/emqx.conf.rendered
Mnesia.*/
*.DS_Store
_checkouts
./rebar.config
./rebar.config.erl
rebar.config.rendered
/rebar3
rebar.lock
@ -90,6 +88,9 @@ erlang_ls.config
*~
# Emacs temporary files
.#*
*#
# For direnv
.envrc
mix.lock
.gitconfig.tmp
*.log

2
.tool-versions
View File

@ -1 +1 @@
erlang 23.2.7.2-emqx-3
erlang 24.3.4.2-1

72
Makefile
View File

@ -1,13 +1,19 @@
$(shell $(CURDIR)/scripts/git-hooks-init.sh)
REBAR_VERSION = 3.14.3-emqx-7
REBAR = $(CURDIR)/rebar3
BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts
export EMQX_RELUP ?= true
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-alpine3.15.1
export EMQX_DEFAULT_RUNNER = alpine:3.15.1
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
export EMQX_DESC ?= EMQ X
export EMQX_CE_DASHBOARD_VERSION ?= v4.3.8
export DOCKERFILE := deploy/docker/Dockerfile
export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
ifeq ($(OS),Windows_NT)
export REBAR_COLOR=none
FIND=/usr/bin/find
else
FIND=find
endif
GET_DASHBOARD=$(SCRIPTS)/get-dashboard.sh
@ -16,6 +22,7 @@ PROFILE ?= emqx
REL_PROFILES := emqx emqx-edge
PKG_PROFILES := emqx-pkg emqx-edge-pkg
PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default
CT_READABLE ?= true
ifeq ($(OS),Windows_NT)
QUIKRUN=$(CURDIR)/_build/$(PROFILE)/rel/emqx/bin/emqx.cmd console
@ -34,7 +41,7 @@ all: $(REBAR) $(PROFILES)
.PHONY: ensure-rebar3
ensure-rebar3:
@$(SCRIPTS)/fail-on-old-otp-version.escript
@$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION)
@$(SCRIPTS)/ensure-rebar3.sh
$(REBAR): ensure-rebar3
@ -59,11 +66,19 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh)
## app/name-ct targets are intended for local tests hence cover is not enabled
.PHONY: $(APPS:%=%-ct)
define gen-app-ct-target
$1-ct:
$(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
$1-ct: $(REBAR)
$(REBAR) ct --name 'test@127.0.0.1' -v --readable $(CT_READABLE) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
endef
$(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
## app/name-ct-pipeline targets are used in pipeline -> make cover data for each app
.PHONY: $(APPS:%=%-ct-pipeline)
define gen-app-ct-target-pipeline
$1-ct-pipeline: $(REBAR)
$(REBAR) ct --name 'test@127.0.0.1' -c -v --readable $(CT_READABLE) --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
endef
$(foreach app,$(APPS),$(eval $(call gen-app-ct-target-pipeline,$(app))))
## apps/name-prop targets
.PHONY: $(APPS:%=%-prop)
define gen-app-prop-target
@ -81,6 +96,7 @@ coveralls: $(REBAR)
@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
.PHONY: $(REL_PROFILES)
$(REL_PROFILES:%=%): $(REBAR) get-dashboard
@$(REBAR) as $(@) do compile,release
@ -93,15 +109,17 @@ $(REL_PROFILES:%=%): $(REBAR) get-dashboard
clean: $(PROFILES:%=clean-%)
$(PROFILES:%=clean-%):
@if [ -d _build/$(@:clean-%=%) ]; then \
rm rebar.lock \
rm -f rebar.lock; \
rm -rf _build/$(@:clean-%=%)/rel; \
find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
find _build/$(@:clean-%=%) -type l -delete; \
$(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
$(FIND) _build/$(@:clean-%=%) -type l -delete; \
fi
.PHONY: clean-all
clean-all:
@rm -f rebar.lock
@rm -rf _build
@rm -f rebar.lock
.PHONY: deps-all
deps-all: $(REBAR) $(PROFILES:%=deps-%)
@ -116,8 +134,9 @@ $(PROFILES:%=deps-%): $(REBAR) get-dashboard
@rm -f rebar.lock
.PHONY: xref
xref: $(REBAR)
xref: $(REBAR) $(REL_PROFILES:%=%-rel)
@$(REBAR) as check xref
@scripts/xref-check.escript
.PHONY: dialyzer
dialyzer: $(REBAR)
@ -130,10 +149,19 @@ COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS)
$(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS)
@$(BUILD) $(subst -rel,,$(@)) rel
## download relup base packages
.PHONY: $(REL_PROFILES:%=%-relup-downloads)
define download-relup-packages
$1-relup-downloads:
@if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-base-packages.sh $1; fi
endef
ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt))))
## relup target is to create relup instructions
.PHONY: $(REL_PROFILES:%=%-relup)
define gen-relup-target
$1-relup: $(COMMON_DEPS)
$1-relup: $1-relup-downloads $(COMMON_DEPS)
@$(BUILD) $1 relup
endef
ALL_ZIPS = $(REL_PROFILES)
@ -156,6 +184,27 @@ $1: $1-rel
endef
$(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))
## docker target is to create docker instructions
.PHONY: $(REL_PROFILES:%=%-docker)
define gen-docker-target
$1-docker: $(COMMON_DEPS)
@$(BUILD) $1 docker
endef
ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
## emqx-docker-testing
## emqx-ee-docker-testing
## is to directly copy a unzipped zip-package to a
## base image such as ubuntu20.04. Mostly for testing
.PHONY: $(REL_PROFILES:%=%-docker-testing)
define gen-docker-target-testing
$1-docker-testing: $(COMMON_DEPS)
@$(BUILD) $1 docker-testing
endef
ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt))))
.PHONY: run
run: $(PROFILE) quickrun
@ -166,4 +215,3 @@ ci: $(REBAR) $(PROFILE)
.PHONY: quickrun
quickrun:
@$(QUIKRUN)
include docker.mk

30
Windows.md
View File

@ -6,7 +6,7 @@ NOTE: The instructions and examples are based on Windows 10.
### Visual studio for C/C++ compile and link
EMQ X includes Erlang NIF (Native Implmented Function) components, implemented
EMQX includes Erlang NIF (Native Implmented Function) components, implemented
in C/C++. To compile and link C/C++ libraries, the easiest way is perhaps to
install Visual Studio.
@ -25,17 +25,17 @@ C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build
Depending on your visual studio version and OS, the paths may differ.
The first path is for rebar3 port compiler to find `cl.exe` and `link.exe`
The second path is for Powershell or CMD to setup environment variables.
The second path is for CMD to setup environment variables.
### Erlang/OTP
Install Erlang/OTP 23.2 from https://www.erlang.org/downloads
Install Erlang/OTP 24.2.1 from https://www.erlang.org/downloads
You may need to edit the `Path` environment variable to allow running
Erlang commands such as `erl` from powershell.
Erlang commands such as `erl` from CMD.
To validate Erlang installation in CMD or powershell:
To validate Erlang installation in CMD :
* Start (or restart) CMD or powershell
* Start (or restart) CMD
* Execute `erl` command to enter Erlang shell
@ -45,13 +45,13 @@ e.g.
```
PS C:\Users\zmsto> erl
Eshell V11.1.4 (abort with ^G)
Eshell V12.2.1 (abort with ^G)
1> halt().
```
### bash
All EMQ X build/run scripts are either in `bash` or `escript`.
All EMQX build/run scripts are either in `bash` or `escript`.
`escript` is installed as a part of Erlang. To install a `bash`
environment in Windows, there are quite a few options.
@ -63,12 +63,12 @@ Cygwin is what we tested with.
to `Path` list.
* Validate installation.
Start (restart) CMD or powershell console and execute `which bash`, it should
Start (restart) CMD console and execute `which bash`, it should
print out `/usr/bin/bash`
### Other tools
Some of the unix world tools are required to build EMQ X. Including:
Some of the unix world tools are required to build EMQX. Including:
* git
* curl
@ -84,11 +84,11 @@ When using scoop:
scoop install git curl make jq zip unzip
```
## Build EMQ X source code
## Build EMQX source code
* Clone the repo: `git clone https://github.com/emqx/emqx.git`
* Start CMD or Powershell
* Start CMD
* Execute `vcvarsall.bat x86_amd64` to load environment variables
@ -112,11 +112,11 @@ scoop install git curl make jq zip unzip
To fix it, Visual Studio's bin paths should be ordered prior to Cygwin's (or similar installation's)
bin paths in `Path` environment variable.
## Run EMQ X
## Run EMQX
To start EMQ X broker.
To start EMQX broker.
Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQ X.
Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQX.
Then execute `_build\emqx\rel\emqx>.\bin\emqx_ctl status` to check status.
If everything works fine, it should print out

79
apps/dgiot/conf/rebar.config
View File

@ -1,3 +1,4 @@
%% -*- mode: erlang -*-
%% This config file is the very basic config to compile emqx
%% This allows emqx to be used as a dependency for other applications
%% such as emqx module/plugin develpments and tests.
@ -6,23 +7,24 @@
%% with rebar.config.erl module. Final result is written to
%% rebar.config.rendered if environment DEBUG is set.
{minimum_otp_vsn, "23"}.
{edoc_opts, [{preprocess,true}]}.
{erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import,
warn_obsolete_guard,compressed,
{d, snk_kind, msg}]}.
warn_obsolete_guard,compressed,
{d, snk_kind, msg}]}.
{extra_src_dirs, [{"etc", [{recursive,true}]}]}.
{xref_checks,[undefined_function_calls,undefined_functions,locals_not_used,
deprecated_function_calls,warnings_as_errors,deprecated_functions]}.
deprecated_function_calls,warnings_as_errors,deprecated_functions]}.
{dialyzer, [
{warnings, [unmatched_returns, error_handling, race_conditions]},
{warnings, [unmatched_returns, error_handling]},
{plt_location, "."},
{plt_prefix, "emqx_dialyzer"},
{plt_apps, all_apps},
{statistics, true}
]
]
}.
{cover_opts, [verbose]}.
@ -37,39 +39,40 @@
{deps,
[
{gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
, {ehttpc, {git, "https://gitee.com/fastdgiot/ehttpc", {tag, "0.1.14"}}}
, {gproc, {git, "https://gitee.com/fastdgiot/gproc", {tag, "0.8.0"}}}
, {jiffy, {git, "https://gitee.com/fastdgiot/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://gitee.com/fastdgiot/cowboy", {tag, "2.8.2"}}}
, {esockd, {git, "https://gitee.com/fastdgiot/esockd", {tag, "5.8.0"}}}
, {ekka, {git, "https://gitee.com/fastdgiot/ekka", {tag, "0.8.1.7"}}}
, {gen_rpc, {git, "https://gitee.com/fastdgiot/gen_rpc", {tag, "2.5.1"}}}
, {cuttlefish, {git, "https://gitee.com/fastdgiot/cuttlefish", {tag, "v4.3.7"}}}
, {minirest, {git, "https://gitee.com/fastdgiot/minirest", {tag, "0.3.7"}}}
, {ecpool, {git, "https://gitee.com/fastdgiot/ecpool", {tag, "0.5.2"}}}
, {replayq, {git, "https://gitee.com/fastdgiot/replayq", {tag, "0.3.2"}}}
, {pbkdf2, {git, "https://gitee.com/fastdgiot/erlang-pbkdf2.git", {branch, "2.0.4"}}}
, {emqtt, {git, "https://gitee.com/fastdgiot/emqtt", {tag, "1.2.3.1"}}}
, {rulesql, {git, "https://gitee.com/fastdgiot/rulesql", {tag, "0.1.2"}}}
, {recon, {git, "https://gitee.com/fastdgiot/recon", {tag, "2.5.1"}}}
, {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1
, {getopt, "1.0.1"}
, {snabbkaffe, {git, "https://gitee.com/fastdgiot/snabbkaffe.git", {tag, "0.12.2"}}}
, {ejdbc, {git, "https://gitee.com/fastdgiot/ejdbc", {tag, "1.0.1"}}}
, {gun, {git, "https://gitee.com/fastdgiot/gun", {tag, "1.3.5"}}}
, {gen_smtp, {git, "https://gitee.com/fastdgiot/gen_smtp", "1.1.2"}}
, {srly, {git, "https://gitee.com/fastdgiot/srly.git", "1.0.0"}}
{erlfmt, {git, "https://gitee.com/fastdgiot/erlfmt", {tag, "v1.1.0"}}}
, {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
, {redbug, {git, "https://gitee.com/fastdgiot/redbug.git", {tag, "2.0.7"}}}
, {ehttpc, {git, "https://gitee.com/fastdgiot/ehttpc", {tag, "0.3.0"}}}
, {gun, {git, "https://gitee.com/fastdgiot/gun", {tag, "1.3.7"}}}
, {gproc, {git, "https://gitee.com/fastdgiot/gproc", {tag, "0.8.0"}}}
, {jiffy, {git, "https://gitee.com/fastdgiot/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://gitee.com/fastdgiot/cowboy", {tag, "2.9.0"}}}
, {esockd, {git, "https://gitee.com/fastdgiot/esockd", {tag, "5.8.7"}}}
, {ekka, {git, "https://gitee.com/fastdgiot/ekka", {tag, "0.8.1.11"}}}
, {gen_rpc, {git, "https://gitee.com/fastdgiot/gen_rpc", {tag, "2.8.1"}}}
, {cuttlefish, {git, "https://gitee.com/fastdgiot/cuttlefish", {tag, "v4.3.7"}}}
, {minirest, {git, "https://gitee.com/fastdgiot/minirest", {tag, "0.3.10"}}}
, {ecpool, {git, "https://gitee.com/fastdgiot/ecpool", {tag, "0.5.2"}}}
, {replayq, {git, "https://gitee.com/fastdgiot/replayq", {tag, "0.3.4"}}}
, {pbkdf2, {git, "https://gitee.com/fastdgiot/erlang-pbkdf2.git", {branch, "2.0.4"}}}
, {emqtt, {git, "https://gitee.com/fastdgiot/emqtt", {tag, "1.2.3.1"}}}
, {rulesql, {git, "https://gitee.com/fastdgiot/rulesql", {tag, "0.1.5"}}}
, {recon, {git, "https://gitee.com/fastdgiot/recon", {tag, "2.5.1"}}}
, {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1
, {getopt, "1.0.1"}
, {snabbkaffe, {git, "https://gitee.com/fastdgiot/snabbkaffe.git", {tag, "1.0.1"}}}
, {lc, {git, "https://gitee.com/fastdgiot/lc.git", {tag, "0.3.2"}}}
, {grpc, {git, "https://gitee.com/fastdgiot/grpc-erl", {tag, "0.6.7"}}}
]}.
{xref_ignores,
[ %% schema registry is for enterprise
{emqx_schema_registry,get_all_schemas,0},
{emqx_schema_api,format_schema,1},
{emqx_schema_api,make_schema_params,1},
{emqx_schema_parser,decode,3},
{emqx_schema_parser,encode,3},
{emqx_schema_registry,add_schema,1},
emqx_exhook_pb, % generated code for protobuf
emqx_exproto_pb % generated code for protobuf
]}.
[ %% schema registry is for enterprise
{emqx_schema_registry,get_all_schemas,0},
{emqx_schema_api,format_schema,1},
{emqx_schema_api,make_schema_params,1},
{emqx_schema_parser,decode,3},
{emqx_schema_parser,encode,3},
{emqx_schema_registry,add_schema,1},
emqx_exhook_pb, % generated code for protobuf
emqx_exproto_pb % generated code for protobuf
]}.

383
apps/dgiot/conf/rebar.config.erl
View File

@ -18,9 +18,9 @@ bcrypt() ->
deps(Config) ->
{deps, OldDeps} = lists:keyfind(deps, 1, Config),
MoreDeps = case provide_bcrypt_dep() of
true -> [bcrypt()];
false -> []
end,
true -> [bcrypt()];
false -> []
end,
{HasElixir, ExtraDeps} = extra_deps(),
{HasElixir, lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps ++ ExtraDeps})}.
@ -48,29 +48,31 @@ filter_extra_deps([{Plugin, _} = P | More], Filter, Acc) ->
end.
overrides() ->
[{add, [{extra_src_dirs, [{"etc", [{recursive, true}]}]}
, {erl_opts, [{compile_info, [{emqx_vsn, get_vsn()}]}]}
]}
, {add, snabbkaffe,
[{erl_opts, common_compile_opts()}]}
[ {add, [ {extra_src_dirs, [{"etc", [{recursive,true}]}]}
, {erl_opts, [{compile_info, [{emqx_vsn, get_vsn()}]}]}
]}
, {add, relx, [{erl_opts, [{d, 'RLX_LOG', rlx_log}]}]}
, {add, snabbkaffe,
[{erl_opts, common_compile_opts()}]}
] ++ community_plugin_overrides().
community_plugin_overrides() ->
[{add, App, [{erl_opts, [{i, "include"}]}]} || App <- relx_plugin_apps_extra()].
[{add, App, [ {erl_opts, [{i, "include"}]}]} || App <- relx_plugin_apps_extra()].
config(HasElixir) ->
[{cover_enabled, is_cover_enabled()}
, {profiles, profiles()}
, {project_app_dirs, project_app_dirs()}
, {plugins, plugins(HasElixir)}
| [{provider_hooks, [{pre, [{compile, {mix, find_elixir_libs}}]}
, {post, [{compile, {mix, consolidate_protocols}}]}
]} || HasElixir]
[ {cover_enabled, is_cover_enabled()}
, {profiles, profiles()}
, {project_app_dirs, project_app_dirs()}
, {plugins, plugins(HasElixir)}
| [ {provider_hooks, [ {pre, [{compile, {mix, find_elixir_libs}}]}
, {post, [{compile, {mix, consolidate_protocols}}]}
]} || HasElixir ]
].
is_cover_enabled() ->
case os:getenv("ENABLE_COVER_COMPILE") of
"1" -> true;
"1"-> true;
"true" -> true;
_ -> false
end.
@ -88,12 +90,11 @@ project_app_dirs() ->
["apps/*", alternative_lib_dir() ++ "/*", "."].
plugins(HasElixir) ->
[{relup_helper, {git, "https://gitee.com/fastdgiot/relup_helper", {tag, "2.0.0"}}}
, {er_coap_client, {git, "https://gitee.com/fastdgiot/er_coap_client", {tag, "v1.0"}}}
[{relup_helper, {git, "https://gitee.com/fastdgiot/relup_helper", {tag, "2.1.0"}}}
%% emqx main project does not require port-compiler
%% pin at root level for deterministic
, {pc, {git, "https://gitee.com/fastdgiot/port_compiler.git", {tag, "v1.11.1"}}}
| [rebar_mix || HasElixir]
| [ {rebar_mix, "v0.4.0"} || HasElixir ]
]
%% test plugins are concatenated to default profile plugins
%% otherwise rebar3 test profile runs are super slow
@ -101,18 +102,18 @@ plugins(HasElixir) ->
test_plugins() ->
[rebar3_proper,
{coveralls, {git, "https://gitee.com/fastdgiot/coveralls-erl", {branch, "fix-git-info"}}}
{coveralls, {git, "https://gitee.com/fastdgiot/coveralls-erl", {tag, "v2.2.0-emqx-1"}}}
].
test_deps() ->
[{bbmustache, "1.10.0"}
, {emqx_ct_helpers, {git, "https://gitee.com/fastdgiot/emqx-ct-helpers", {tag, "1.3.9"}}}
, {emqx_ct_helpers, {git, "https://gitee.com/fastdgiot/emqx-ct-helpers", {tag, "1.3.11"}}}
, meck
].
common_compile_opts() ->
[debug_info % alwyas include debug_info
, {compile_info, [{emqx_vsn, get_vsn()}]}
, {compile_info, [{emqx_vsn, get_vsn()}]}
, {d, snk_kind, msg}
] ++
[{d, 'EMQX_ENTERPRISE'} || is_enterprise()] ++
@ -126,58 +127,64 @@ prod_compile_opts() ->
].
prod_overrides() ->
[{add, [{erl_opts, [deterministic]}]}].
relup_deps(Profile) ->
{post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "scripts/inject-deps.escript " ++ atom_to_list(Profile)}]}.
[{add, [ {erl_opts, [deterministic]}]}].
profiles() ->
Vsn = get_vsn(),
[{'emqx', [{erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, cloud, bin)}
, {overrides, prod_overrides()}
, relup_deps('emqx')
]}
, {'emqx-pkg', [{erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, cloud, pkg)}
, {overrides, prod_overrides()}
, relup_deps('emqx-pkg')
]}
, {'emqx-edge', [{erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, edge, bin)}
, {overrides, prod_overrides()}
, relup_deps('emqx-edge')
]}
, {'emqx-edge-pkg', [{erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, edge, pkg)}
, {overrides, prod_overrides()}
, relup_deps('emqx-edge-pkg')
]}
, {check, [{erl_opts, common_compile_opts()}
]}
, {test, [{deps, test_deps()}
, {erl_opts, common_compile_opts() ++ erl_opts_i()}
, {extra_src_dirs, [{"test", [{recursive, true}]}]}
]}
[ {'emqx', [ {erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, cloud, bin)}
, {overrides, prod_overrides()}
]}
, {'emqx-pkg', [ {erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, cloud, pkg)}
, {overrides, prod_overrides()}
]}
, {'emqx-edge', [ {erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, edge, bin)}
, {overrides, prod_overrides()}
]}
, {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, edge, pkg)}
, {overrides, prod_overrides()}
]}
, {check, [ {erl_opts, common_compile_opts()}
]}
, {test, [ {deps, test_deps()}
, {erl_opts, common_compile_opts() ++ erl_opts_i()}
, {extra_src_dirs, [{"test", [{recursive,true}]}]}
]}
] ++ ee_profiles(Vsn).
%% RelType: cloud (full size) | edge (slim size)
%% PkgType: bin | pkg
relx(Vsn, RelType, PkgType) ->
IsEnterprise = is_enterprise(),
[{include_src, false}
, {include_erts, true}
, {extended_start_script, false}
, {generate_start_script, false}
, {sys_config, false}
, {vm_args, false}
, {release, {emqx, Vsn}, relx_apps(RelType)}
, {overlay, relx_overlay(RelType)}
, {overlay_vars, [{built_on_arch, rebar_utils:get_arch()}
, {emqx_description, emqx_description(RelType, IsEnterprise)}
| overlay_vars(RelType, PkgType, IsEnterprise)]}
[ {include_src,false}
, {include_erts, true}
, {extended_start_script,false}
, {generate_start_script,false}
, {sys_config,false}
, {vm_args,false}
, {release, {emqx, Vsn}, relx_apps(RelType)}
, {overlay, relx_overlay(RelType)}
, {overlay_vars, [ {built_on_platform, built_on()}
, {emqx_description, emqx_description(RelType, IsEnterprise)}
| overlay_vars(RelType, PkgType, IsEnterprise)]}
].
built_on() ->
On = rebar_utils:get_arch(),
case distro() of
false -> On;
Distro -> On ++ "-" ++ Distro
end.
distro() ->
case os:type() of
{unix, _} -> string:strip(os:cmd("scripts/get-distro.sh"), both, $\n);
_ -> false
end.
emqx_description(cloud, true) -> "EMQ X Enterprise";
emqx_description(cloud, false) -> "EMQ X Broker";
emqx_description(edge, _) -> "EMQ X Edge".
@ -198,7 +205,6 @@ overlay_vars_rel(RelType) ->
, {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce
, {enable_plugin_emqx_recon, true}
, {enable_plugin_emqx_retainer, true}
, {enable_plugin_emqx_web_hook, true}
, {enable_plugin_emqx_telemetry, true}
%% dgiot base plugin
, {enable_plugin_dgiot, true}
@ -228,54 +234,42 @@ overlay_vars_rel(RelType) ->
%% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb)
overlay_vars_pkg(bin) ->
[{platform_bin_dir, "bin"}
, {platform_data_dir, "data"}
, {platform_etc_dir, "etc"}
, {platform_lib_dir, "lib"}
, {platform_log_dir, "log"}
, {platform_plugins_dir, "etc/plugins"}
, {runner_root_dir, "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)"}
, {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"}
, {runner_etc_dir, "$RUNNER_ROOT_DIR/etc"}
, {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"}
, {runner_log_dir, "$RUNNER_ROOT_DIR/log"}
, {runner_data_dir, "$RUNNER_ROOT_DIR/data"}
, {runner_user, ""}
[ {platform_bin_dir, "bin"}
, {platform_data_dir, "data"}
, {platform_etc_dir, "etc"}
, {platform_lib_dir, "lib"}
, {platform_log_dir, "log"}
, {platform_plugins_dir, "etc/plugins"}
, {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"}
, {runner_etc_dir, "$RUNNER_ROOT_DIR/etc"}
, {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"}
, {runner_log_dir, "$RUNNER_ROOT_DIR/log"}
, {runner_data_dir, "$RUNNER_ROOT_DIR/data"}
, {runner_user, ""}
];
overlay_vars_pkg(pkg) ->
[{platform_bin_dir, ""}
, {platform_data_dir, "/var/lib/emqx"}
, {platform_etc_dir, "/etc/emqx"}
, {platform_lib_dir, ""}
, {platform_log_dir, "/var/log/emqx"}
, {platform_plugins_dir, "/var/lib/emqx/plugins"}
, {runner_root_dir, "/usr/lib/emqx"}
, {runner_bin_dir, "/usr/bin"}
, {runner_etc_dir, "/etc/emqx"}
, {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"}
, {runner_log_dir, "/var/log/emqx"}
, {runner_data_dir, "/var/lib/emqx"}
, {runner_user, "emqx"}
[ {platform_bin_dir, ""}
, {platform_data_dir, "/var/lib/emqx"}
, {platform_etc_dir, "/etc/emqx"}
, {platform_lib_dir, ""}
, {platform_log_dir, "/var/log/emqx"}
, {platform_plugins_dir, "/var/lib/emqx/plugins"}
, {runner_bin_dir, "/usr/bin"}
, {runner_etc_dir, "/etc/emqx"}
, {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"}
, {runner_log_dir, "/var/log/emqx"}
, {runner_data_dir, "/var/lib/emqx"}
, {runner_user, "emqx"}
].
relx_apps(ReleaseType) ->
[kernel
, sasl
, crypto
, public_key
, asn1
, syntax_tools
, ssl
, os_mon
, inets
, compiler
, runtime_tools
relx_otp_apps() ++
[ redbug
, cuttlefish
, jsx
, jesse
, jwerl
, odbc
, srly
, erlydtl
, erlport
, ecpool
@ -283,7 +277,7 @@ relx_apps(ReleaseType) ->
, gpb
, poolboy
, ibrowse
, gen_smtp
, emqx
, {mnesia, load}
, {ekka, load}
@ -291,15 +285,19 @@ relx_apps(ReleaseType) ->
, observer_cli
]
++ [emqx_modules || not is_enterprise()]
++ [emqx_license || is_enterprise()]
++ [bcrypt || provide_bcrypt_release(ReleaseType)]
++ relx_apps_per_rel(ReleaseType)
++ [{N, load} || N <- relx_plugin_apps(ReleaseType)].
++ [emqx_license || is_enterprise()]
++ [bcrypt || provide_bcrypt_release(ReleaseType)]
++ relx_apps_per_rel(ReleaseType)
++ [{N, load} || N <- relx_plugin_apps(ReleaseType)].
relx_otp_apps() ->
{ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"),
true = is_list(Apps),
Apps.
relx_apps_per_rel(cloud) ->
[luerl
, xmerl
| [{observer, load} || is_app(observer)]
[
{observer, load} || is_app(observer)
];
relx_apps_per_rel(edge) ->
[].
@ -307,31 +305,28 @@ relx_apps_per_rel(edge) ->
is_app(Name) ->
case application:load(Name) of
ok -> true;
{error, {already_loaded, _}} -> true;
{error,{already_loaded, _}} -> true;
_ -> false
end.
relx_plugin_apps(ReleaseType) ->
[emqx_retainer
, emqx_management
, emqx_dashboard
, emqx_bridge_mqtt
, emqx_recon
, emqx_rule_engine
, emqx_sasl
, emqx_exhook
, emqx_web_hook
, emqx_auth_mnesia
[ emqx_retainer
, emqx_management
, emqx_dashboard
, emqx_bridge_mqtt
, emqx_recon
, emqx_rule_engine
, emqx_sasl
, emqx_auth_mnesia
]
++ [emqx_telemetry || not is_enterprise()]
++ relx_plugin_apps_per_rel(ReleaseType)
++ relx_plugin_apps_enterprise(is_enterprise())
++ relx_plugin_apps_extra().
++ relx_plugin_apps_per_rel(ReleaseType)
++ relx_plugin_apps_enterprise(is_enterprise())
++ relx_plugin_apps_extra().
relx_plugin_apps_per_rel(cloud) ->
[emqx_lwm2m
, emqx_lua_hook
, emqx_exhook
[
emqx_exhook
, emqx_prometheus
, emqx_psk_file
, emqx_auth_mnesia
@ -362,7 +357,7 @@ relx_plugin_apps_per_rel(edge) ->
relx_plugin_apps_enterprise(true) ->
[list_to_atom(A) || A <- filelib:wildcard("*", "lib-ee"),
filelib:is_dir(filename:join(["lib-ee", A]))];
filelib:is_dir(filename:join(["lib-ee", A]))];
relx_plugin_apps_enterprise(false) -> [].
relx_plugin_apps_extra() ->
@ -370,29 +365,31 @@ relx_plugin_apps_extra() ->
[Plugin || {Plugin, _} <- ExtraDeps].
relx_overlay(ReleaseType) ->
[{mkdir, "log/"}
, {mkdir, "data/"}
, {mkdir, "data/mnesia"}
, {mkdir, "data/configs"}
, {mkdir, "data/patches"}
, {mkdir, "data/scripts"}
, {template, "data/loaded_plugins.tmpl", "data/loaded_plugins"}
, {template, "data/loaded_modules.tmpl", "data/loaded_modules"}
, {template, "data/emqx_vars", "releases/emqx_vars"}
, {copy, "bin/emqx", "bin/emqx"}
, {copy, "bin/emqx_ctl", "bin/emqx_ctl"}
, {copy, "bin/node_dump", "bin/node_dump"}
, {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"}
, {copy, "bin/emqx", "bin/emqx-{{release_version}}"} %% for relup
, {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"} %% for relup
, {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"} %% for relup
, {template, "bin/emqx.cmd", "bin/emqx.cmd"}
, {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"}
, {copy, "bin/nodetool", "bin/nodetool"}
, {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"}
, {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish"}
, {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish-{{release_version}}"}
, {copy, "priv/emqx.schema", "releases/{{release_version}}/"}
[ {mkdir, "log/"}
, {mkdir, "data/"}
, {mkdir, "data/mnesia"}
, {mkdir, "data/configs"}
, {mkdir, "data/patches"}
, {mkdir, "data/scripts"}
, {mkdir, "data/backup"}
, {template, "data/loaded_plugins.tmpl", "data/loaded_plugins"}
, {template, "data/loaded_modules.tmpl", "data/loaded_modules"}
, {template, "data/emqx_vars", "releases/emqx_vars"}
, {copy, "bin/emqx", "bin/emqx"}
, {copy, "bin/emqx_ctl", "bin/emqx_ctl"}
, {copy, "bin/emqx_cluster_rescue", "bin/emqx_cluster_rescue"}
, {copy, "bin/node_dump", "bin/node_dump"}
, {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"}
, {copy, "bin/emqx", "bin/emqx-{{release_version}}"} %% for relup
, {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"} %% for relup
, {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"} %% for relup
, {template, "bin/emqx.cmd", "bin/emqx.cmd"}
, {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"}
, {copy, "bin/nodetool", "bin/nodetool"}
, {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"}
, {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish"}
, {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish-{{release_version}}"}
, {copy, "priv/emqx.schema", "releases/{{release_version}}/"}
] ++ case is_enterprise() of
true -> ee_etc_overlay(ReleaseType);
false -> etc_overlay(ReleaseType)
@ -401,50 +398,50 @@ relx_overlay(ReleaseType) ->
etc_overlay(ReleaseType) ->
PluginApps = relx_plugin_apps(ReleaseType),
Templates = emqx_etc_overlay(ReleaseType) ++
lists:append([plugin_etc_overlays(App) || App <- PluginApps]) ++
[community_plugin_etc_overlays(App) || App <- relx_plugin_apps_extra()],
[{mkdir, "etc/"}
, {mkdir, "etc/plugins"}
, {template, "etc/BUILT_ON", "releases/{{release_version}}/BUILT_ON"}
, {copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"}
lists:append([plugin_etc_overlays(App) || App <- PluginApps]) ++
[community_plugin_etc_overlays(App) || App <- relx_plugin_apps_extra()],
[ {mkdir, "etc/"}
, {mkdir, "etc/plugins"}
, {template, "etc/BUILT_ON", "releases/{{release_version}}/BUILT_ON"}
, {copy, "{{base_dir}}/lib/emqx/etc/certs","etc/"}
] ++
lists:map(
fun({From, To}) -> {template, From, To};
(FromTo) -> {template, FromTo, FromTo}
end, Templates)
++ extra_overlay(ReleaseType).
lists:map(
fun({From, To}) -> {template, From, To};
(FromTo) -> {template, FromTo, FromTo}
end, Templates)
++ extra_overlay(ReleaseType).
extra_overlay(cloud) ->
[{copy, "{{base_dir}}/lib/emqx_lwm2m/lwm2m_xml", "etc/"}
, {copy, "{{base_dir}}/lib/emqx_psk_file/etc/psk.txt", "etc/psk.txt"}
[
{copy, "{{base_dir}}/lib/emqx_psk_file/etc/psk.txt", "etc/psk.txt"}
];
extra_overlay(edge) ->
[].
emqx_etc_overlay(cloud) ->
emqx_etc_overlay_common() ++
[{"etc/emqx_cloud/vm.args", "etc/vm.args"}
[ {"etc/emqx_cloud/vm.args","etc/vm.args"}
];
emqx_etc_overlay(edge) ->
emqx_etc_overlay_common() ++
[{"etc/emqx_edge/vm.args", "etc/vm.args"}
[ {"etc/emqx_edge/vm.args","etc/vm.args"}
].
emqx_etc_overlay_common() ->
["etc/acl.conf", "etc/emqx.conf", "etc/ssl_dist.conf",
%% TODO: check why it has to end with .paho
%% and why it is put to etc/plugins dir
{"etc/acl.conf.paho", "etc/plugins/acl.conf.paho"}].
%% TODO: check why it has to end with .paho
%% and why it is put to etc/plugins dir
{"etc/acl.conf.paho", "etc/plugins/acl.conf.paho"}].
plugin_etc_overlays(App0) ->
App = atom_to_list(App0),
ConfFiles = find_conf_files(App),
%% NOTE: not filename:join here since relx translates it for windows
[{"{{base_dir}}/lib/" ++ App ++ "/etc/" ++ F, "etc/plugins/" ++ F}
|| F <- ConfFiles].
[{"{{base_dir}}/lib/"++ App ++"/etc/" ++ F, "etc/plugins/" ++ F}
|| F <- ConfFiles].
community_plugin_etc_overlays(App0) ->
App = atom_to_list(App0),
{"{{base_dir}}/lib/" ++ App ++ "/etc/" ++ App ++ ".conf", "etc/plugins/" ++ App ++ ".conf"}.
{"{{base_dir}}/lib/"++ App ++"/etc/" ++ App ++ ".conf", "etc/plugins/" ++ App ++ ".conf"}.
%% NOTE: for apps fetched as rebar dependency (there is so far no such an app)
%% the overlay should be hand-coded but not to rely on build-time wildcards.
@ -465,7 +462,7 @@ get_vsn() ->
false -> os:cmd("./pkg-vsn.sh");
Vsn -> Vsn
end,
re:replace(PkgVsn, "\n", "", [{return, list}]).
re:replace(PkgVsn, "\n", "", [{return ,list}]).
maybe_dump(Config) ->
is_debug() andalso file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]),
@ -491,22 +488,22 @@ provide_bcrypt_release(ReleaseType) ->
erl_opts_i() ->
[{i, "apps"}] ++
[{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++
[{i, Dir} || Dir <- filelib:wildcard(filename:join([alternative_lib_dir(), "*", "include"]))].
[{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++
[{i, Dir} || Dir <- filelib:wildcard(filename:join([alternative_lib_dir(), "*", "include"]))].
dialyzer(Config) ->
{dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
AppsToAnalyse = case os:getenv("DIALYZER_ANALYSE_APP") of
false ->
[];
Value ->
[list_to_atom(App) || App <- string:tokens(Value, ",")]
end,
false ->
[];
Value ->
[ list_to_atom(App) || App <- string:tokens(Value, ",")]
end,
AppNames = [emqx | list_dir("apps")] ++ list_dir(alternative_lib_dir()),
KnownApps = [Name || Name <- AppsToAnalyse, lists:member(Name, AppNames)],
KnownApps = [Name || Name <- AppsToAnalyse, lists:member(Name, AppNames)],
AppsToExclude = AppNames -- KnownApps,
@ -519,21 +516,21 @@ dialyzer(Config) ->
coveralls() ->
case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of
{"true", Token} when is_list(Token) ->
Cfgs = [{coveralls_repo_token, Token},
{"true", Token} when is_list(Token) ->
Cfgs = [{coveralls_repo_token, Token},
{coveralls_service_job_id, os:getenv("GITHUB_RUN_ID")},
{coveralls_commit_sha, os:getenv("GITHUB_SHA")},
{coveralls_coverdata, "_build/test/cover/*.coverdata"},
{coveralls_service_name, "github"}],
case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request"
andalso string:tokens(os:getenv("GITHUB_REF"), "/") of
[_, "pull", PRNO, _] ->
[{coveralls_service_pull_request, PRNO} | Cfgs];
_ ->
Cfgs
end;
_ ->
[]
case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request"
andalso string:tokens(os:getenv("GITHUB_REF"), "/") of
[_, "pull", PRNO, _] ->
[{coveralls_service_pull_request, PRNO} | Cfgs];
_ ->
Cfgs
end;
_ ->
[]
end.
list_dir(Dir) ->

9
apps/dgiot/include/logger.hrl
View File

@ -1,5 +1,3 @@
-compile({parse_transform, emqx_logger}).
-define(DEBUG(Format), ?LOG(debug, Format, [])).
-define(DEBUG(Format, Args), ?LOG(debug, Format, Args)).
@ -26,7 +24,7 @@
-define(LOG(Level, Format, Args),
begin
(logger:log(Level, #{}, #{
report_cb => fun(_) -> {'$logger_header'() ++ (Format), (Args)} end,
report_cb => fun(_) -> { (Format), (Args)} end,
domain => [dgiot_public],
mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY},
line => ?LINE}))
@ -35,7 +33,7 @@
-define(LOG(Level, Format, Args, ACL),
begin
(logger:log(Level, #{}, #{
report_cb => fun(_) -> {'$logger_header'() ++ (Format), (Args)} end,
report_cb => fun(_) -> {(Format), (Args)} end,
domain => ACL,
mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY},
line => ?LINE}))
@ -59,9 +57,10 @@
line => ?LINE}))
end).
-define(PLOG(Level, Map),
begin
(dgiot_parse:log(#{
(dgiot_parse_log:log(#{
<<"pid">> => erlang:pid_to_list(self()),
<<"time">> => dgiot_datetime:now_microsecs(),
<<"node">> => node(),

23
apps/dgiot/include/rule_actions.hrl
View File

@ -1,4 +1,20 @@
-compile({parse_transform, dgiot_rule_actions_trans}).
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-compile({parse_transform, emqx_rule_actions_trans}).
-type selected_data() :: map().
-type env_vars() :: map().
@ -6,6 +22,11 @@
-define(BINDING_KEYS, '__bindings__').
-define(LOG_RULE_ACTION(Level, Metadata, Fmt, Args),
emqx_rule_utils:log_action(Level, Metadata, Fmt, Args)).
-define(bound_v(Key, ENVS0),
maps:get(Key,
maps:get(?BINDING_KEYS, ENVS0, #{}))).
-define(JWT_TABLE, emqx_rule_engine_jwt_table).

11
apps/dgiot/src/transport/dgiot_mqtt.erl
View File

@ -20,7 +20,16 @@
-include("dgiot_mqtt.hrl").
-include_lib("dgiot/include/logger.hrl").
-include_lib("emqx_rule_engine/include/rule_engine.hrl").
-include_lib("emqx_rule_engine/include/rule_actions.hrl").
%%-include_lib("emqx_rule_engine/include/rule_actions.hrl").
-define(LOG_RULE_ACTION(Level, Metadata, Fmt, Args),
emqx_rule_utils:log_action(Level, Metadata, Fmt, Args)).
-define(bound_v(Key, ENVS0),
maps:get(Key,
maps:get(?BINDING_KEYS, ENVS0, #{}))).
-define(BINDING_KEYS, '__bindings__').
%% ETS tables for PubSub
-define(SUBOPTION, emqx_suboption).

2
apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_plugin_libs,
[{description, "EMQ X Plugin utility libs"},
{vsn, "4.3.1"},
{vsn, "4.4.6"},
{modules, []},
{applications, [kernel,stdlib]},
{env, []}

54
apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src
View File

@ -1,16 +1,44 @@
%% -*-: erlang -*-
%% -*- mode: erlang -*-
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[
{<<"4.3.0">>, [
{load_module, emqx_plugin_libs_ssl, brutal_purge, soft_purge, []}
]},
{<<".*">>, []}
],
{<<"4\\.4\\.[3-5]">>,
[{load_module,emqx_trace,brutal_purge,soft_purge,[]},
{load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]},
{"4.4.2",[
{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]},
{load_module,emqx_trace,brutal_purge,soft_purge,[]},
{load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]},
{"4.4.1",
[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]},
{load_module,emqx_trace,brutal_purge,soft_purge,[]},
{load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]},
{"4.4.0",
[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]},
{load_module,emqx_trace,brutal_purge,soft_purge,[]},
{load_module,emqx_trace_api,brutal_purge,soft_purge,[]},
{update,emqx_slow_subs,{advanced,["4.4.0"]}},
{load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[
{<<"4.3.0">>, [
{load_module, emqx_plugin_libs_ssl, brutal_purge, soft_purge, []}
]},
{<<".*">>, []}
]
}.
{<<"4\\.4\\.[3-5]">>,
[{load_module,emqx_trace,brutal_purge,soft_purge,[]},
{load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]},
{"4.4.3",
[{load_module,emqx_trace,brutal_purge,soft_purge,[]},
{load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]},
{"4.4.2",
[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]},
{load_module,emqx_trace,brutal_purge,soft_purge,[]},
{load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]},
{"4.4.1",
[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]},
{load_module,emqx_trace,brutal_purge,soft_purge,[]},
{load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]},
{"4.4.0",
[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]},
{load_module,emqx_trace,brutal_purge,soft_purge,[]},
{load_module,emqx_trace_api,brutal_purge,soft_purge,[]},
{update,emqx_slow_subs,{advanced,["4.4.0"]}},
{load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}]}.

68
apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -58,21 +58,31 @@ save_files_return_opts(Options, Dir) ->
KeyFile = Get(<<"keyfile">>),
CertFile = Get(<<"certfile">>),
CAFile = GetD(<<"cacertfile">>, Get(<<"cafile">>)),
Key = do_save_file(KeyFile, Dir),
Cert = do_save_file(CertFile, Dir),
CA = do_save_file(CAFile, Dir),
Key = maybe_save_file(KeyFile, Dir),
Cert = maybe_save_file(CertFile, Dir),
CA = maybe_save_file(CAFile, Dir),
Verify = case GetD(<<"verify">>, false) of
false -> verify_none;
_ -> verify_peer
end,
SNI = case Get(<<"server_name_indication">>) of
<<"disable">> -> disable;
"disable" -> disable;
"" -> undefined;
<<>> -> undefined;
undefined -> undefined;
SNI0 -> ensure_str(SNI0)
end,
Versions = emqx_tls_lib:integral_versions(Get(<<"tls_versions">>)),
Ciphers = emqx_tls_lib:integral_ciphers(Versions, Get(<<"ciphers">>)),
filter([{keyfile, Key}, {certfile, Cert}, {cacertfile, CA},
{verify, Verify}, {server_name_indication, SNI}, {versions, Versions}, {ciphers, Ciphers}]).
filter([ {keyfile, Key}
, {certfile, Cert}
, {cacertfile, CA}
, {verify, Verify}
, {server_name_indication, SNI}
, {versions, Versions}
, {ciphers, Ciphers}
]).
%% @doc Save a key or certificate file in data dir,
%% and return path of the saved file.
@ -80,25 +90,47 @@ save_files_return_opts(Options, Dir) ->
-spec save_file(file_input(), atom() | string() | binary()) -> string().
save_file(Param, SubDir) ->
Dir = filename:join([emqx:get_env(data_dir), SubDir]),
do_save_file( Param, Dir).
maybe_save_file(Param, Dir).
filter([]) -> [];
filter([{_, ""} | T]) -> filter(T);
filter([{_, undefined} | T]) -> filter(T);
filter([H | T]) -> [H | filter(T)].
do_save_file(#{<<"filename">> := FileName, <<"file">> := Content}, Dir)
maybe_save_file(#{<<"filename">> := FileName, <<"file">> := Content}, Dir)
when FileName =/= undefined andalso Content =/= undefined ->
do_save_file(ensure_str(FileName), iolist_to_binary(Content), Dir);
do_save_file(FilePath, _) when is_binary(FilePath) ->
maybe_save_file(ensure_str(FileName), iolist_to_binary(Content), Dir);
maybe_save_file(FilePath, _) when is_binary(FilePath) ->
ensure_str(FilePath);
do_save_file(FilePath, _) when is_list(FilePath) ->
maybe_save_file(FilePath, _) when is_list(FilePath) ->
FilePath;
do_save_file(_, _) -> "".
maybe_save_file(_, _) -> "".
do_save_file("", _, _Dir) -> ""; %% ignore
do_save_file(_, <<>>, _Dir) -> ""; %% ignore
do_save_file(FileName, Content, Dir) ->
maybe_save_file("", _, _Dir) -> ""; %% no filename, ignore
maybe_save_file(FileName, <<>>, Dir) -> %% no content, see if file exists
{ok, Cwd} = file:get_cwd(),
%% NOTE: when FileName is an absolute path, filename:join has no effect
CwdFile = ensure_str(filename:join([Cwd, FileName])),
DataDirFile = ensure_str(filename:join([Dir, FileName])),
Possibles0 = case CwdFile =:= DataDirFile of
true -> [CwdFile];
false -> [CwdFile, DataDirFile]
end,
Possibles = Possibles0 ++
case FileName of
"etc/certs/" ++ Path ->
%% this is the dir hard-coded in rule-engine resources as
%% default, unfortunatly we cannot change the deaults
%% due to compatibilty reasons, so we have to make a guess
["/etc/emqx/certs/" ++ Path];
_ ->
[]
end,
case find_exist_file(FileName, Possibles) of
false -> erlang:throw({bad_cert_file, Possibles});
Found -> Found
end;
maybe_save_file(FileName, Content, Dir) ->
FullFilename = filename:join([Dir, FileName]),
ok = filelib:ensure_dir(FullFilename),
case file:write_file(FullFilename, Content) of
@ -112,3 +144,9 @@ do_save_file(FileName, Content, Dir) ->
ensure_str(L) when is_list(L) -> L;
ensure_str(B) when is_binary(B) -> unicode:characters_to_list(B, utf8).
find_exist_file(_Name, []) -> false;
find_exist_file(Name, [F | Rest]) ->
case filelib:is_regular(F) of
true -> F;
false -> find_exist_file(Name, Rest)
end.

41
baiduMap_upgrade.sh
View File

@ -1,41 +0,0 @@
#!/usr/bin/env bash
#wget -qO baiduMap_upgrade.sh https://gitee.com/dgiiot/dgiot/raw/master/baiduMap_upgrade.sh && chmod 777 baiduMap_upgrade.sh && sh baiduMap_upgrade.sh 'WpeAb6pL4tsX2ZVd11156GHbO9Ut6c4HZhG'
#if [ ! -d "dgiot-dashboard" ]; then
echo "当前进度: 安装node"
mkdir /usr/local/node/
cd /usr/local/node/
wget https://npm.taobao.org/mirrors/node/v14.17.6/node-v14.17.6-linux-x64.tar.gz
tar -zxvf node-v14.17.6-linux-x64.tar.gz
rm -rf node-v14.17.6-linux-x64.tar.gz
ln -s /usr/local/node/node-v14.17.6-linux-x64/bin/npm /usr/local/bin/npm
ln -s /usr/local/node/node-v14.17.6-linux-x64/bin/node /usr/local/bin/node
node -v
#fi
mapKey=$1
echo "你输入的百度地图key为: ${mapKey}"
cd /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv
if [ ! -d "dgiot-dashboard" ]; then
echo "当前进度: 克隆dgiot-dashboard"
git clone --depth 1 https://gitee.com/dgiiot/dgiot-dashboard.git dgiot-dashboard
fi
cd dgiot-dashboard
if [ ! -d "node_modules" ]; then
echo "进度: 安装依赖"
echo "node_modules not found, install node_modules..."
npm install
fi
echo "进度: 替换key"
sed -i "s/'WpeAb6pL4tsX2ZVd56GHbO9Ut6c4HZhG'/${mapKey}/g" /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/dgiot-dashboard/src/config/secret.config.js
echo "进度: 打包编译"
npm build
echo "进度: 拷贝文件夹dist"
mv /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/www /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/wwwback
mkdir /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/www
cp -r /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/dgiot-dashboard/dist/* /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/www/
echo "进度: 删除下载文件dgiot-dashboard"
#rm -rf /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/dgiot-dashboard
echo '替换百度地图key完成'

BIN
bin/cuttlefish-4.3.10
View File

Binary file not shown.

613
bin/emqx
View File

@ -1,12 +1,22 @@
#!/bin/bash
#!/usr/bin/env bash
# -*- tab-width:4;indent-tabs-mode:nil -*-
# ex: ts=4 sw=4 et
set -e
ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
DEBUG="${DEBUG:-0}"
if [ "$DEBUG" -eq 1 ]; then
set -x
fi
RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
# shellcheck disable=SC1090
. "$ROOT_DIR"/releases/emqx_vars
. "$RUNNER_ROOT_DIR"/releases/emqx_vars
EMQX_LICENSE_CONF=''
REL_NAME="emqx"
ERTS_PATH="$RUNNER_ROOT_DIR/erts-$ERTS_VSN/bin"
export EMQX_DESCRIPTION
RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME"
CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}"
@ -28,91 +38,166 @@ export PROGNAME="erl"
DYNLIBS_DIR="$RUNNER_ROOT_DIR/dynlibs"
ERTS_LIB_DIR="$ERTS_DIR/../lib"
# Echo to stderr on errors
echoerr() { echo "$*" 1>&2; }
check_eralng_start() {
"$BINDIR/$PROGNAME" -noshell -boot "$REL_DIR/start_clean" -s crypto start -s init stop
}
if ! check_eralng_start >/dev/null 2>&1; then
BUILT_ON="$(head -1 "${REL_DIR}/BUILT_ON")"
## failed to start, might be due to missing libs, try to be portable
export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH"
if ! check_eralng_start; then
## it's hopeless
echoerr "FATAL: Unable to start Erlang (with libcrypto)."
echoerr "Please make sure it's running on the correct platform with all required dependencies."
echoerr "This EMQ X release is built for $BUILT_ON"
exit 1
fi
echoerr "WARNING: There seem to be missing dynamic libs from the OS. Using libs from ${DYNLIBS_DIR}"
fi
## backward compatible
if [ -d "$ERTS_DIR/lib" ]; then
export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH"
fi
# Fix bin permission for all erts bin files
# the 'x' attributes may get lost if the files are extracted from a relup package
find "$BINDIR" -exec chmod a+x {} \;
# cuttlefish try to read environment variables starting with "EMQX_"
export CUTTLEFISH_ENV_OVERRIDE_PREFIX='EMQX_'
relx_usage() {
command="$1"
usage() {
local command="$1"
case "$command" in
unpack)
echo "Usage: $REL_NAME unpack [VERSION]"
echo "Unpacks a release package VERSION, it assumes that this"
echo "release package tarball has already been deployed at one"
echo "of the following locations:"
echo " releases/<relname>-<version>.tar.gz"
echo " releases/<relname>-<version>.zip"
;;
install)
echo "Usage: $REL_NAME install [VERSION]"
echo "Installs a release package VERSION, it assumes that this"
echo "release package tarball has already been deployed at one"
echo "of the following locations:"
echo " releases/<relname>-<version>.tar.gz"
echo " releases/<relname>-<version>.zip"
echo ""
echo " --no-permanent Install release package VERSION but"
echo " don't make it permanent"
;;
uninstall)
echo "Usage: $REL_NAME uninstall [VERSION]"
echo "Uninstalls a release VERSION, it will only accept"
echo "versions that are not currently in use"
;;
upgrade)
echo "Usage: $REL_NAME upgrade [VERSION]"
echo "Upgrades the currently running release to VERSION, it assumes"
echo "that a release package tarball has already been deployed at one"
echo "of the following locations:"
echo " releases/<relname>-<version>.tar.gz"
echo " releases/<relname>-<version>.zip"
echo ""
echo " --no-permanent Install release package VERSION but"
echo " don't make it permanent"
;;
downgrade)
echo "Usage: $REL_NAME downgrade [VERSION]"
echo "Downgrades the currently running release to VERSION, it assumes"
echo "that a release package tarball has already been deployed at one"
echo "of the following locations:"
echo " releases/<relname>-<version>.tar.gz"
echo " releases/<relname>-<version>.zip"
echo ""
echo " --no-permanent Install release package VERSION but"
echo " don't make it permanent"
;;
*)
echo "Usage: $REL_NAME {start|start_boot <file>|ertspath|foreground|stop|restart|reboot|pid|ping|console|console_clean|console_boot <file>|attach|remote_console|upgrade|downgrade|install|uninstall|versions|escript|rpc|rpcterms|eval|root_dir}"
;;
start)
echo "Start EMQX service in daemon mode"
;;
stop)
echo "Stop the running EMQX program"
;;
console)
echo "Boot up EMQX service in an interactive Erlang shell"
echo "This command needs a tty"
;;
console_clean)
echo "This command does NOT boot up the EMQX service"
echo "It only starts an interactive Erlang shell with all the"
echo "EMQX code available"
;;
foreground)
echo "Start EMQX in foreground mode without an interactive shell"
;;
pid)
echo "Print out EMQX process identifier"
;;
ping)
echo "Check if the EMQX node is up and running"
echo "This command exit with 0 silently if node is running"
;;
escript)
echo "Execute a escript using the Erlang runtime from EMQX package installation"
echo "For example $REL_NAME escript /path/to/my/escript my_arg1 my_arg2"
;;
attach)
echo "This command is applicable when EMQX is started in daemon mode."
echo "It attaches the current shell to EMQX's control console"
echo "through a named pipe."
echo "WARNING: try to use the safer alternative, remote_console command."
;;
remote_console)
echo "Start an interactive shell running an Erlang node which "
echo "hidden-connects to the running EMQX node".
echo "This command is mostly used for troubleshooting."
;;
ertspath)
echo "Print path to Erlang runtime dir"
;;
rpc)
echo "Usge $REL_NAME rpc MODULE FUNCTION [ARGS, ...]"
echo "Connect to the EMQX node and make an Erlang RPC"
echo "This command blocks for at most 60 seconds."
echo "It exits with non-zero code in case of any RPC failure"
echo "including connection error and runtime exception"
;;
rpcterms)
echo "Usge $REL_NAME rpcterms MODULE FUNCTION [ARGS, ...]"
echo "Connect to the EMQX node and make an Erlang RPC"
echo "The result of the RPC call is pretty-printed as an "
echo "Erlang term"
;;
root_dir)
echo "Print EMQX installation root dir"
;;
eval)
echo "Evaluate an Erlang expression in the EMQX node"
;;
versions)
echo "List installed EMQX versions and their status"
;;
unpack)
echo "Usage: $REL_NAME unpack [VERSION]"
echo "Unpacks a release package VERSION, it assumes that this"
echo "release package tarball has already been deployed at one"
echo "of the following locations:"
echo " releases/<relname>-<version>.zip"
;;
install)
echo "Usage: $REL_NAME install [VERSION]"
echo "Installs a release package VERSION, it assumes that this"
echo "release package tarball has already been deployed at one"
echo "of the following locations:"
echo " releases/<relname>-<version>.zip"
echo ""
echo " --no-permanent Install release package VERSION but"
echo " don't make it permanent"
;;
uninstall)
echo "Usage: $REL_NAME uninstall [VERSION]"
echo "Uninstalls a release VERSION, it will only accept"
echo "versions that are not currently in use"
;;
upgrade)
echo "Usage: $REL_NAME upgrade [VERSION]"
echo "Upgrades the currently running release to VERSION, it assumes"
echo "that a release package tarball has already been deployed at one"
echo "of the following locations:"
echo " releases/<relname>-<version>.zip"
echo ""
echo " --no-permanent Install release package VERSION but"
echo " don't make it permanent"
;;
downgrade)
echo "Usage: $REL_NAME downgrade [VERSION]"
echo "Downgrades the currently running release to VERSION, it assumes"
echo "that a release package tarball has already been deployed at one"
echo "of the following locations:"
echo " releases/<relname>-<version>.zip"
echo ""
echo " --no-permanent Install release package VERSION but"
echo " don't make it permanent"
;;
*)
echo "Usage: $REL_NAME COMMAND [help]"
echo ''
echo "Commonly used COMMANDs:"
echo " start: Start EMQX in daemon mode"
echo " console: Start EMQX in an interactive Erlang shell"
echo " foreground: Start EMQX in foreground mode without an interactive shell"
echo " stop: Stop the running EMQX node"
echo " restart: Restart the running EMQX node"
echo " ctl: Administration commands, execute '$REL_NAME ctl help' for more details"
echo ''
echo "More:"
echo " Shell attach: remote_console | attach"
echo " Up/Down-grade: upgrade | downgrade | install | uninstall"
echo " Install info: ertspath | root_dir | versions"
echo " Runtime info: pid | ping | versions"
echo " Config check: check_conf"
echo " Advanced: console_clean | escript | rpc | rpcterms | eval"
echo ''
echo "Execute '$REL_NAME COMMAND help' for more information"
;;
esac
}
COMMAND="${1:-}"
if [ -z "$COMMAND" ]; then
usage 'help'
exit 1
elif [ "$COMMAND" = 'help' ]; then
usage 'help'
exit 0
fi
if [ "${2:-}" = 'help' ]; then
## 'ctl' command has its own usage info
if [ "$COMMAND" != 'ctl' ]; then
usage "$COMMAND"
exit 0
fi
fi
# Simple way to check the correct user and fail early
check_user() {
# Validate that the user running the script is the owner of the
@ -132,7 +217,6 @@ check_user() {
fi
}
# Make sure the user running this script is the owner and/or su to that user
check_user "$@"
ES=$?
@ -140,10 +224,54 @@ if [ "$ES" -ne 0 ]; then
exit $ES
fi
# Echo to stderr on errors
echoerr() { echo "$*" 1>&2; }
die() {
set +x
echoerr "ERROR: $1"
errno=${2:-1}
exit "$errno"
}
assert_node_alive() {
if ! relx_nodetool "ping" > /dev/null; then
die "node_is_not_running!" 1
fi
}
check_erlang_start() {
# set ERL_CRASH_DUMP_BYTES to zero so it will not write a crash dump file
env ERL_CRASH_DUMP_BYTES=0 "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()"
}
if ! check_erlang_start >/dev/null 2>&1; then
BUILT_ON="$(head -1 "${REL_DIR}/BUILT_ON")"
## failed to start, might be due to missing libs, try to be portable
export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH"
if ! check_erlang_start; then
## it's hopeless
echoerr "FATAL: Unable to start Erlang."
echoerr "Please make sure openssl-1.1.1 (libcrypto) and libncurses are installed."
echoerr "Also ensure it's running on the correct platform,"
echoerr "this EMQX release is built for $BUILT_ON"
exit 1
fi
echoerr "There seem to be missing dynamic libs from the OS."
echoerr "Using libs from ${DYNLIBS_DIR} instead."
echoerr "NOTE: EMQX's rpm or deb package installation is recommended!"
fi
## backward compatible (old EMQX versions does this)
if [ -d "$ERTS_DIR/lib" ]; then
export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH"
fi
if [ -z "$WITH_EPMD" ]; then
EPMD_ARG="-start_epmd false -epmd_module ekka_epmd -proto_dist ekka"
else
EPMD_ARG="-start_epmd true $PROTO_DIST_ARG"
PROTO_DIST=$(grep -E '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}')
EPMD_ARG="-start_epmd true -proto_dist $PROTO_DIST"
fi
# Warn the user if ulimit -n is less than 1024
@ -154,9 +282,6 @@ if [ "$ULIMIT_F" -lt 1024 ]; then
echo "!!!!"
fi
# By default, use cuttlefish to generate app.config and vm.args
CUTTLEFISH="${USE_CUTTLEFISH:-yes}"
SED_REPLACE="sed -i "
case $(sed --help 2>&1) in
*GNU*) SED_REPLACE="sed -i ";;
@ -227,75 +352,119 @@ relx_start_command() {
"$START_OPTION"
}
trim() {
echo -e "${1}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'
}
# Function to generate app.config and vm.args
generate_config() {
## Delete the *.siz files first or it cann't start after
## changing the config 'log.rotation.size'
rm -rf "${RUNNER_LOG_DIR}"/*.siz
check_only="$1"
if [ "$check_only" != "check_only" ]; then
## Delete the *.siz files first or it cann't start after
## changing the config 'log.rotation.size'
rm -rf "${RUNNER_LOG_DIR}"/*.siz
fi
if [ "$CUTTLEFISH" != "yes" ]; then
# Note: we have added a parameter '-vm_args' to this. It
# appears redundant but it is not! the erlang vm allows us to
# access all arguments to the erl command EXCEPT '-args_file',
# so in order to get access to this file location from within
# the vm, we need to pass it in twice.
CONFIG_ARGS=" -config $RUNNER_ETC_DIR/app.config -args_file $RUNNER_ETC_DIR/vm.args -vm_args $RUNNER_ETC_DIR/vm.args "
set +e
if [ "${EMQX_LICENSE_CONF:-}" = "" ]; then
CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)"
else
EMQX_LICENSE_CONF_OPTION=""
if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then
EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}"
fi
CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -i "${EMQX_LICENSE_CONF}" -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)"
fi
# shellcheck disable=SC2181
RESULT=$?
set -e
if [ $RESULT -gt 0 ]; then
echo "$CUTTLEFISH_OUTPUT"
exit $RESULT
fi
## transform a single line args list like '-config ... -args_file ... -vm_args ...' to lines and get path for each file respectively
## NOTE: the -args_file and -vm_args are the same file passed twice because args_file is used by beam, but not possible to get at runtime
## by calling init:get_arguments/0
lines="$(echo "$CUTTLEFISH_OUTPUT" | tail -1 \
| sed -e $'s/-config/\\\nconfig=/g' \
| sed -e $'s/-args_file/\\\nargs_file=/g' \
| sed -e $'s/-vm_args/\\\nvm_args=/g')"
CONFIG_FILE="$(trim "$(echo -e "$lines" | grep 'config=' | sed 's/config=//g')")"
CUTTLE_GEN_ARG_FILE="$(trim "$(echo -e "$lines" | grep 'vm_args=' | sed 's/vm_args=//g')")"
set +e
# shellcheck disable=SC2086
CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)"
# shellcheck disable=SC2181
RESULT=$?
set -e
if [ $RESULT -gt 0 ]; then
echo "$CUTTLEFISH_OUTPUT"
exit $RESULT
fi
# print override from environment variables (EMQX_*)
echo "$CUTTLEFISH_OUTPUT" | sed -e '$d'
CONFIG_ARGS=$(echo "$CUTTLEFISH_OUTPUT" | tail -n 1)
## Merge cuttlefish generated *.args into the vm.args
CUTTLE_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}')
TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp"
cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE"
echo "" >> "$TMP_ARG_FILE"
echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE"
sed '/^#/d' "$CUTTLE_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do
ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}')
ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}')
if [ "$ARG_KEY" = '' ]; then
## for the flags, e.g. -heart -emu_args etc
ARG_KEY=$(echo "$ARG_LINE" | awk '{print $1}')
ARG_VALUE=''
TMP_ARG_KEY=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $1}')
if [ "$TMP_ARG_KEY" = '' ]; then
echo "$ARG_KEY" >> "$TMP_ARG_FILE"
fi
else
TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}')
if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then
if [ -n "$TMP_ARG_VALUE" ]; then
sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' $TMP_ARG_FILE"
else
echo "$ARG_LINE" >> "$TMP_ARG_FILE"
fi
## Merge cuttlefish generated *.args into the vm.args
TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp"
cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE"
echo "" >> "$TMP_ARG_FILE"
echo "-pa \"${REL_DIR}/consolidated\"" >> "$TMP_ARG_FILE"
sed '/^#/d' "$CUTTLE_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do
ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}')
ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}')
if [ "$ARG_KEY" = '' ]; then
## for the flags, e.g. -heart -emu_args etc
ARG_KEY=$(echo "$ARG_LINE" | awk '{print $1}')
ARG_VALUE=''
TMP_ARG_KEY=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $1}')
if [ "$TMP_ARG_KEY" = '' ]; then
echo "$ARG_KEY" >> "$TMP_ARG_FILE"
fi
else
TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}')
if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then
if [ -n "$TMP_ARG_VALUE" ]; then
sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' \"$TMP_ARG_FILE\""
else
echo "$ARG_LINE" >> "$TMP_ARG_FILE"
fi
fi
done
mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE"
fi
fi
done
# shellcheck disable=SC2086
if ! relx_nodetool chkconfig $CONFIG_ARGS; then
echoerr "Error reading $CONFIG_ARGS"
if ! relx_nodetool chkconfig -config "$CONFIG_FILE"; then
echoerr "Error reading $CONFIG_FILE"
exit 1
fi
if [ "$check_only" = "check_only" ]; then
rm -f "$TMP_ARG_FILE"
rm -f "$CUTTLE_GEN_ARG_FILE"
rm -f "$CONFIG_FILE"
else
mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE"
fi
}
# check if a PID is down
is_down() {
PID="$1"
if ps -p "$PID" >/dev/null; then
# still around
# shellcheck disable=SC2009 # this grep pattern is not a part of the progra names
if ps -p "$PID" | grep -q 'defunct'; then
# zombie state, print parent pid
parent="$(ps -o ppid= -p "$PID" | tr -d ' ')"
echo "WARN: $PID is marked <defunct>, parent:"
ps -p "$parent"
return 0
fi
return 1
fi
# it's gone
return 0
}
wait_for() {
local WAIT_TIME
local CMD
WAIT_TIME="$1"
shift
CMD="$*"
while true; do
if $CMD >/dev/null 2>&1; then
return 0
fi
if [ "$WAIT_TIME" -le 0 ]; then
return 1
fi
WAIT_TIME=$((WAIT_TIME - 1))
sleep 1
done
}
# Call bootstrapd for daemon commands like start/stop/console
@ -305,14 +474,38 @@ bootstrapd() {
fi
}
# Use $CWD/etc/sys.config if exists
if [ -z "$RELX_CONFIG_PATH" ]; then
if [ -f "$RUNNER_ETC_DIR/sys.config" ]; then
RELX_CONFIG_PATH="-config $RUNNER_ETC_DIR/sys.config"
else
RELX_CONFIG_PATH=""
# check if a PID is down
is_down() {
PID="$1"
if ps -p "$PID" >/dev/null; then
# still around
# shellcheck disable=SC2009 # this grep pattern is not a part of the progra names
if ps -p "$PID" | grep -q 'defunct'; then
return 0
fi
return 1
fi
fi
# it's gone
return 0
}
wait_for() {
local WAIT_TIME
local CMD
WAIT_TIME="$1"
shift
CMD="$*"
while true; do
if $CMD >/dev/null 2>&1; then
return 0
fi
if [ "$WAIT_TIME" -le 0 ]; then
return 1
fi
WAIT_TIME=$((WAIT_TIME - 1))
sleep 1
done
}
IS_BOOT_COMMAND='no'
case "$1" in
@ -325,8 +518,21 @@ case "$1" in
foreground)
IS_BOOT_COMMAND='yes'
;;
check_conf)
IS_BOOT_COMMAND='yes'
;;
esac
if [ "$IS_BOOT_COMMAND" = 'no' ]; then
# for non-boot commands, inspect vm.<time>.args for node name
# shellcheck disable=SC2012
LATEST_VM_ARGS_FILE="$(ls -t "$RUNNER_DATA_DIR"/configs/vm.*.args 2>/dev/null | head -1)"
if [ -z "$LATEST_VM_ARGS_FILE" ]; then
echoerr "There is no vm.*.args config file found in '$RUNNER_DATA_DIR/configs/'"
echoerr "Please make sure the node is initialized (started for at least once)"
exit 1
fi
fi
if [ -z "$NAME_ARG" ]; then
NODENAME="${EMQX_NODE_NAME:-}"
@ -334,14 +540,7 @@ if [ -z "$NAME_ARG" ]; then
[ -z "$NODENAME" ] && [ -n "$EMQX_NAME" ] && [ -n "$EMQX_HOST" ] && NODENAME="${EMQX_NAME}@${EMQX_HOST}"
if [ -z "$NODENAME" ]; then
if [ "$IS_BOOT_COMMAND" = 'no' ]; then
# for non-boot commands, inspect vm.<time>.args for node name
# shellcheck disable=SC2012,SC2086
LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)"
if [ -z "$LATEST_VM_ARGS" ]; then
echo "For command $1, there is no vm.*.args config file found in $RUNNER_DATA_DIR/configs/"
exit 1
fi
NODENAME="$(grep -E '^-name' "$LATEST_VM_ARGS" | awk '{print $2}')"
NODENAME="$(grep -E '^-name' "$LATEST_VM_ARGS_FILE" | awk '{print $2}')"
else
# for boot commands, inspect emqx.conf for node name
NODENAME=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.name)
@ -363,6 +562,10 @@ fi
NAME_TYPE="$(echo "$NAME_ARG" | awk '{print $1}')"
NAME="$(echo "$NAME_ARG" | awk '{print $2}')"
NODENAME="$(echo "$NAME" | awk -F'@' '{print $1}')"
if ! (echo "$NODENAME" | grep -q '^[0-9A-Za-z_\-]\+$'); then
echo "Invalid node name, should be of format '^[0-9A-Za-z_-]+$'."
exit 1
fi
export ESCRIPT_NAME="$NODENAME"
PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}"
@ -372,13 +575,7 @@ if [ -z "$COOKIE" ]; then
if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
COOKIE=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.cookie)
else
# shellcheck disable=SC2012,SC2086
LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)"
if [ -z "$LATEST_VM_ARGS" ]; then
echo "For command $1, there is no vm.*.args config file found in $RUNNER_DATA_DIR/configs/"
exit 1
fi
COOKIE="$(grep -E '^-setcookie' "$LATEST_VM_ARGS" | awk '{print $2}')"
COOKIE="$(grep -E '^-setcookie' "$LATEST_VM_ARGS_FILE" | awk '{print $2}')"
fi
fi
@ -387,14 +584,6 @@ if [ -z "$COOKIE" ]; then
exit 1
fi
# Support for IPv6 Dist. See: https://github.com/emqtt/emqttd/issues/1460
PROTO_DIST=$(grep -E '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}')
if [ -z "$PROTO_DIST" ]; then
PROTO_DIST_ARG=""
else
PROTO_DIST_ARG="-proto_dist $PROTO_DIST"
fi
cd "$ROOTDIR"
# User can specify an sname without @hostname
@ -461,7 +650,7 @@ case "$1" in
"$BINDIR/run_erl" -daemon "$PIPE_DIR" "$RUNNER_LOG_DIR" \
"$(relx_start_command)"
WAIT_TIME=${WAIT_FOR_ERLANG:-15}
WAIT_TIME=${WAIT_FOR_ERLANG:-150}
while [ "$WAIT_TIME" -gt 0 ]; do
if ! relx_nodetool "ping" >/dev/null 2>&1; then
WAIT_TIME=$((WAIT_TIME - 1))
@ -473,7 +662,7 @@ case "$1" in
echo "$EMQX_DESCRIPTION $REL_VSN is started successfully!"
exit 0
fi
done && echo "$EMQX_DESCRIPTION $REL_VSN failed to start within ${WAIT_FOR_ERLANG:-15} seconds,"
done && echo "$EMQX_DESCRIPTION $REL_VSN failed to start within ${WAIT_FOR_ERLANG:-150} seconds,"
echo "see the output of '$0 console' for more information."
echo "If you want to wait longer, set the environment variable"
echo "WAIT_FOR_ERLANG to the number of seconds to wait."
@ -484,11 +673,21 @@ case "$1" in
# Wait for the node to completely stop...
PID="$(relx_get_pid)"
if ! relx_nodetool "stop"; then
echoerr "Graceful shutdown failed PID=[$PID]"
exit 1
fi
while kill -s 0 "$PID" 2>/dev/null; do
sleep 1
done
WAIT_TIME="${EMQX_WAIT_FOR_STOP:-120}"
if ! wait_for "$WAIT_TIME" 'is_down' "$PID"; then
msg="dangling after ${WAIT_TIME} seconds"
# also log to syslog
logger -t "${REL_NAME}[${PID}]" "STOP: $msg"
# log to user console
echoerr "Stop failed, $msg"
echo "ERROR: $PID is still around"
ps -p "$PID"
exit 1
fi
logger -t "${REL_NAME}[${PID}]" "STOP: OK"
;;
restart|reboot)
@ -622,14 +821,17 @@ case "$1" in
# Store passed arguments since they will be erased by `set`
ARGS="$*"
# shellcheck disable=SC2086 # $RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace
# shellcheck disable=SC2086 # $EPMD_ARG is supposed to be split by whitespace
# Build an array of arguments to pass to exec later on
# Build it here because this command will be used for logging.
set -- "$BINDIR/erlexec" \
-boot "$BOOTFILE" -mode "$CODE_LOADING_MODE" \
-boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
-mnesia dir "\"${MNESIA_DATA_DIR}\"" \
$RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG
-config "$CONFIG_FILE" \
-args_file "$CUTTLE_GEN_ARG_FILE" \
-vm_args "$CUTTLE_GEN_ARG_FILE" \
$EPMD_ARG
# Log the startup
logger -t "${REL_NAME}[$$]" "$* -- ${1+$ARGS}"
@ -663,14 +865,17 @@ case "$1" in
# Store passed arguments since they will be erased by `set`
ARGS="$*"
# shellcheck disable=SC2086 # $RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace
# shellcheck disable=SC2086 # $EPMD_ARG is supposed to be split by whitespace
# Build an array of arguments to pass to exec later on
# Build it here because this command will be used for logging.
set -- "$BINDIR/erlexec" $FOREGROUNDOPTIONS \
-boot "$REL_DIR/$BOOTFILE" -mode "$CODE_LOADING_MODE" \
-boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
-mnesia dir "\"${MNESIA_DATA_DIR}\"" \
$RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG
-config "$CONFIG_FILE" \
-args_file "$CUTTLE_GEN_ARG_FILE" \
-vm_args "$CUTTLE_GEN_ARG_FILE" \
$EPMD_ARG
# Log the startup
logger -t "${REL_NAME}[$$]" "$* -- ${1+$ARGS}"
@ -681,50 +886,44 @@ case "$1" in
ertspath)
echo "$ERTS_PATH"
;;
rpc)
# Make sure a node IS running
if ! relx_nodetool "ping" > /dev/null; then
echo "Node is not running!"
exit 1
check_conf)
generate_config "check_only"
INCLUDE_CONFS=$(< "$RUNNER_ETC_DIR/emqx.conf" grep "include" |awk '{print $2}'|xargs)
if [ "$INCLUDE_CONFS" == "" ]; then
echo "$RUNNER_ETC_DIR/emqx.conf is ok"
else
echo "$RUNNER_ETC_DIR/emqx.conf(include $INCLUDE_CONFS) is ok"
fi
;;
ctl)
assert_node_alive
shift
relx_nodetool rpc_infinity emqx_ctl run_command "$@"
;;
rpc)
assert_node_alive
shift
relx_nodetool rpc "$@"
;;
rpcterms)
# Make sure a node IS running
if ! relx_nodetool "ping" > /dev/null; then
echo "Node is not running!"
exit 1
fi
assert_node_alive
shift
relx_nodetool rpcterms "$@"
;;
root_dir)
# Make sure a node IS running
if ! relx_nodetool "ping" > /dev/null; then
echo "Node is not running!"
exit 1
fi
assert_node_alive
shift
relx_nodetool "eval" 'code:root_dir()'
;;
eval)
# Make sure a node IS running
if ! relx_nodetool "ping" > /dev/null; then
echo "Node is not running!"
exit 1
fi
assert_node_alive
shift
relx_nodetool "eval" "$@"
;;
*)
relx_usage "$1"
usage "$COMMAND"
exit 1
;;
esac

24
bin/emqx.cmd
View File

@ -8,8 +8,10 @@
:: * restart - run the stop command and start command
:: * uninstall - uninstall the service and kill a running node
:: * ping - check if the node is running
:: * ctl - run management commands
:: * console - start the Erlang release in a `werl` Windows shell
:: * attach - connect to a running node and open an interactive console
:: * remote_console - same as attach
:: * list - display a listing of installed Erlang services
:: * usage - display available commands
@ -24,6 +26,9 @@
@set script=%~n0
@set EPMD_ARG=-start_epmd false -epmd_module ekka_epmd -proto_dist ekka
@set ERL_FLAGS=%EPMD_ARG%
:: Discover the release root directory from the directory
:: of this script
@set script_dir=%~dp0
@ -46,12 +51,12 @@
@set service_name=%rel_name%_%rel_vsn%
@set bindir=%erts_dir%\bin
@set progname=erl.exe
@set clean_boot_script=%rel_root_dir%\bin\start_clean
@set clean_boot_script=%rel_dir%\start_clean
@set erlsrv="%bindir%\erlsrv.exe"
@set epmd="%bindir%\epmd.exe"
@set escript="%bindir%\escript.exe"
@set werl="%bindir%\werl.exe"
@set erl_exe="%bindir%\erl.exe"
@set erl_exe="%bindir%\werl.exe"
@set nodetool="%rel_root_dir%\bin\nodetool"
@set cuttlefish="%rel_root_dir%\bin\cuttlefish"
@set node_type="-name"
@ -83,8 +88,10 @@
::@if "%1"=="downgrade" @goto relup
@if "%1"=="console" @goto console
@if "%1"=="ping" @goto ping
@if "%1"=="ctl" @goto ctl
@if "%1"=="list" @goto list
@if "%1"=="attach" @goto attach
@if "%1"=="remote_console" @goto attach
@if "%1"=="" @goto usage
@echo Unknown command: "%1"
@ -239,7 +246,7 @@ cd /d %rel_root_dir%
@echo off
cd /d %rel_root_dir%
@echo on
@start "bin\%rel_name% console" %werl% -boot "%boot_script%" %args%
%erl_exe% -boot "%boot_script%" %args%
@echo emqx is started!
@goto :eof
@ -248,6 +255,12 @@ cd /d %rel_root_dir%
@%escript% %nodetool% ping %node_type% "%node_name%" -setcookie "%node_cookie%"
@goto :eof
:: ctl to execute management commands
:ctl
@for /f "usebackq tokens=1*" %%i in (`echo %*`) DO @ set params=%%j
@%escript% %nodetool% %node_type% "%node_name%" -setcookie "%node_cookie%" rpc_infinity emqx_ctl run_command %params%
@goto :eof
:: List installed Erlang services
:list
@%erlsrv% list %service_name%
@ -255,9 +268,8 @@ cd /d %rel_root_dir%
:: Attach to a running node
:attach
:: @start "%node_name% attach"
@start "%node_name% attach" %werl% -boot "%clean_boot_script%" ^
-remsh %node_name% %node_type% console_%node_name% -setcookie %node_cookie%
:: @start "%node_name% attach"
%erl_exe% -boot "%clean_boot_script%" -remsh %node_name% %node_type% remsh_%node_name% -setcookie %node_cookie%
@goto :eof
:: Trim variable

187
bin/emqx_cluster_rescue Normal file
View File

@ -0,0 +1,187 @@
#!/usr/bin/env bash
set -euo pipefail
# ==================================
# RESCUE THE UNBOOTABLE EMQX CLUSTER
# ==================================
## Global Vars
# Steal from emqx_ctl
THIS_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")" || true; pwd -P)"
usage() {
local Script
Script=$(basename "$0")
echo "
RESCUE THE UNBOOTABLE EMQX CLUSTER
Use this script only when the entire cluster is stuck at booting & loading.
This script provides a list of methods to *hack* the DB of EMQX to bring back
the cluster back to service but MAY come with some side effects including:
- Data loss
- Inconsistent data in the cluster
- Other undefined behaviors
*DO NOT* use this script unless you understand the consequences.
*DO NOT* use this script when EMQX cluster is partitioned.
Use Case:
- Lost one node due to unrecoverable failures (hardware, cloud resource outage)
and this node prevents other nodes in the cluster from starting.
Usage:
# For troubleshooting, find out all the tables that are pending at loading
$Script pending-tables
# For troubleshooting, debug print detailed table info that is pending at loading.
$Script table-details
# Force load one [Tab] or all pending tables from node local storage to bring this node up
# Use local data as the data source for the pending tables, should bring up the node immediately and
# spread the data to other nodes in the cluster.
#
# * Take effect immediately
# * This is a node local change but the change will be lost after restart.
$Script force-load [Tab]
# Remove Node from mnesia cluster.
# Most likely will fail if the remote Node is unreachable.
#
# * This is a cluster wide schema change.
$Script remove-node Node
# Set master node for distributed DB
# The master node will be the data source for pending tables.
#
# * This is a node local change
# * Node could be a remote Erlang node in the cluster or local erlang node
# * Use command: 'unset-master' to rollback
$Script set-master Node
# Unset master node for distributed DB, this is a node local change
$Script unset-master
# Cheat the local node that RemoteNode is down so that it will not wait for it to come up.
# Local node will take local data as the data source for pending tables and spread the data
# to the other pending nodes.
#
# * Check EMQX logs to find out which remote node(s) the local node is waiting for
# * To take effect, restart this EMQX node
# * This is a node local setting
$Script lie-node-down RemoteNode
Tips:
- Override local node name with envvar: \$EMQX_NODE_NAME
"
}
# Functions
#
print_pending_tables() {
local erl_cmd='[ io:format("~p :: ~p~n", [T, maps:with([all_nodes, load_order, storage_type,
active_replicas, local_content, load_by_force,
load_node, load_reason, master_nodes]
, maps:from_list(mnesia:table_info(T, all)))])
|| T <- mnesia:system_info(local_tables), unknown =:= mnesia:table_info(T, load_node) ],
ok
'
exec "$THIS_DIR/emqx" eval "$erl_cmd"
}
print_details_per_table() {
local erl_cmd='[ io:format("~p :: ~p~n", [T, mnesia:table_info(T, all)])
|| T <- mnesia:system_info(local_tables), unknown =:= mnesia:table_info(T, load_node) ],
ok
'
exec "$THIS_DIR/emqx" eval "$erl_cmd"
}
force-load() {
if [ $# -eq 1 ]; then
local erl_cmd="mnesia:force_load_table(${1})"
else
local erl_cmd='[ {T, mnesia:force_load_table(T)}
|| T <- mnesia:system_info(local_tables),
unknown =:= mnesia:table_info(T, load_node)
]
'
fi
exec "$THIS_DIR/emqx" eval "$erl_cmd"
}
remove-node() {
local target_node=$1
local erl_cmd="
case [T || T <- mnesia:system_info(local_tables), unknown =:= mnesia:table_info(T, load_node)] of
[] ->
io:format(\"No table need to load\\n\"),
skipped;
TargetTables ->
io:format(\"Going to remove node ${target_node} from schema of the tables:~n~p~n\", [TargetTables]),
case io:read(\"confirm? [yes.] OR Ctrl-D to skip: \") of
{ok, yes} ->
lists:map(fun(T) ->
mnesia:force_load_table(T),
{T, mnesia:del_table_copy(T, '${target_node}') }
end, TargetTables);
eof -> skipped;
R -> {skipped, R}
end
end
"
exec "$THIS_DIR/emqx" eval "$erl_cmd"
}
set-master-node() {
if [ $# -eq 1 ]; then
local erl_cmd="mnesia:set_master_nodes(['${1}']), mnesia_recover:dump_decision_tab()"
else
local erl_cmd="mnesia:set_master_nodes([]), mnesia_recover:dump_decision_tab()"
fi
exec "$THIS_DIR/emqx" eval "$erl_cmd"
}
lie-node-down() {
if [ $# -eq 1 ]; then
local erl_cmd="mnesia_recover:log_mnesia_down('${1}'), mnesia_recover:dump_decision_tab()"
exec "$THIS_DIR/emqx" eval "$erl_cmd"
else
usage
fi
}
CMD=${1:-usage}
[ $# -gt 0 ] && shift 1
case "$CMD" in
force-load)
force-load "$@"
;;
remove-node)
remove-node "$@"
;;
pending-tables)
print_pending_tables
;;
table-details)
print_details_per_table
;;
set-master)
set-master-node "$@"
;;
unset-master)
set-master-node
;;
lie-node-down)
lie-node-down "$@"
;;
*)
usage
esac

84
bin/emqx_ctl
View File

@ -1,84 +1,6 @@
#!/bin/bash
#!/usr/bin/env bash
# -*- tab-width:4;indent-tabs-mode:nil -*-
# ex: ts=4 sw=4 et
set -e
ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
# shellcheck disable=SC1090
. "$ROOT_DIR"/releases/emqx_vars
export RUNNER_ROOT_DIR
export REL_VSN
# shellcheck disable=SC2012,SC2086
LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)"
if [ -z "$LATEST_VM_ARGS" ]; then
echo "No vm.*.args config file found in $RUNNER_DATA_DIR/configs/"
exit 1
fi
# Echo to stderr on errors
echoerr() { echo "$@" 1>&2; }
if [ -z "$WITH_EPMD" ]; then
EPMD_ARG="-start_epmd false -epmd_module ekka_epmd -proto_dist ekka"
else
EPMD_ARG="-start_epmd true"
fi
relx_get_nodename() {
id="longname$(relx_gen_id)-${NAME}"
"$BINDIR/erl" -boot start_clean -eval '[Host] = tl(string:tokens(atom_to_list(node()),"@")), io:format("~s~n", [Host]), halt()' -noshell "${NAME_TYPE}" "$id"
}
# Control a node
relx_nodetool() {
command="$1"; shift
ERL_FLAGS="$ERL_FLAGS $EPMD_ARG $PROTO_DIST_ARG" \
ERL_LIBS="${LIB_EKKA_DIR}:${ERL_LIBS:-}" \
"$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \
-setcookie "$COOKIE" "$command" "$@"
}
if [ -z "$NAME_ARG" ]; then
NODENAME="${EMQX_NODE_NAME:-}"
[ -z "$NODENAME" ] && [ -n "$EMQX_NAME" ] && [ -n "$EMQX_HOST" ] && NODENAME="${EMQX_NAME}@${EMQX_HOST}"
[ -z "$NODENAME" ] && NODENAME="$(grep -E '^-name' "$LATEST_VM_ARGS" | awk '{print $2}')"
if [ -z "$NODENAME" ]; then
echoerr "vm.args needs to have a -name parameter."
echoerr " -sname is not supported."
echoerr "perhaps you do not have read permissions on $RUNNER_ETC_DIR/emqx.conf"
exit 1
else
NAME_ARG="-name ${NODENAME# *}"
fi
fi
# Extract the name type and name from the NAME_ARG for REMSH
NAME_TYPE="$(echo "$NAME_ARG" | awk '{print $1}')"
NAME="$(echo "$NAME_ARG" | awk '{print $2}')"
COOKIE="${EMQX_NODE_COOKIE:-}"
[ -z "$COOKIE" ] && COOKIE="$(grep -E '^-setcookie' "$LATEST_VM_ARGS" | awk '{print $2}')"
if [ -z "$COOKIE" ]; then
echoerr "Please set node.cookie in $RUNNER_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE_COOKIE"
exit 1
fi
# Support for IPv6 Dist. See: https://github.com/emqtt/emqttd/issues/1460
PROTO_DIST=$(grep -E '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' "$RUNNER_ETC_DIR"/emqx.conf 2> /dev/null | tail -1 | awk -F"= " '{print $NF}')
if [ -z "$PROTO_DIST" ]; then
PROTO_DIST_ARG=""
else
PROTO_DIST_ARG="-proto_dist $PROTO_DIST"
fi
export ROOTDIR="$RUNNER_ROOT_DIR"
export ERTS_DIR="$ROOTDIR/erts-$ERTS_VSN"
export BINDIR="$ERTS_DIR/bin"
cd "$ROOTDIR"
relx_nodetool rpc emqx_ctl run_command "$@"
THIS_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")" || true; pwd -P)"
exec "$THIS_DIR/emqx" ctl "$@"

83
bin/emqx_ctl.cmd
View File

@ -2,91 +2,10 @@
@set args=%*
:: Set variables that describe the release
@set rel_name=emqx
@set rel_vsn={{ release_version }}
@set erts_vsn={{ erts_vsn }}
@set erl_opts={{ erl_opts }}
:: Discover the release root directory from the directory
:: of this script
@set script_dir=%~dp0
@for %%A in ("%script_dir%\..") do @(
set rel_root_dir=%%~fA
)
@set rel_dir=%rel_root_dir%\releases\%rel_vsn%
@set emqx_conf=%rel_root_dir%\etc\emqx.conf
@call :find_erts_dir
@set bindir=%erts_dir%\bin
@set progname=erl.exe
@set escript="%bindir%\escript.exe"
@set nodetool="%rel_root_dir%\bin\nodetool"
@set node_type="-name"
:: Extract node name from emqx.conf
@for /f "usebackq delims=\= tokens=2" %%I in (`findstr /b node\.name "%emqx_conf%"`) do @(
@call :set_trim node_name %%I
)
:: Extract node cookie from emqx.conf
@for /f "usebackq delims=\= tokens=2" %%I in (`findstr /b node\.cookie "%emqx_conf%"`) do @(
@call :set_trim node_cookie= %%I
)
:: Write the erl.ini file to set up paths relative to this script
@call :write_ini
:: If a start.boot file is not present, copy one from the named .boot file
@if not exist "%rel_dir%\start.boot" (
copy "%rel_dir%\%rel_name%.boot" "%rel_dir%\start.boot" >nul
)
@%escript% %nodetool% %node_type% "%node_name%" -setcookie "%node_cookie%" rpc emqx_ctl run_command %args%
:: Find the ERTS dir
:find_erts_dir
@set possible_erts_dir=%rel_root_dir%\erts-%erts_vsn%
@if exist "%possible_erts_dir%" (
call :set_erts_dir_from_default
) else (
call :set_erts_dir_from_erl
)
@goto :eof
:: Set the ERTS dir from the passed in erts_vsn
:set_erts_dir_from_default
@set erts_dir=%possible_erts_dir%
@set rootdir=%rel_root_dir%
@goto :eof
:: Set the ERTS dir from erl
:set_erts_dir_from_erl
@for /f "delims=" %%i in ('where erl') do @(
set erl=%%i
)
@set dir_cmd="%erl%" -noshell -eval "io:format(\"~s\", [filename:nativename(code:root_dir())])." -s init stop
@for /f %%i in ('%%dir_cmd%%') do @(
set erl_root=%%i
)
@set erts_dir=%erl_root%\erts-%erts_vsn%
@set rootdir=%erl_root%
@goto :eof
:: Write the erl.ini file
:write_ini
@set erl_ini=%erts_dir%\bin\erl.ini
@set converted_bindir=%bindir:\=\\%
@set converted_rootdir=%rootdir:\=\\%
@echo [erlang] > "%erl_ini%"
@echo Bindir=%converted_bindir% >> "%erl_ini%"
@echo Progname=%progname% >> "%erl_ini%"
@echo Rootdir=%converted_rootdir% >> "%erl_ini%"
@goto :eof
:: Trim variable
:set_trim
@set %1=%2
@goto :eof
@%rel_root_dir%\bin\emqx.cmd ctl %args%

138
bin/install_upgrade.escript
View File

@ -5,6 +5,9 @@
-define(TIMEOUT, 300000).
-define(INFO(Fmt,Args), io:format(Fmt++"~n",Args)).
-define(SEMVER_RE, <<"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[a-zA-Z\\d][-a-zA-Z.\\d]*)?(\\+[a-zA-Z\\d][-a-zA-Z.\\d]*)?$">>).
-mode(compile).
main([Command0, DistInfoStr | CommandArgs]) ->
%% convert the distribution info arguments string to an erlang term
@ -52,6 +55,7 @@ unpack(_, Args) ->
install({RelName, NameTypeArg, NodeName, Cookie}, Opts) ->
TargetNode = start_distribution(NodeName, NameTypeArg, Cookie),
Version = proplists:get_value(version, Opts),
validate_target_version(Version, TargetNode),
case unpack_release(RelName, TargetNode, Version) of
{ok, Vsn} ->
?INFO("Unpacked successfully: ~p.", [Vsn]),
@ -107,6 +111,7 @@ uninstall({_RelName, NameTypeArg, NodeName, Cookie}, Opts) ->
TargetNode = start_distribution(NodeName, NameTypeArg, Cookie),
WhichReleases = which_releases(TargetNode),
Version = proplists:get_value(version, Opts),
ensure_latest_boot_file(current_release_version(TargetNode)),
case proplists:get_value(Version, WhichReleases) of
undefined ->
?INFO("Release ~s is already uninstalled.", [Version]);
@ -142,9 +147,10 @@ parse_arguments([VersionStr|Rest], Acc) ->
parse_arguments(Rest, [{version, Version}] ++ Acc).
unpack_release(RelName, TargetNode, Version) ->
StartScriptExists = filelib:is_dir(filename:join(["releases", Version, "start.boot"])),
WhichReleases = which_releases(TargetNode),
case proplists:get_value(Version, WhichReleases) of
undefined ->
Res when Res =:= undefined; (Res =:= unpacked andalso not StartScriptExists) ->
%% not installed, so unpack tarball:
%% look for a release package with the intended version in the following order:
%% releases/<relname>-<version>.tar.gz
@ -159,10 +165,39 @@ unpack_release(RelName, TargetNode, Version) ->
case rpc:call(TargetNode, release_handler, unpack_release,
[ReleasePackageLink], ?TIMEOUT) of
{ok, Vsn} -> {ok, Vsn};
{error, {existing_release, Vsn}} ->
%% sometimes the user may have removed the release/<vsn> dir
%% for an `unpacked` release, then we need to re-unpack it from
%% the .tar ball
untar_for_unpacked_release(str(RelName), Vsn),
{ok, Vsn};
{error, _} = Error -> Error
end
end;
Other -> Other
Other ->
Other
end.
untar_for_unpacked_release(RelName, Vsn) ->
{ok, Root} = file:get_cwd(),
RelDir = filename:join([Root, "releases"]),
%% untar the .tar file, so release/<vsn> will be created
Tar = filename:join([RelDir, Vsn, RelName ++ ".tar.gz"]),
extract_tar(Root, Tar),
%% create RELEASE file
RelFile = filename:join([RelDir, Vsn, RelName ++ ".rel"]),
release_handler:create_RELEASES(Root, RelFile),
%% Clean release
_ = file:delete(Tar),
_ = file:delete(RelFile).
extract_tar(Cwd, Tar) ->
case erl_tar:extract(Tar, [keep_old_files, {cwd, Cwd}, compressed]) of
ok -> ok;
{error, {Name, Reason}} -> % New erl_tar (R3A).
throw({error, {cannot_extract_file, Name, Reason}})
end.
%% 1. look for a release package tarball with the provided version in the following order:
@ -210,30 +245,45 @@ find_and_link_release_package(Version, RelName) ->
ok = filelib:ensure_dir(filename:join([filename:dirname(ReleaseLink), "dummy"])),
%% create the symlink pointing to the full path name of the
%% release package we found
case file:make_symlink(filename:absname(Filename), ReleaseLink) of
ok ->
ok;
{error, eperm} -> % windows!
{ok,_} = file:copy(filename:absname(Filename), ReleaseLink)
end,
make_symlink_or_copy(filename:absname(Filename), ReleaseLink),
{Filename, ReleaseHandlerPackageLink}
end.
make_symlink_or_copy(Filename, ReleaseLink) ->
case file:make_symlink(Filename, ReleaseLink) of
ok -> ok;
{error, eexist} ->
?INFO("symlink ~p already exists, recreate it", [ReleaseLink]),
ok = file:delete(ReleaseLink),
make_symlink_or_copy(Filename, ReleaseLink);
{error, Reason} when Reason =:= eperm; Reason =:= enotsup ->
{ok, _} = file:copy(Filename, ReleaseLink);
{error, Reason} ->
?INFO("create symlink ~p failed", [ReleaseLink]),
error({Reason, ReleaseLink})
end.
unpack_zipballs(RelNameStr, Version) ->
{ok, Cwd} = file:get_cwd(),
GzFile = filename:absname(filename:join(["releases", RelNameStr ++ "-" ++ Version ++ ".tar.gz"])),
ZipFiles = filelib:wildcard(filename:join(["releases", RelNameStr ++ "-*" ++ Version ++ "*.zip"])),
?INFO("unzip ~p", [ZipFiles]),
[begin
TmdTarD="/tmp/emqx_untar_" ++ integer_to_list(erlang:system_time()),
ok = filelib:ensure_dir(filename:join([TmdTarD, "dummy"])),
{ok, _} = file:copy(Zip, filename:join([TmdTarD, "emqx.zip"])),
ok = file:set_cwd(filename:join([TmdTarD])),
{ok, _FileList} = zip:unzip("emqx.zip"),
ok = file:set_cwd(filename:join([TmdTarD, "emqx"])),
ok = erl_tar:create(GzFile, filelib:wildcard("*"), [compressed])
end || Zip <- ZipFiles],
file:set_cwd(Cwd).
try
GzFile = filename:absname(filename:join(["releases", RelNameStr ++ "-" ++ Version ++ ".tar.gz"])),
ZipFiles = filelib:wildcard(filename:join(["releases", RelNameStr ++ "-*" ++ Version ++ "*.zip"])),
?INFO("unzip ~p", [ZipFiles]),
lists:foreach(
fun(Zip) ->
TmdTarD = "/tmp/emqx_untar_" ++ integer_to_list(erlang:system_time()),
ok = filelib:ensure_dir(filename:join([TmdTarD, "dummy"])),
{ok, _} = file:copy(Zip, filename:join([TmdTarD, "emqx.zip"])),
ok = file:set_cwd(filename:join([TmdTarD])),
{ok, _FileList} = zip:unzip("emqx.zip"),
ok = file:set_cwd(filename:join([TmdTarD, "emqx"])),
ok = erl_tar:create(GzFile, filelib:wildcard("*"), [compressed])
end,
ZipFiles)
after
% restore cwd
ok = file:set_cwd(Cwd)
end.
first_value(_Fun, []) -> no_value;
first_value(Fun, [Value | Rest]) ->
@ -313,6 +363,9 @@ permafy(TargetNode, RelName, Vsn) ->
[{ok, _} = file:copy(filename:join(["bin", File++"-"++Vsn]),
filename:join(["bin", File]))
|| File <- Scripts],
ensure_latest_boot_file(Vsn),
%% update the vars
UpdatedVars = io_lib:format("REL_VSN=\"~s\"~nERTS_VSN=\"~s\"~n", [Vsn, erts_vsn()]),
file:write_file(filename:absname(filename:join(["releases", "emqx_vars"])), UpdatedVars, [append]).
@ -351,8 +404,7 @@ start_distribution(TargetNode, NameTypeArg, Cookie) ->
MyNode = make_script_node(TargetNode),
{ok, _Pid} = net_kernel:start([MyNode, get_name_type(NameTypeArg)]),
erlang:set_cookie(node(), Cookie),
case {net_kernel:connect_node(TargetNode),
net_adm:ping(TargetNode)} of
case {net_kernel:hidden_connect_node(TargetNode), net_adm:ping(TargetNode)} of
{true, pong} ->
ok;
{_, pang} ->
@ -365,7 +417,7 @@ start_distribution(TargetNode, NameTypeArg, Cookie) ->
make_script_node(Node) ->
[Name, Host] = string:tokens(atom_to_list(Node), "@"),
list_to_atom(lists:concat([Name, "_upgrader_", os:getpid(), "@", Host])).
list_to_atom(lists:concat(["remsh_", Name, "_upgrader_", os:getpid(), "@", Host])).
%% get name type from arg
get_name_type(NameTypeArg) ->
@ -380,3 +432,41 @@ erts_vsn() ->
{ok, Str} = file:read_file(filename:join(["releases", "start_erl.data"])),
[ErtsVsn, _] = string:tokens(binary_to_list(Str), " "),
ErtsVsn.
validate_target_version(TargetVersion, TargetNode) ->
CurrentVersion = current_release_version(TargetNode),
case {get_major_minor_vsn(CurrentVersion), get_major_minor_vsn(TargetVersion)} of
{{Major, Minor}, {Major, Minor}} -> ok;
{{<<"4">>, <<"5">>}, {<<"4">>, <<"4">>}} -> ok;
{{<<"4">>, <<"4">>}, {<<"4">>, <<"5">>}} -> ok;
_ ->
?INFO("Cannot upgrade/downgrade to ~s from ~s~n"
"We only support relup between patch versions",
[TargetVersion, CurrentVersion]),
error({relup_not_allowed, unsupported_target_version})
end.
get_major_minor_vsn(Version) ->
Parts = parse_semver(Version),
[Major | Rem0] = Parts,
[Minor | _Rem1] = Rem0,
{Major, Minor}.
parse_semver(Version) ->
case re:run(Version, ?SEMVER_RE, [{capture, all_but_first, binary}]) of
{match, Parts} -> Parts;
nomatch -> error({invalid_semver, Version})
end.
str(A) when is_atom(A) ->
atom_to_list(A);
str(A) when is_binary(A) ->
binary_to_list(A);
str(A) when is_list(A) ->
(A).
%%% For otp upgrade, the boot file need match the otp version
%%% This is for emqx cli tool
ensure_latest_boot_file(Vsn) ->
{ok, _} = file:copy(filename:join(["releases", Vsn, "no_dot_erlang.boot"]),
filename:join(["bin", "no_dot_erlang.boot"])).

34
bin/node_dump
View File

@ -1,15 +1,17 @@
#!/bin/sh
set -eu
ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
echo "Running node dump in ${RUNNER_ROOT_DIR}"
echo "Running node dump in ${ROOT_DIR}"
# shellcheck disable=SC1090
. "$RUNNER_ROOT_DIR"/releases/emqx_vars
cd "${ROOT_DIR}"
cd "${RUNNER_ROOT_DIR}"
DUMP="log/node_dump_$(date +"%Y%m%d_%H%M%S").tar.gz"
CONF_DUMP="log/conf.dump"
SYSINFO="log/sysinfo.txt"
DUMP="$RUNNER_LOG_DIR/node_dump_$(date +"%Y%m%d_%H%M%S").tar.gz"
CONF_DUMP="$RUNNER_LOG_DIR/conf.dump"
SYSINFO="$RUNNER_LOG_DIR/sysinfo.txt"
LOG_MAX_AGE_DAYS=3
@ -22,11 +24,11 @@ collect() {
}
show_help() {
echo "Collect information about the EMQ X node
echo "Collect information about the EMQX node
USAGE:
bin/node_dump [-a DAYS]
$0 [-a DAYS]
OPTIONS:
@ -44,29 +46,29 @@ done
# Collect system info:
{
collect bin/emqx_ctl broker
collect bin/emqx eval "'emqx_node_dump:sys_info()'"
collect "$RUNNER_BIN_DIR"/emqx_ctl broker
collect "$RUNNER_BIN_DIR"/emqx eval "'emqx_node_dump:sys_info()'"
collect uname -a
collect uptime
collect free
collect netstat -tnl
collect bin/emqx_ctl plugins list
collect bin/emqx_ctl modules list
collect "$RUNNER_BIN_DIR"/emqx_ctl plugins list
collect "$RUNNER_BIN_DIR"/emqx_ctl modules list
collect bin/emqx_ctl vm all
collect bin/emqx_ctl listeners
collect "$RUNNER_BIN_DIR"/emqx_ctl vm all
collect "$RUNNER_BIN_DIR"/emqx_ctl listeners
} > "${SYSINFO}"
# Collect information about the configuration:
{
collect bin/emqx eval "'emqx_node_dump:app_env_dump()'"
collect "$RUNNER_BIN_DIR"/emqx eval "'emqx_node_dump:app_env_dump()'"
} > "${CONF_DUMP}"
# Pack files
{
find log -mtime -"${LOG_MAX_AGE_DAYS}" \( -name '*.log.*' -or -name 'run_erl.log*' \)
find "$RUNNER_LOG_DIR" -mtime -"${LOG_MAX_AGE_DAYS}" \( -name '*.log.*' -or -name 'run_erl.log*' \)
echo "${SYSINFO}"
echo "${CONF_DUMP}"
} | tar czf "${DUMP}" -T -

269
build
View File

@ -1,9 +1,12 @@
#!/bin/bash
#!/usr/bin/env bash
# This script helps to build release artifacts.
# arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
# arg2: artifact, e.g. rel | relup | zip | pkg
if [[ -n "$DEBUG" ]]; then
set -x
fi
set -euo pipefail
PROFILE="$1"
@ -15,21 +18,7 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
export PKG_VSN
SYSTEM="$(./scripts/get-distro.sh)"
ARCH="$(uname -m)"
case "$ARCH" in
x86_64)
ARCH='amd64'
;;
aarch64)
ARCH='arm64'
;;
arm*)
ARCH=arm
;;
esac
export ARCH
SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
##
## Support RPM and Debian based linux systems
@ -45,6 +34,28 @@ if [ "$(uname -s)" = 'Linux' ]; then
esac
fi
if [ "${SYSTEM}" = 'windows' ]; then
# windows does not like the find
FIND="/usr/bin/find"
else
FIND='find'
fi
UNAME="$(uname -m)"
case "$UNAME" in
x86_64)
ARCH='amd64'
;;
aarch64)
ARCH='arm64'
;;
arm*)
ARCH=arm
;;
esac
# used in -pkg Makefile
export ARCH
log() {
local msg="$1"
# rebar3 prints ===>, so we print ===<
@ -56,27 +67,43 @@ make_rel() {
./rebar3 as "$PROFILE" do release,tar
}
relup_db() {
case "$PROFILE" in
*-ee*)
./scripts/relup-base-vsns.escript "$@" ./data/relup-paths-ee.eterm
;;
*)
./scripts/relup-base-vsns.escript "$@" ./data/relup-paths.eterm
;;
esac
}
## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup
make_relup() {
local lib_dir="_build/$PROFILE/rel/emqx/lib"
local releases_dir="_build/$PROFILE/rel/emqx/releases"
mkdir -p "$lib_dir" "$releases_dir"
local zip_file
mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base'
local releases=()
local OTP_CHANGED='no'
if [ -d "$releases_dir" ]; then
while read -r zip; do
local base_vsn
base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-e]{8})?")"
if [ ! -d "$releases_dir/$base_vsn" ]; then
for BASE_VSN in $(relup_db base-vsns "$PKG_VSN"); do
OTP_BASE=$(relup_db otp-vsn-for "$BASE_VSN")
if [[ "$OTP_BASE" != "$OTP_VSN" ]]; then
OTP_CHANGED='yes'
fi
zip_file="_upgrade_base/${PROFILE}-$(env OTP_VSN="$OTP_BASE" PKG_VSN="$BASE_VSN" ./scripts/pkg-full-vsn.sh 'vsn_exact').zip"
if [ ! -d "$releases_dir/$BASE_VSN" ]; then
local tmp_dir
tmp_dir="$(mktemp -d -t emqx.XXXXXXX)"
unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir"
unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir"
cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir"
cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir"
unzip -q "$zip_file" "emqx/releases/*" -d "$tmp_dir"
unzip -q "$zip_file" "emqx/lib/*" -d "$tmp_dir"
cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" || true
cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" || true
rm -rf "$tmp_dir"
fi
releases+=( "$base_vsn" )
done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f)
releases+=( "$BASE_VSN" )
done
fi
if [ ${#releases[@]} -eq 0 ]; then
log "No upgrade base found, relup ignored"
@ -84,9 +111,41 @@ make_relup() {
fi
RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")"
export RELX_BASE_VERSIONS
if [[ ${PKG_VSN} == 4.[3,4]* && ${OTP_CHANGED} == 'yes' ]]; then
echo "EMQX 4.[3,4] specific, overwrite OTP app versions"
local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel"
if [ ! -f "${emqx_rel_file}" ]; then
./rebar3 as "${PROFILE}" release
fi
scripts/emqx_rel_otp_app_overwrite.escript "${releases_dir}" "${PKG_VSN}" "${RELX_BASE_VERSIONS}"
fi
./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}"
# assert that there is no 'restart_emulator' in relup
# EMQX wants hot upgrade without VM restart
if grep restart_emulator "${releases_dir}/${PKG_VSN}/relup"; then
echo "Error: restart_emulator instruction found in relup";
exit 1
fi
# rollback rel file per releases
#
if [[ ${PKG_VSN} == 4.[3,4]* && ${OTP_CHANGED} == 'yes' ]]; then
echo "restore upgrade base rel files... "
for rel in ${releases[*]};
do
bakfile="${releases_dir}/${rel}/${PROFILE}.rel.bak"
echo "restore $bakfile ..."
if [ -f "$bakfile" ]; then
echo "restore from $bakfile"
mv "${bakfile}" "${bakfile%.bak}"
fi
done
fi
}
## try to be portable for zip packages.
## for DEB and RPM packages the dependencies are resoved by yum and apt
cp_dyn_libs() {
local rel_dir="$1"
local target_dir="${rel_dir}/dynlibs"
@ -96,36 +155,158 @@ cp_dyn_libs() {
mkdir -p "$target_dir"
while read -r so_file; do
cp -L "$so_file" "$target_dir/"
done < <(find "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \
done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \
| xargs -0 ldd \
| grep -E '(libcrypto)|(libtinfo)' \
| awk '{print $3}' \
| sort -u)
}
## make_zip turns .tar.gz into a .zip with a slightly different name.
## It assumes the .tar.gz has been built -- relies on Makefile dependency
make_zip() {
# build the tarball again to ensure relup is included
make_rel
tard="/tmp/emqx_untar_${PKG_VSN}"
rm -rf "${tard}"
# use relative path because abs path is tricky in windows
tard="tmp/zip-wd-${PKG_VSN}"
rm -rf "${tard}/emqx"
mkdir -p "${tard}/emqx"
local relpath="_build/${PROFILE}/rel/emqx"
local pkgpath="_packages/${PROFILE}"
local pkgname
pkgname="${PROFILE}-$(./scripts/pkg-full-vsn.sh).zip"
mkdir -p "${pkgpath}"
local tarball="${relpath}/emqx-${PKG_VSN}.tar.gz"
if [ ! -f "$tarball" ]; then
log "ERROR: $tarball is not found"
fi
local zipball
zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip"
local tarname="emqx-${PKG_VSN}.tar.gz"
local tarball="${relpath}/${tarname}"
local target_zip="${pkgpath}/${pkgname}"
tar zxf "${tarball}" -C "${tard}/emqx"
## try to be portable for zip packages.
## for DEB and RPM packages the dependencies are resoved by yum and apt
has_relup='yes'
case "$SYSTEM" in
windows*)
# no relup support for windows
has_relup='no'
;;
debian11)
case "$PKG_VSN" in
4.4.2*)
# this is the first version for debian11, no relup
has_relup='no'
;;
esac
;;
esac
# shellcheck disable=SC2207
bases=($(relup_db base-vsns "$PKG_VSN"))
if [[ "${#bases[@]}" -eq 0 ]]; then
has_relup='no'
fi
if [ "$has_relup" = 'yes' ]; then
./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup"
fi
cp_dyn_libs "${tard}/emqx"
(cd "${tard}" && zip -qr - emqx) > "${zipball}"
case "$SYSTEM" in
macos*)
# if the flag to sign macos binaries is set, but developer certificate
# or certificate password is not configured, reset the flag
# could happen, for example, when people submit PR from a fork, in this
# case they cannot access secrets
if [[ "${APPLE_SIGN_BINARIES:-0}" == 1 && \
( "${APPLE_DEVELOPER_ID_BUNDLE:-0}" == 0 || \
"${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD:-0}" == 0 ) ]]; then
echo "Apple developer certificate is not configured, skip signing"
APPLE_SIGN_BINARIES=0
fi
if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
./scripts/macos-sign-binaries.sh "${tard}/emqx"
fi
(cd "${tard}" && zip -qr - emqx) > "${target_zip}"
if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
# notarize the package
# if fails, you can check what went wrong with this command:
# xcrun notarytool log --apple-id <apple id> \
# --apple-id <apple id> \
# --password <apple id password>
# --team-id <apple team id> <submission-id>
xcrun notarytool submit \
--apple-id "${APPLE_ID}" \
--password "${APPLE_ID_PASSWORD}" \
--team-id "${APPLE_TEAM_ID}" "${target_zip}" --wait
fi
# sha256sum may not be available on macos
openssl dgst -sha256 "${target_zip}" | cut -d ' ' -f 2 > "${target_zip}.sha256"
;;
windows*)
pushd "${tard}" >/dev/null
7z a "${pkgname}" emqx
popd >/dev/null
mv "${tard}/${pkgname}" "${target_zip}"
sha256sum "${target_zip}" | head -c 64 > "${target_zip}.sha256"
;;
*)
(cd "${tard}" && zip -qr - emqx) > "${target_zip}"
sha256sum "${target_zip}" | head -c 64 > "${target_zip}.sha256"
;;
esac
log "Zip package successfully created: ${target_zip}"
log "Zip package sha256sum: $(cat "${target_zip}.sha256")"
}
## This function builds the default docker image
## based images is by default $EMQX_DEFAULT_BUILDER (see Makefile)
make_docker() {
EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}"
EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
set -x
docker build --no-cache --pull \
--build-arg BUILD_FROM="${EMQX_BUILDER}" \
--build-arg RUN_FROM="${EMQX_RUNNER}" \
--build-arg EMQX_NAME="$PROFILE" \
--tag "emqx/$PROFILE:${PKG_VSN}" \
-f "${DOCKERFILE}" .
}
## This function accepts any base docker image,
## a emqx zip-image, and a image tag (for the image to be built),
## to build a docker image which runs EMQ X
##
## Export below variables to quickly build an image
##
## Name Default Example
## ---------------------------------------------------------------------
## EMQX_BASE_IMAGE current os centos:7
## EMQX_ZIP_PACKAGE _packages/<current-zip-target> /tmp/emqx-4.4.0-otp24.3.4.2-1-el7-amd64.zip
## EMQX_IMAGE_TAG emqx/emqx:<current-vns-rel> emqx/emqx:testing-tag
##
make_docker_testing() {
if [ -z "${EMQX_BASE_IMAGE:-}" ]; then
case "$SYSTEM" in
ubuntu20*)
EMQX_BASE_IMAGE="ubuntu:20.04"
;;
el8)
EMQX_BASE_IMAGE="rockylinux:8"
;;
*)
echo "Unsupported testing base image for $SYSTEM"
exit 1
;;
esac
fi
EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}"
local defaultzip
defaultzip="_packages/${PROFILE}/${PROFILE}-$(./scripts/pkg-full-vsn.sh).zip"
local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}"
if [ ! -f "$zip" ]; then
log "ERROR: $zip not built?"
exit 1
fi
set -x
docker build \
--build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \
--build-arg EMQX_ZIP_PACKAGE="${zip}" \
--tag "$EMQX_IMAGE_TAG" \
-f "${DOCKERFILE_TESTING}" .
}
log "building artifact=$ARTIFACT for profile=$PROFILE"
@ -145,9 +326,15 @@ case "$ARTIFACT" in
log "Skipped making deb/rpm package for $SYSTEM"
exit 0
fi
make -C "deploy/packages/${PKGERDIR}" clean
EMQX_REL="$(pwd)" make -C "deploy/packages/${PKGERDIR}" clean
EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}"
;;
docker)
make_docker
;;
docker-testing)
make_docker_testing
;;
*)
log "Unknown artifact $ARTIFACT"
exit 1

82
changes/v4.3.22-en.md Normal file
View File

@ -0,0 +1,82 @@
# v4.3.22
This marks the last release of EMQX v4.3 Opensource Edition.
## Enhancements
- Make sure listener's `tls_versions` config value is one or more of `tlsv1`, `tlsv1.1`, `tlsv1.2`, `tlsv1.3` [#9260](https://github.com/emqx/emqx/pull/9260).
- Remove useless information from the dashboard listener failure log [#9260](https://github.com/emqx/emqx/pull/9260).
- We now trigger the `'message.acked'` hook after the CoAP gateway sends a message to the device and receives the ACK from the device [#9264](https://github.com/emqx/emqx/pull/9264).
With this change, the CoAP gateway can be combined with the offline message caching function (in the
emqx enterprise), so that CoAP devices are able to read the missed messages from the database when
it is online again.
- Support to use placeholders like `${var}` in the HTTP `Headers` of rule-engine's Webhook actions [#9239](https://github.com/emqx/emqx/pull/9239).
- Asynchronously refresh the resources and rules during emqx boot-up [#9199](https://github.com/emqx/emqx/pull/9199).
This is to avoid slowing down the boot if some resources spend long time establishing the connection.
- Add a warning log if the ACL check failed for subscription [#9124](https://github.com/emqx/emqx/pull/9124).
This is to make the ACL deny logging for subscription behave the same as for publish.
- JWT ACL claim supports `all` action to imply the rules applie to both `pub` and `sub` [#9044](https://github.com/emqx/emqx/pull/9044).
- Added a log censor to avoid logging sensitive data [#9189](https://github.com/emqx/emqx/pull/9189).
If the data to be logged is a map or key-value list which contains sensitive key words such as `password`, the value is obfuscated as `******`.
- Enhanced log security in ACL modules, sensitive data will be obscured [#9242](https://github.com/emqx/emqx/pull/9242).
- Add `management.bootstrap_apps_file` configuration to bulk import default app/secret when EMQX initializes the database [#9273](https://github.com/emqx/emqx/pull/9273).
- Added two new configs for deterministic order of authentication and ACL checks [#9283](https://github.com/emqx/emqx/pull/9283).
The two new global config names are `auth_order` and `acl_order`.
When multiple ACL or auth plugins (or modules) are enabled, without this config, the order (in which each backend is queried)
is determined by the start/restart order of the plugin (or module).
Meaning, if a plugin (or module) is restarted after initial boot, it may get ordered to the end of the list.
With this config, you may set the order with a comma-speapated ACL or auth plugin names (or aliases).
For example: `acl_order = jwt,http`, this will make sure `jwt` is always checked before `http`,
meaning if JWT is not found (or no `acl` cliam) for a client, then the ACL check will fallback to use the HTTP backend.
- Added configurations to enable more `client.disconnected` events (and counter bumps) [#9267](https://github.com/emqx/emqx/pull/9267).
Prior to this change, the `client.disconnected` event (and counter bump) is triggered when a client
performs a 'normal' disconnect, or is 'kicked' by system admin, but NOT triggered when a
stale connection had to be 'discarded' (for clean session) or 'takeovered' (for non-clean session) by new connection.
Now it is possible to set configs `broker.client_disconnect_discarded` and `broker.client_disconnect_takeovered` to `on` to enable the event in these scenarios.
- For Rule-Engine resource creation failure, delay before the first retry [#9313](https://github.com/emqx/emqx/pull/9313).
Prior to this change, the retry delay was added *after* the retry failure.
## Bug fixes
- Fix that after uploading a backup file with an non-ASCII filename, HTTP API `GET /data/export` fails with status code 500 [#9224](https://github.com/emqx/emqx/pull/9224).
- Improve the display of rule's 'Maximum Speed' counter to only reserve 2 decimal places [#9185](https://github.com/emqx/emqx/pull/9185).
This is to avoid displaying floats like `0.30000000000000004` on the dashboard.
- Fix the issue that emqx prints too many error logs when connecting to mongodb but auth failed [#9184](https://github.com/emqx/emqx/pull/9184).
- Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic [#9024](https://github.com/emqx/emqx/pull/9024).
- "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134).
- Restore old `emqx_auth_jwt` module API, so the hook callback functions registered in older version will not be invalidated after hot-upgrade [#9144](https://github.com/emqx/emqx/pull/9144).
- Fixed the response status code for the `/status` endpoint [#9210](https://github.com/emqx/emqx/pull/9210).
Before the fix, it always returned `200` even if the EMQX application was not running. Now it returns `503` in that case.
- Fix message delivery related event encoding [#9226](https://github.com/emqx/emqx/pull/9226)
For rule-engine's input events like `$events/message_delivered`, and `$events/message_dropped`,
if the message was delivered to a shared-subscription, the encoding (to JSON) of the event will fail.
Affected versions: `v4.3.21`, `v4.4.10`, `e4.3.16` and `e4.4.10`.
- Make sure Rule-Engine API supports Percent-encoding `rule_id` and `resource_id` in HTTP request path [#9190](https://github.com/emqx/emqx/pull/9190).
Note that the `id` in `POST /api/v4/rules` should be literals (not encoded) when creating a `rule` or `resource`.
See docs [Create Rule](https://docs.emqx.com/en/enterprise/v4.4/advanced/http-api.html#post-api-v4-rules) [Create Resource](https://docs.emqx.com/en/enterprise/v4.4/advanced/http-api.html#post-api-v4-resources).
- Calling 'DELETE /alarms/deactivated' now deletes deactived alarms on all nodes, including remote nodes, not just the local node [#9280](https://github.com/emqx/emqx/pull/9280).
- When republishing messages or bridge messages to other brokers, check the validity of the topic and make sure it does not have topic wildcards [#9291](https://github.com/emqx/emqx/pull/9291).
- Disable authorization for `api/v4/emqx_prometheus` endpoint on management api listener (default 8081) [#9294](https://github.com/emqx/emqx/pull/9294).

78
changes/v4.3.22-zh.md Normal file
View File

@ -0,0 +1,78 @@
# v4.3.22
这是 EMQX 开原版 v4.3 系列的最后一个版本。
## 增强
- 检查监听器的 `tls_versions` 配置值是 `tlsv1`，`tlsv1.1`，`tlsv1.2`，`tlsv1.3` 中的一个或多个组合 [#9260](https://github.com/emqx/emqx/pull/9260)。
- 删除 Dashboard 监听器失败时日志中的无用信息 [#9260](https://github.com/emqx/emqx/pull/9260).
- 当 CoAP 网关给设备投递消息并收到设备发来的确认之后，回调 `'message.acked'` 钩子 [#9264](https://github.com/emqx/emqx/pull/9264)。
有了这个改动，CoAP 网关可以配合 EMQX (企业版)的离线消息缓存功能，让 CoAP 设备重新上线之后，从数据库读取其离线状态下错过的消息。
- 支持在规则引擎的 Webhook 动作的 HTTP Headers 里使用 `${var}` 格式的占位符 [#9239](https://github.com/emqx/emqx/pull/9239)。
- 在 emqx 启动时，异步地刷新资源和规则 [#9199](https://github.com/emqx/emqx/pull/9199)。
这个改动是为了避免因为一些资源连接建立过慢，而导致启动时间过长。
- 订阅时，如果 ACL 检查不通过，打印一个警告日志 [#9124](https://github.com/emqx/emqx/pull/9124)。
该行为的改变主要是为了跟发布失败时的行为保持一致。
- 基于 JWT 的 ACL 支持 `all` 动作，指定同时适用于 `pub` 和 `sub` 两个动作的规则列表 [#9044](https://github.com/emqx/emqx/pull/9044)。
- 增强包含敏感数据的日志的安全性 [#9189](https://github.com/emqx/emqx/pull/9189)。
如果日志中包含敏感关键词，例如 `password`，那么关联的数据回被模糊化处理，替换成 `******`。
- 增强 ACL 模块中的日志安全性，敏感数据将被模糊化 [#9242](https://github.com/emqx/emqx/pull/9242)。
- 增加 `management.bootstrap_apps_file` 配置，可以让 EMQX 初始化数据库时，从该文件批量导入一些 APP / Secret [#9273](https://github.com/emqx/emqx/pull/9273)。
- 增加了固化认证和 ACL 模块调用顺序的配置 [#9283](https://github.com/emqx/emqx/pull/9283)。
这两个新的全局配置名称为 `auth_order` 和 `acl_order`。
当有多个认证或 ACL 插件（或模块）开启时，没有该配置的话，模块调用的顺序取决于它们的启动顺序。
例如，如果一个插件（或模块）在系统启动之后单独重启了，那么它就有可能排到其他插件（或模块）的后面去。
有了这个配置之后，用户可以使用用逗号分隔的插件（或模块）的名字（或别名）来固化他们被调用的顺序。
例如，`acl_order = jwt,http`，可以用于保证 `jwt` 这个模块总是排在 `http` 的前面，
也就是说，在对客户端进行 ACL 检查时，如果 JWT 不存在（或者没有定义 ACL），那么回退到使用 HTTP。
- 为更多类型的 `client.disconnected` 事件（计数器触发）提供可配置项 [#9267](https://github.com/emqx/emqx/pull/9267)。
此前，`client.disconnected` 事件及计数器仅会在客户端正常断开连接或客户端被系统管理员踢出时触发，
但不会在旧 session 被新连接废弃时 (clean_session = true) ，或旧 session 被新连接接管时 (clean_session = false) 被触发。
可将 `broker.client_disconnect_discarded` 和 `broker.client_disconnect_takovered` 选项设置为 `on` 来启用此场景下的客户端断连事件。
- 规则引擎资源创建失败后，第一次重试前增加一个延迟 [#9313](https://github.com/emqx/emqx/pull/9313)。
在此之前，重试的延迟发生在重试失败之后。
## 修复
- 修复若上传的备份文件名中包含非 ASCII 字符，`GET /data/export` HTTP 接口返回 500 错误 [#9224](https://github.com/emqx/emqx/pull/9224)。
- 改进规则的 "最大执行速度" 的计数，只保留小数点之后 2 位 [#9185](https://github.com/emqx/emqx/pull/9185)。
避免在 dashboard 上展示类似这样的浮点数：`0.30000000000000004`。
- 修复在尝试连接 MongoDB 数据库过程中，如果认证失败会不停打印错误日志的问题 [#9184](https://github.com/emqx/emqx/pull/9184)。
- 修复 emqx-sn 插件在“空闲”状态下收到消息发布请求时可能崩溃的情况 [#9024](https://github.com/emqx/emqx/pull/9024)。
- 限速 “Pause due to rate limit” 的日志级别从原先的 `warning` 降级到 `notice` [#9134](https://github.com/emqx/emqx/pull/9134)。
- 保留老的 `emqx_auth_jwt` 模块的接口函数，保障热升级之前添加的回调函数在热升级之后也不会失效 [#9144](https://github.com/emqx/emqx/pull/9144)。
- 修正了 `/status` API 的响应状态代码 [#9210](https://github.com/emqx/emqx/pull/9210)。
在修复之前，它总是返回 `200`，即使 EMQX 应用程序没有运行。 现在它在这种情况下返回 `503`。
- 修复规则引擎的消息事件编码失败 [#9226](https://github.com/emqx/emqx/pull/9226)。
带消息的规则引擎事件，例如 `$events/message_delivered` 和 `$events/message_dropped`,
如果消息事件是共享订阅产生的，在编码（到 JSON 格式）过程中会失败。
影响到的版本：`v4.3.21`, `v4.4.10`, `e4.3.16` 和 `e4.4.10`。
- 使规则引擎 API 在 HTTP 请求路径中支持百分号编码的 `rule_id` 及 `resource_id` [#9190](https://github.com/emqx/emqx/pull/9190)。
注意在创建规则或资源时，HTTP body 中的 `id` 字段仍为字面值，而不是编码之后的值。
详情请参考 [创建规则](https://docs.emqx.com/zh/enterprise/v4.4/advanced/http-api.html#post-api-v4-rules) 和 [创建资源](https://docs.emqx.com/zh/enterprise/v4.4/advanced/http-api.html#post-api-v4-resources)。
- 修复调用 'DELETE /alarms/deactivated' 只在单个节点上生效的问题，现在将会删除所有节点上的非活跃警告 [#9280](https://github.com/emqx/emqx/pull/9280)。
- 在进行消息重发布或桥接消息到其他 mqtt broker 时，检查 topic 合法性，确定其不带有主题通配符 [#9291](https://github.com/emqx/emqx/pull/9291)。
- 关闭管理端口（默认为8081）上对 HTTP API `api/v4/emqx_prometheus` 的认证，Prometheus 对时序数据抓取不在需要配置认证 [#9294](https://github.com/emqx/emqx/pull/9294)。

29
changes/v4.4.11-en.md Normal file
View File

@ -0,0 +1,29 @@
### Enhancements
- OTP upgrade from 24.1.5-3 to 24.3.4.2-1 [#9265](https://github.com/emqx/emqx/pull/9265).
Change highlights:
- Erlang/OTP [SSL library vulnerability fix](https://nvd.nist.gov/vuln/detail/CVE-2022-37026)
- Added support for OCSP (Online Certificate Status Protocol) Stapling
- Added CRL (Certificate Revocation List) cache auto refresh
- Added support for OCSP stapling and CRL
caching [#9297](https://github.com/emqx/emqx/pull/9297).
- Added support for specifying custom modules for adding clientid and common name
aliases [#9297](https://github.com/emqx/emqx/pull/9297).
Now you can implement a simple callback to enrich clients with aliases, and then use the aliases
in the authentication and authorization (ACL) rules' place holders (`%cida` for clientid alias
and `%cna` for username alias).
- Added support for specifying custom modules for custom authentication [#9297](https://github.com/emqx/emqx/pull/9297).
To support simple authentication rules, it is no longer necessary to implement a full-blown plugin.
- Added a JWT management for Rule-Engin, for creating and refreshing JWT tokens in rule engine actions [#9241](https://github.com/emqx/emqx/pull/9241).
This feature is so far only used in EMQX Enterprise Google PubSub integration.
Can be used as webhook integration's JWT authenticationa against the webhook service endpoint.
### Bug fixes
- Fix get trace list crash when trace not initialize. [#9156](https://github.com/emqx/emqx/pull/9156)
- Fix create trace sometime failed by end_at time has already passed. [#9156](https://github.com/emqx/emqx/pull/9156)

26
changes/v4.4.11-zh.md Normal file
View File

@ -0,0 +1,26 @@
### 增强
- OTP 升级: 从 24.1.5-3 至 24.3.4.2-1 [#9265](https://github.com/emqx/emqx/pull/9265)。
重要更新:
- Erlang/OTP [SSL库漏洞修复](https://nvd.nist.gov/vuln/detail/CVE-2022-37026)
- 增加了对 OCSP (Online Certificate Status Protocol) Stapling 的支持
- 增加了 CRL（证书吊销列表）缓存的自动刷新功能
- 增加了 OCSP stapling 和 CRL 缓存 [#9297](https://github.com/emqx/emqx/pull/9297)。
- 增加了可定制的 clientid 或 username 别名的回调模块 [#9297](https://github.com/emqx/emqx/pull/9297)。
有了这个回调模块后，可以简单实现一个 Erlang 的回调函数用来给客户端增加别名，然后在认证和授权规则的占位符中使用这些别名
（`%cida` 用作 clientid 别名，`%cna` 用作 用户名别名）。
- 增加了可定制的认证回调模块 [#9297](https://github.com/emqx/emqx/pull/9297)。
对于一些简单的认证检查，不需要去实现一个完整的认证插件。
- 为规则引擎增加了一个 JWT 令牌管理，用于在规则引擎动作中创建和刷新 JWT 令牌 [#9241](https://github.com/emqx/emqx/pull/9241)。
该功能现在仅用于 EMQX 企业版的 Google PubSub 集成中。
后续会用于 webhook 集成的 JWT 认证。
### 修复
- 修复日志追踪模块没开启时，GET Trace 列表接口报错的问题。[#9156](https://github.com/emqx/emqx/pull/9156)
- 修复创建追踪日志时偶尔会报`end_at time has already passed`错误，导致创建失败。[#9156](https://github.com/emqx/emqx/pull/9156)

13
data/emqx_vars
View File

@ -6,19 +6,16 @@
REL_VSN="{{ release_version }}"
ERTS_VSN="{{ erts_vsn }}"
ERL_OPTS="{{ erl_opts }}"
RUNNER_ROOT_DIR="{{ runner_root_dir }}"
RUNNER_BIN_DIR="{{ runner_bin_dir }}"
RUNNER_LOG_DIR="{{ runner_log_dir }}"
RUNNER_LIB_DIR="{{ runner_lib_dir }}"
RUNNER_ETC_DIR="{{ runner_etc_dir }}"
RUNNER_DATA_DIR="{{ runner_data_dir }}"
RUNNER_USER="{{ runner_user }}"
EMQX_DESCRIPTION='{{ emqx_description }}'
EMQX_LICENSE_CONF=''
export EMQX_DESCRIPTION='{{ emqx_description }}'
## Warning: DO NOT create new variables using the above vars in this file,
## as the vars above can be overwritten by the relup scripts later, like:
## REL_VSN="new_version"
## computed vars
REL_NAME="emqx"
ERTS_PATH="$RUNNER_ROOT_DIR/erts-$ERTS_VSN/bin"
## updated vars here
## overwritten vars here

6
deploy/charts/emqx/Chart.yaml
View File

@ -1,6 +1,6 @@
apiVersion: v2
name: emqx
description: A Helm chart for EMQ X
description: A Helm chart for EMQX
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
@ -13,8 +13,8 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
version: 4.3.0
version: 4.4.11
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application.
appVersion: latest
appVersion: 4.4.11

164
deploy/charts/emqx/README.md
View File

@ -1,14 +1,14 @@
# Introduction
This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm package manager.
This chart bootstraps an [EMQX](https://www.emqx.io/) deployment on a [Kubernetes](https://kubernetes.io/) (K8s) cluster using the [Helm](https://helm.sh/) package manager.
# Prerequisites
+ Kubernetes 1.6+
+ Helm
+ [Kubernetes](https://kubernetes.io/) 1.6+
+ [Helm](https://helm.sh/)
# Installing the Chart
To install the chart with the release name `my-emqx`:
+ From github
+ From github
```
$ git clone https://github.com/emqx/emqx.git
$ cd emqx/deploy/charts/emqx
@ -25,55 +25,115 @@ To install the chart with the release name `my-emqx`:
# Uninstalling the Chart
To uninstall/delete the `my-emqx` deployment:
```
$ helm del my-emqx
$ helm del my-emqx
```
# Configuration
The following table lists the configurable parameters of the emqx chart and their default values.
The following sections describe the configurable parameters of the chart and their default values.
## [K8s]((https://kubernetes.io/)) specific settings
The following table lists the configurable K8s parameters of the [EMQX](https://www.emqx.io/) chart and their default values.
Parameter | Description | Default Value
--- | --- | ---
`replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | `3`
`image.tag` | EMQX Image tag (defaults to `.Chart.AppVersion`) | `nil`
`image.repository` | EMQX Image repository | `emqx/emqx`
`image.pullPolicy` | The image pull policy | `IfNotPresent`
`image.pullSecrets ` | The image pull secrets (does not add image pull secrets to deployed pods) |``[]``
`recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | `false`
`podAnnotations ` | Annotations for pod | `{}`
`podManagementPolicy`| To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock | `Parallel`
`persistence.enabled` | Enable EMQX persistence using PVC | `false`
`persistence.storageClass` | Storage class of backing PVC (uses alpha storage class annotation) | `nil`
`persistence.existingClaim` | EMQX data Persistent Volume existing claim name, evaluated as a template | `""`
`persistence.accessMode` | PVC Access Mode for EMQX volume | `ReadWriteOnce`
`persistence.size` | PVC Storage Request for EMQX volume | `20Mi`
`initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. |`{}`
`resources` | CPU/Memory resource requests/limits |`{}`
`nodeSelector` | Node labels for pod assignment |`{}`
`tolerations` | Toleration labels for pod assignment |``[]``
`affinity` | Map of node/pod affinities |`{}`
`service.type` | Kubernetes Service type. | `ClusterIP`
`service.mqtt` | Port for MQTT. | `1883`
`service.mqttssl` | Port for MQTT(SSL). | `8883`
`service.mgmt` | Port for mgmt API. | `8081`
`service.ws` | Port for WebSocket/HTTP. | `8083`
`service.wss` | Port for WSS/HTTPS. | `8084`
`service.dashboard` | Port for dashboard. | `18083`
`service.nodePorts.mqtt` | Kubernetes node port for MQTT. | `nil`
`service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | `nil`
`service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | `nil`
`service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. | `nil`
`service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. | `nil`
`service.nodePorts.dashboard` | Kubernetes node port for dashboard. | `nil`
`service.loadBalancerIP` | loadBalancerIP for Service | `nil`
`service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | `[]`
`service.externalIPs` | ExternalIPs for the service | `[]`
`service.annotations` | Service annotations (evaluated as a template) | `{}`
`ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false
`ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard
`ingress.dashboard.path` | Ingress path for EMQX Dashboard | `/`
`ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific`
`ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local
`ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | `[]`
`ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | `{}`
`ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | `false`
`ingress.mqtt.ingressClassName` | Set the ingress class for EMQX Mgmt API | `nil`
`ingress.mgmt.path` | Ingress path for EMQX Mgmt API | `/`
`ingress.mgmt.pathType` | Ingress pathType for EMQX Mgmt API | `ImplementationSpecific`
`ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | `api.emqx.local`
`ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | `[]`
`ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | `{}`
`ingress.wss.enabled` | Enable ingress for EMQX Mgmt API | `false`
`ingress.wss.ingressClassName` | Set the ingress class for EMQX Mgmt API | `nil`
`ingress.wss.path` | Ingress path for EMQX WSS | `/`
`ingress.wss.pathType` | Ingress pathType for EMQX WSS | `ImplementationSpecific`
`ingress.wss.hosts` | Ingress hosts for EMQX WSS | `wss.emqx.local`
`ingress.wss.tls` | Ingress tls for EMQX WSS | `[]`
`ingress.wss.annotations` | Ingress annotations for EMQX WSS | `{}`
| `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and [emqx_prometheus](https://github.com/emqx/emqx/tree/main-v4.4/apps/emqx_prometheus) needs to enable | false |
| `metrics.type` | Now we only supported "prometheus" | "prometheus" |
`extraEnv` | Aditional container env vars | `[]`
`extraEnvFrom` | Aditional container env from vars (eg. [config map](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/), [secrets](https://kubernetes.io/docs/concepts/configuration/secret/) | `[]`
`extraArgs` | Additional container executable arguments | `[]`
`extraVolumes` | Additional container volumes (eg. for mounting certs from secrets) | `[]`
`extraVolumeMounts` | Additional container volume mounts (eg. for mounting certs from secrets) | `[]`
| Parameter | Description | Default Value |
| --- | --- | --- |
| `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. |3|
| `image.repository` | EMQ X Image name |emqx/emqx|
| `image.pullPolicy` | The image pull policy |IfNotPresent|
| `image.pullSecrets ` | The image pull secrets |`[]` (does not add image pull secrets to deployed pods)|
| `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false |
| `persistence.enabled` | Enable EMQX persistence using PVC |false|
| `persistence.storageClass` | Storage class of backing PVC |`nil` (uses alpha storage class annotation)|
| `persistence.existingClaim` | EMQ X data Persistent Volume existing claim name, evaluated as a template |""|
| `persistence.accessMode` | PVC Access Mode for EMQX volume |ReadWriteOnce|
| `persistence.size` | PVC Storage Request for EMQX volume |20Mi|
| `initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. |`{}`|
| `resources` | CPU/Memory resource requests/limits |{}|
| `nodeSelector` | Node labels for pod assignment |`{}`|
| `tolerations` | Toleration labels for pod assignment |`[]`|
| `affinity` | Map of node/pod affinities |`{}`|
| `service.type` | Kubernetes Service type. |ClusterIP|
| `service.mqtt` | Port for MQTT. |1883|
| `service.mqttssl` | Port for MQTT(SSL). |8883|
| `service.mgmt` | Port for mgmt API. |8081|
| `service.ws` | Port for WebSocket/HTTP. |8083|
| `service.wss` | Port for WSS/HTTPS. |8084|
| `service.dashboard` | Port for dashboard. |18083|
| `service.nodePorts.mqtt` | Kubernetes node port for MQTT. |nil|
| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). |nil|
| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. |nil|
| `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. |nil|
| `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. |nil|
| `service.nodePorts.dashboard` | Kubernetes node port for dashboard. |nil|
| `service.loadBalancerIP` | loadBalancerIP for Service | nil |
| `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] |
| `service.externalIPs` | ExternalIPs for the service | [] |
| `service.annotations` | Service annotations | {}(evaluated as a template)|
| `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false |
| `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / |
| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local |
| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] |
| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} |
| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false |
| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / |
| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local |
| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] |
| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} |
| `emqxConfig` | Emqx configuration item, see the [documentation](https://hub.docker.com/r/emqx/emqx) | |
| `emqxAclConfig` | Emqx acl configuration item, see the [documentation](https://docs.emqx.io/broker/latest/en/advanced/acl-file.html) | |
## EMQX specific settings
The following table lists the configurable [EMQX](https://www.emqx.io/)-specific parameters of the chart and their default values.
Parameter | Description | Default Value
--- | --- | ---
`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix can be omitted) or using the configuration files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil`
`emqxLicenseSecretName` | Name of the secret that holds the license information | `nil`
`emqxAclConfig` | [ACL](https://docs.emqx.io/broker/latest/en/advanced/acl-file.html) configuration | `{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. {allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. {deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. {allow, all}.`
`emqxLoadedModules` | Modules to load on startup | `{emqx_mod_acl_internal, true}. {emqx_mod_presence, true}. {emqx_mod_delayed, false}. {emqx_mod_rewrite, false}. {emqx_mod_subscription, false}. {emqx_mod_topic_metrics, false}.`
`emqxLoadedPlugins` | Plugins to load on startup | `{emqx_management, true}. {emqx_recon, true}. {emqx_retainer, true}. {emqx_dashboard, true}. {emqx_telemetry, true}. {emqx_rule_engine, true}. {emqx_bridge_mqtt, false}.`
# Examples
This section provides some examples for the configuration of common scenarios.
## Enable Websockets SSL via [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/)
The following settings describe a working scenario for acessing [EMQX](https://www.emqx.io/) Websockets with SSL termination at the [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/).
```yaml
ingress:
wss:
enabled: true
# ingressClassName: nginx
annotations:
nginx.ingress.kubernetes.io/backend-protocol: "http"
nginx.ingress.kubernetes.io/use-forwarded-headers: "true"
nginx.ingress.kubernetes.io/enable-real-ip: "true"
nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "120"
nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
nginx.ingress.kubernetes.io/use-proxy-protocol: "false"
nginx.ingress.kubernetes.io/proxy-protocol-header-timeout: "5s"
path: /mqtt
pathType: ImplementationSpecific
hosts:
- myhost.example.com
tls:
- hosts:
- myhost.example.com
secretName: myhost-example-com-tls # Name of the secret that holds the certificates for the domain
```

39
deploy/charts/emqx/service-monitor.yaml Normal file
View File

@ -0,0 +1,39 @@
{{- if and (.Values.metrics.enabled) (eq .Values.metrics.type "prometheus") }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ include "emqx.fullname" . }}
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- if .Values.service.annotations }}
annotations:
{{ toYaml .Values.service.annotations | indent 4 }}
{{- end }}
spec:
endpoints:
- interval: 10s
port: mgmt
scheme: http
path: /api/v4/emqx_prometheus
params:
type:
- prometheus
basicAuth:
password:
name: {{ include "emqx.fullname" . }}-basic-auth
key: password
username:
name: {{ include "emqx.fullname" . }}-basic-auth
key: username
jobLabel: {{ .Release.Name }}-scraping
namespaceSelector:
matchNames:
- {{ .Release.Namespace }}
selector:
matchLabels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
{{- end }}

78
deploy/charts/emqx/templates/StatefulSet.yaml
View File

@ -1,3 +1,12 @@
{{ $cfgEnv := printf "%s" (include (print $.Template.BasePath "/configmap.env.yaml") .) }}
{{ $cfgAcl := printf "%s" (include (print $.Template.BasePath "/configmap.acl.yaml") .) }}
{{ $cfgPlugins := printf "%s" (include (print $.Template.BasePath "/configmap.loadedPlugins.yaml") .) }}
{{ $cfgModules := printf "%s" (include (print $.Template.BasePath "/configmap.loadedModules.yaml") .) }}
{{ $configData := printf "%s\n%s\n%s\n%s" $cfgEnv $cfgAcl $cfgPlugins $cfgModules}}
## Compatible with previous misspellings
{{ $licenseSecretName := coalesce .Values.emqxLicenseSecretName .Values.emqxLicneseSecretName }}
{{ $image := printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
@ -18,7 +27,6 @@ spec:
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
annotations:
@ -48,30 +56,39 @@ spec:
version: {{ .Chart.AppVersion }}
app.kubernetes.io/name: {{ include "emqx.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- if .Values.recreatePods }}
annotations:
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum | quote }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if .Values.recreatePods }}
checksum/config: {{ $configData | sha256sum | quote }}
{{- end }}
spec:
volumes:
{{- if .Values.emqxLoadedPlugins }}
- name: emqx-loaded-plugins
configMap:
name: {{ include "emqx.fullname" . }}-loaded-plugins
items:
- key: loaded_plugins
path: loaded_plugins
{{- end }}
{{- if .Values.emqxLoadedModules }}
- name: emqx-loaded-modules
configMap:
name: {{ include "emqx.fullname" . }}-loaded-modules
items:
- key: loaded_modules
path: loaded_modules
{{- end }}
{{- if .Values.emqxAclConfig }}
- name: emqx-acl
configMap:
name: {{ include "emqx.fullname" . }}-acl
items:
- key: acl.conf
path: acl.conf
{{- end }}
{{- if not .Values.persistence.enabled }}
- name: emqx-data
emptyDir: {}
@ -82,12 +99,17 @@ spec:
claimName: {{ tpl . $ }}
{{- end }}
{{- end }}
{{- if .Values.emqxLicneseSecretName }}
{{- if .Values.extraVolumes }}
{{ toYaml .Values.extraVolumes | indent 6 }}
{{- end }}
{{- if $licenseSecretName }}
- name: emqx-license
secret:
secretName: {{ .Values.emqxLicneseSecretName }}
secretName: {{ $licenseSecretName }}
{{- end }}
{{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY | default "k8s") "k8s" }}
serviceAccountName: {{ include "emqx.fullname" . }}
{{- end }}
{{- if .Values.podSecurityContext.enabled }}
securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }}
{{- end }}
@ -103,7 +125,7 @@ spec:
{{- end }}
containers:
- name: emqx
image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}"
image: {{ $image }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- if .Values.containerSecurityContext.enabled }}
securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }}
@ -132,45 +154,61 @@ spec:
- name: ekka
containerPort: 4370
envFrom:
- configMapRef:
name: {{ include "emqx.fullname" . }}-env
- configMapRef:
name: {{ include "emqx.fullname" . }}-env
{{- if .Values.extraEnvFrom }}
{{ toYaml .Values.extraEnvFrom | indent 10 }}
{{- end }}
{{- if .Values.extraEnv }}
env:
- name: EMQX_NAME
value: {{ .Release.Name }}
- name: EMQX_CLUSTER__K8S__APP_NAME
value: {{ .Release.Name }}
- name: EMQX_CLUSTER__DISCOVERY
value: k8s
- name: EMQX_CLUSTER__K8S__SERVICE_NAME
value: {{ include "emqx.fullname" . }}-headless
- name: EMQX_CLUSTER__K8S__NAMESPACE
value: {{ .Release.Namespace }}
{{ toYaml .Values.extraEnv | indent 10 }}
{{- end }}
resources:
{{ toYaml .Values.resources | indent 12 }}
volumeMounts:
- name: emqx-data
mountPath: "/opt/emqx/data"
{{- if .Values.emqxAclConfig }}
- name: emqx-acl
mountPath: "/opt/emqx/etc/acl.conf"
subPath: "acl.conf"
{{- end }}
{{- if .Values.emqxLoadedPlugins }}
- name: emqx-loaded-plugins
mountPath: "/opt/emqx/data/loaded_plugins"
subPath: "loaded_plugins"
{{- end }}
{{- if .Values.emqxLoadedModules }}
- name: emqx-loaded-modules
mountPath: "/opt/emqx/data/loaded_modules"
subPath: "loaded_modules"
{{ if .Values.emqxLicneseSecretName }}
{{- end }}
{{- if $licenseSecretName }}
- name: emqx-license
mountPath: "/opt/emqx/etc/emqx.lic"
subPath: "emqx.lic"
readOnly: true
{{ end }}
{{- if and .Values.extraVolumes .Values.extraVolumeMounts }}
{{ toYaml .Values.extraVolumeMounts | indent 10 }}
{{- end }}
{{- if .Values.extraArgs }}
args: {{ toYaml .Values.extraArgs | nindent 10 }}
{{- end }}
readinessProbe:
httpGet:
path: /status
port: {{ .Values.emqxConfig.EMQX_MANAGEMENT__LISTENER__HTTP | default 8081 }}
initialDelaySeconds: 5
initialDelaySeconds: 10
periodSeconds: 5
failureThreshold: 30
livenessProbe:
httpGet:
path: /status
port: {{ .Values.emqxConfig.EMQX_MANAGEMENT__LISTENER__HTTP | default 8081 }}
initialDelaySeconds: 60
periodSeconds: 30
failureThreshold: 10
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}

15
deploy/charts/emqx/templates/configmap.acl.yaml Normal file
View File

@ -0,0 +1,15 @@
{{ if .Values.emqxAclConfig }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "emqx.fullname" . }}-acl
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
"acl.conf": |
{{ .Values.emqxAclConfig }}
{{ end }}

20
deploy/charts/emqx/templates/configmap.env.yaml Normal file
View File

@ -0,0 +1,20 @@
{{- if .Values.emqxConfig }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "emqx.fullname" . }}-env
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
{{- range $index, $value := .Values.emqxConfig }}
{{- if $value }}
{{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }}
{{ print "EMQX_" $key }}: "{{ tpl (printf "%v" $value) $ }}"
{{- end }}
{{- end}}
{{- end }}

15
deploy/charts/emqx/templates/configmap.loadedModules.yaml Normal file
View File

@ -0,0 +1,15 @@
{{ if .Values.emqxLoadedModules }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "emqx.fullname" . }}-loaded-modules
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
"loaded_modules": |
{{ .Values.emqxLoadedModules }}
{{ end }}

15
deploy/charts/emqx/templates/configmap.loadedPlugins.yaml Normal file
View File

@ -0,0 +1,15 @@
{{ if .Values.emqxLoadedPlugins }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "emqx.fullname" . }}-loaded-plugins
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
"loaded_plugins": |
{{ .Values.emqxLoadedPlugins }}
{{ end }}

59
deploy/charts/emqx/templates/configmap.yaml
View File

@ -1,59 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "emqx.fullname" . }}-env
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
{{- range $index, $value := .Values.emqxConfig}}
{{$index}}: "{{ $value }}"
{{- end}}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "emqx.fullname" . }}-acl
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
"acl.conf": |
{{ .Values.emqxAclConfig }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "emqx.fullname" . }}-loaded-plugins
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
"loaded_plugins": |
{{ .Values.emqxLoadedPlugins }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "emqx.fullname" . }}-loaded-modules
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
"loaded_modules": |
{{ .Values.emqxLoadedModules }}

55
deploy/charts/emqx/templates/ingress.dashboard.yaml Normal file
View File

@ -0,0 +1,55 @@
{{- if .Values.ingress.dashboard.enabled -}}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ printf "%s-%s" (include "emqx.fullname" .) "dashboard" }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- if or .Values.ingress.annotations .Values.ingress.dashboard.annotations }}
annotations:
{{- if .Values.ingress.annotations }}
{{- toYaml .Values.ingress.annotations | nindent 4 }}
{{- end }}
{{- if .Values.ingress.dashboard.annotations }}
{{- toYaml .Values.ingress.dashboard.annotations | nindent 4 }}
{{- end }}
{{- end }}
spec:
{{- if and .Values.ingress.dashboard.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.dashboard.ingressClassName }}
{{- end }}
rules:
{{- range $host := .Values.ingress.dashboard.hosts }}
- host: {{ $host }}
http:
paths:
- path: /
{{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ $.Values.ingress.dashboard.pathType | default "ImplementationSpecific" }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ include "emqx.fullname" $ }}
port:
number: {{ $.Values.service.dashboard }}
{{- else }}
serviceName: {{ include "emqx.fullname" $ }}
servicePort: {{ $.Values.service.dashboard }}
{{- end }}
{{- end -}}
{{- if .Values.ingress.dashboard.tls }}
tls:
{{- toYaml .Values.ingress.dashboard.tls | nindent 4 }}
{{- end }}
---
{{- end }}

54
deploy/charts/emqx/templates/ingress.mgmt.yaml Normal file
View File

@ -0,0 +1,54 @@
{{- if .Values.ingress.mgmt.enabled -}}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ printf "%s-%s" (include "emqx.fullname" .) "mgmt" }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- if or .Values.ingress.annotations .Values.ingress.mgmt.annotations }}
annotations:
{{- if .Values.ingress.annotations }}
{{- toYaml .Values.ingress.annotations | nindent 4 }}
{{- end }}
{{- if .Values.ingress.mgmt.annotations }}
{{- toYaml .Values.ingress.mgmt.annotations | nindent 4 }}
{{- end }}
{{- end }}
spec:
{{- if and .Values.ingress.mgmt.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.mgmt.ingressClassName }}
{{- end }}
rules:
{{- range $host := .Values.ingress.mgmt.hosts }}
- host: {{ $host }}
http:
paths:
- path: {{ $.Values.ingress.mgmt.path | default "/" }}
{{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ $.Values.ingress.mgmt.pathType | default "ImplementationSpecific" }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ include "emqx.fullname" $ }}
port:
number: {{ $.Values.service.mgmt }}
{{- else }}
serviceName: {{ include "emqx.fullname" $ }}
servicePort: {{ $.Values.service.mgmt }}
{{- end }}
{{- end -}}
{{- if .Values.ingress.mgmt.tls }}
tls:
{{- toYaml .Values.ingress.mgmt.tls | nindent 4 }}
{{- end }}
{{- end }}

54
deploy/charts/emqx/templates/ingress.wss.yaml Normal file
View File

@ -0,0 +1,54 @@
{{- if .Values.ingress.wss.enabled -}}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ printf "%s-%s" (include "emqx.fullname" .) "wss" }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- if or .Values.ingress.annotations .Values.ingress.wss.annotations }}
annotations:
{{- if .Values.ingress.annotations }}
{{- toYaml .Values.ingress.annotations | nindent 4 }}
{{- end }}
{{- if .Values.ingress.wss.annotations }}
{{- toYaml .Values.ingress.wss.annotations | nindent 4 }}
{{- end }}
{{- end }}
spec:
{{- if and .Values.ingress.wss.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.wss.ingressClassName }}
{{- end }}
rules:
{{- range $host := .Values.ingress.wss.hosts }}
- host: {{ $host }}
http:
paths:
- path: {{ $.Values.ingress.wss.path | default "/mqtt" }}
{{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ $.Values.ingress.wss.pathType | default "ImplementationSpecific" }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ include "emqx.fullname" $ }}
port:
number: {{ $.Values.service.ws }}
{{- else }}
serviceName: {{ include "emqx.fullname" $ }}
servicePort: {{ $.Values.service.ws }}
{{- end }}
{{- end -}}
{{- if .Values.ingress.wss.tls }}
tls:
{{- toYaml .Values.ingress.wss.tls | nindent 4 }}
{{- end }}
{{- end }}

70
deploy/charts/emqx/templates/ingress.yaml
View File

@ -1,70 +0,0 @@
{{- if .Values.ingress.dashboard.enabled -}}
{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ printf "%s-%s" (include "emqx.fullname" .) "dashboard" }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- if .Values.ingress.dashboard.annotations }}
annotations:
{{- toYaml .Values.ingress.dashboard.annotations | nindent 4 }}
{{- end }}
spec:
rules:
{{- range $host := .Values.ingress.dashboard.hosts }}
- host: {{ $host }}
http:
paths:
- path: /
backend:
serviceName: {{ include "emqx.fullname" $ }}
servicePort: {{ $.Values.service.dashboard }}
{{- end -}}
{{- if .Values.ingress.dashboard.tls }}
tls:
{{- toYaml .Values.ingress.dashboard.tls | nindent 4 }}
{{- end }}
---
{{- end }}
{{- if .Values.ingress.mgmt.enabled -}}
{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ printf "%s-%s" (include "emqx.fullname" .) "mgmt" }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- if .Values.ingress.mgmt.annotations }}
annotations:
{{- toYaml .Values.ingress.mgmt.annotations | nindent 4 }}
{{- end }}
spec:
rules:
{{- range $host := .Values.ingress.mgmt.hosts }}
- host: {{ $host }}
http:
paths:
- path: /
backend:
serviceName: {{ include "emqx.fullname" $ }}
servicePort: {{ $.Values.service.mgmt }}
{{- end -}}
{{- if .Values.ingress.mgmt.tls }}
tls:
{{- toYaml .Values.ingress.mgmt.tls | nindent 4 }}
{{- end }}
---
{{- end }}

8
deploy/charts/emqx/templates/rbac.yaml
View File

@ -1,3 +1,4 @@
{{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY | default "k8s") "k8s" }}
apiVersion: v1
kind: ServiceAccount
metadata:
@ -17,8 +18,8 @@ rules:
- apiGroups:
- ""
resources:
- endpoints
verbs:
- endpoints
verbs:
- get
- watch
- list
@ -39,4 +40,5 @@ subjects:
roleRef:
kind: Role
name: {{ include "emqx.fullname" . }}
apiGroup: rbac.authorization.k8s.io
apiGroup: rbac.authorization.k8s.io
{{- end }}

19
deploy/charts/emqx/templates/secret.yaml Normal file
View File

@ -0,0 +1,19 @@
{{- if .Values.metrics.enabled }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "emqx.fullname" . }}-basic-auth
namespace: {{ .Release.Namespace }}
type: kubernetes.io/basic-auth
stringData:
{{- if not (empty .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID) }}
username: admin
{{- else }}
username: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID }}
{{- end }}
{{- if not (empty .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET) }}
password: public
{{- else }}
password: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET}}
{{- end }}
{{- end }}

76
deploy/charts/emqx/values.yaml
View File

@ -17,11 +17,33 @@ image:
## Forces the recreation of pods during helm upgrades. This can be useful to update configuration values even if the container image did not change.
recreatePods: false
podAnnotations: {}
# Pod deployment policy
# value: OrderedReady | Parallel
# To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock
podManagementPolicy: Parallel
## Aditional container env vars
##
extraEnv: []
## Aditional container env from vars
##
extraEnvFrom: []
## Additional container executable args
##
extraArgs: []
## Additional container volumes (eg. for mounting certs from secrets)
##
extraVolumes: []
## Additional container volume mounts (eg. for mounting certs from secrets)
##
extraVolumeMounts: []
persistence:
enabled: false
size: 20Mi
@ -50,7 +72,20 @@ initContainers: {}
## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
emqxConfig:
EMQX_NAME: "{{ .Release.Name }}"
## Cluster discovery by dns
# EMQX_CLUSTER__DISCOVERY: "dns"
# EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
# EMQX_CLUSTER__DNS__APP: "{{ .Release.Name }}"
# EMQX_CLUSTER__DNS__TYPE: "srv"
## Cluster discovery by k8s
EMQX_CLUSTER__DISCOVERY: "k8s"
EMQX_CLUSTER__K8S__APP_NAME: "{{ .Release.Name }}"
EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless"
EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}"
## The address type is used to extract host from k8s service.
## Value: ip | dns | hostname
## Note：Hostname is only supported after v4.0-rc.2
@ -58,6 +93,8 @@ emqxConfig:
EMQX_CLUSTER__K8S__SUFFIX: "svc.cluster.local"
## if EMQX_CLUSTER__K8S__ADDRESS_TYPE eq dns
# EMQX_CLUSTER__K8S__SUFFIX: "pod.cluster.local"
EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID: "admin"
EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET: "public"
## --------------------------------------------------------------------
## [ACL](https://docs.emqx.io/broker/latest/en/advanced/acl-file.html)
@ -94,15 +131,17 @@ emqxLoadedPlugins: >
emqxLoadedModules: >
{emqx_mod_acl_internal, true}.
{emqx_mod_presence, true}.
{emqx_mod_trace, false}.
{emqx_mod_slow_subs, false}.
{emqx_mod_delayed, false}.
{emqx_mod_rewrite, false}.
{emqx_mod_subscription, false}.
{emqx_mod_topic_metrics, false}.
## EMQX Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicneseSecretName"
## EMQX Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicenseSecretName"
## Example:
## kubectl create secret generic emqx-license-secret-name --from-file=/path/to/emqx.lic
emqxLicneseSecretName:
emqxLicenseSecretName:
service:
## Service type
@ -165,6 +204,9 @@ tolerations: []
affinity: {}
ingress:
## Ingress shared annotations
annotations: {}
## ingress for EMQX Dashboard
dashboard:
enabled: false
@ -172,6 +214,7 @@ ingress:
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
path: /
pathType: ImplementationSpecific
hosts:
- dashboard.emqx.local
tls: []
@ -183,10 +226,35 @@ ingress:
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
path: /
pathType: ImplementationSpecific
hosts:
- api.emqx.local
tls: []
## ingress for EMQX Mgmt API
wss:
enabled: false
# ingressClassName: nginx
annotations: {}
# Sample annotations for nginx-ingress community controller
# nginx.ingress.kubernetes.io/rewrite-target: /mqtt$1 # Use to rewrite backend path if needed
# nginx.ingress.kubernetes.io/backend-protocol: "http"
# nginx.ingress.kubernetes.io/use-forwarded-headers: "true"
# nginx.ingress.kubernetes.io/enable-real-ip: "true"
# nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
# nginx.ingress.kubernetes.io/proxy-connect-timeout: "120"
# nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
# nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
# nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
# nginx.ingress.kubernetes.io/use-proxy-protocol: "false"
# nginx.ingress.kubernetes.io/proxy-protocol-header-timeout: "5s"
path: /mqtt
pathType: ImplementationSpecific
# path: /wss(\/.*)?
hosts:
- wss.emqx.local
tls: []
podSecurityContext:
enabled: true
fsGroup: 1000
@ -199,3 +267,7 @@ containerSecurityContext:
enabled: true
runAsNonRoot: true
runAsUser: 1000
metrics:
enabled: false
type: prometheus

29
deploy/docker/Dockerfile
View File

@ -1,10 +1,7 @@
ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-amd64
ARG RUN_FROM=alpine:3.12
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-alpine3.15.1
ARG RUN_FROM=alpine:3.15.1
FROM ${BUILD_FROM} AS builder
ARG QEMU_ARCH=x86_64
COPY tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
RUN apk add --no-cache \
git \
curl \
@ -19,6 +16,7 @@ RUN apk add --no-cache \
libc-dev \
libstdc++ \
bash \
tzdata \
jq
COPY . /emqx
@ -26,31 +24,21 @@ COPY . /emqx
ARG PKG_VSN
ARG EMQX_NAME=emqx
ENV EMQX_RELUP=false
RUN cd /emqx \
&& rm -rf _build/$EMQX_NAME/lib \
&& make $EMQX_NAME
FROM $RUN_FROM
# Basic build-time metadata as defined at http://label-schema.org
LABEL org.label-schema.docker.dockerfile="Dockerfile" \
org.label-schema.license="GNU" \
org.label-schema.name="emqx" \
org.label-schema.version=${PKG_VSN} \
org.label-schema.description="EMQ (Erlang MQTT Broker) is a distributed, massively scalable, highly extensible MQTT messaging broker written in Erlang/OTP." \
org.label-schema.url="https://emqx.io" \
org.label-schema.vcs-type="Git" \
org.label-schema.vcs-url="https://github.com/emqx/emqx" \
maintainer="EMQ X Team <support@emqx.io>"
ARG QEMU_ARCH=x86_64
ARG EMQX_NAME=emqx
COPY deploy/docker/docker-entrypoint.sh tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
COPY deploy/docker/docker-entrypoint.sh /usr/bin/
COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx
RUN ln -s /opt/emqx/bin/* /usr/local/bin/
RUN apk add --no-cache curl ncurses-libs openssl sudo libstdc++ bash
RUN apk add --no-cache curl ncurses-libs openssl sudo libstdc++ bash tzdata
WORKDIR /opt/emqx
@ -62,7 +50,8 @@ RUN chgrp -Rf emqx /opt/emqx && chmod -Rf g+w /opt/emqx \
USER emqx
VOLUME ["/opt/emqx/log", "/opt/emqx/data", "/opt/emqx/etc"]
## NOTE: /opt/emqx/etc is removed from the VOLUME list since 4.3.13
VOLUME ["/opt/emqx/log", "/opt/emqx/data"]
# emqx will occupy these port:
# - 1883 port for MQTT

43
deploy/docker/Dockerfile.testing Normal file
View File

@ -0,0 +1,43 @@
ARG BUILD_FROM
FROM ${BUILD_FROM}
## all we need is the unzip command
RUN if command -v yum; then yum update -y && yum install -y unzip; fi
RUN if command -v apt-get; then apt-get update -y && apt-get install unzip; fi
ARG EMQX_ZIP_PACKAGE
COPY ${EMQX_ZIP_PACKAGE} /opt/emqx.zip
RUN unzip -q /opt/emqx.zip -d /opt/ && rm /opt/emqx.zip
COPY deploy/docker/docker-entrypoint.sh /usr/bin/
RUN ln -s /opt/emqx/bin/* /usr/local/bin/
WORKDIR /opt/emqx
RUN adduser -u 1000 emqx
RUN echo "emqx ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers
RUN chgrp -Rf emqx /opt/emqx && chmod -Rf g+w /opt/emqx \
&& chown -Rf emqx /opt/emqx
USER emqx
VOLUME ["/opt/emqx/log", "/opt/emqx/data", "/opt/emqx/etc"]
# emqx will occupy these port:
# - 1883 port for MQTT
# - 8081 for mgmt API
# - 8083 for WebSocket/HTTP
# - 8084 for WSS/HTTPS
# - 8883 port for MQTT(SSL)
# - 11883 port for internal MQTT/TCP
# - 18083 for dashboard
# - 4369 epmd (Erlang-distrbution port mapper daemon) listener (deprecated)
# - 4370 default Erlang distrbution port
# - 5369 for gen_rpc port mapping
# - 6369 6370 for distributed node
EXPOSE 1883 8081 8083 8084 8883 11883 18083 4369 4370 5369 6369 6370
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
CMD ["/opt/emqx/bin/emqx", "foreground"]

20
deploy/docker/README.md
View File

@ -17,11 +17,11 @@
[the latest release](https://github.com/docker/docker-ce/releases/latest)
# What is EMQ X
# What is EMQX
[EMQ X MQTT broker](https://emqx.io/products/broker) is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients.
[EMQX MQTT broker](https://emqx.io/products/broker) is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients.
Starting from 3.0 release, *EMQ X* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQ X* broker can scaled to 10+ million concurrent MQTT connections on one cluster.
Starting from 3.0 release, *EMQX* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQX* broker can scaled to 10+ million concurrent MQTT connections on one cluster.
# How to use this image
@ -39,7 +39,7 @@ The emqx broker runs as linux user `emqx` in the docker container.
### Configuration
Use the environment variable to configure the EMQ X docker container.
Use the environment variable to configure the EMQX docker container.
By default, the environment variables with ``EMQX_`` prefix are mapped to key-value pairs in configuration files.
@ -72,9 +72,9 @@ EMQX_HOST
These environment variables will ignore for configuration file.
#### EMQ X Configuration
#### EMQX Configuration
> NOTE: All EMQ X Configuration in [etc/emqx.conf](https://github.com/emqx/emqx/blob/main-v4.3/etc/emqx.conf) could config by environment. The following list is just an example, not a complete configuration.
> NOTE: All EMQX Configuration in [etc/emqx.conf](https://github.com/emqx/emqx/blob/main-v4.3/etc/emqx.conf) could config by environment. The following list is just an example, not a complete configuration.
| Options | Default | Mapped | Description |
| ---------------------------| ------------------ | ------------------------- | ------------------------------------- |
@ -148,7 +148,7 @@ EMQX_LOADED_PLUGINS="emqx_auth_redis emqx_auth_mysql"
EMQX_LOADED_PLUGINS="emqx_auth_redis | emqx_auth_mysql"
```
#### EMQ X Plugins Configuration
#### EMQX Plugins Configuration
The environment variables which with ``EMQX_`` prefix are mapped to all emqx plugins' configuration file, ``.`` get replaced by ``__``.
@ -200,7 +200,7 @@ docker run -d --name emqx -p 18083:18083 -p 1883:1883 -p 4369:4369 \
### Cluster
EMQ X supports a variety of clustering methods, see our [documentation](https://docs.emqx.io/broker/latest/en/advanced/cluster.html#emqx-service-discovery) for details.
EMQX supports a variety of clustering methods, see our [documentation](https://docs.emqx.io/broker/latest/en/advanced/cluster.html#emqx-service-discovery) for details.
Let's create a static node list cluster from docker-compose.
@ -256,7 +256,7 @@ Let's create a static node list cluster from docker-compose.
### Persistence
If you want to persist the EMQ X docker container, you need to keep the following directories:
If you want to persist the EMQX docker container, you need to keep the following directories:
+ `/opt/emqx/data`
+ `/opt/emqx/etc`
@ -316,7 +316,7 @@ docker run -d --name emqx -p 18083:18083 -p 1883:1883 -p 4369:4369 \
```
> REMEMBER: DO NOT RUN EMQ X DOCKER PRIVILEGED OR MOUNT SYSTEM PROC IN CONTAINER TO TUNE LINUX KERNEL, IT IS UNSAFE.
> REMEMBER: DO NOT RUN EMQX DOCKER PRIVILEGED OR MOUNT SYSTEM PROC IN CONTAINER TO TUNE LINUX KERNEL, IT IS UNSAFE.
### Thanks

34
deploy/docker/docker-entrypoint.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash
#!/usr/bin/env bash
## EMQ docker image start script
# Huang Rui <vowstar@gmail.com>
# EMQ X Team <support@emqx.io>
# EMQX Team <support@emqx.io>
## Shell setting
if [[ -n "$DEBUG" ]]; then
@ -28,12 +28,20 @@ if [[ -z "$EMQX_NAME" ]]; then
fi
if [[ -z "$EMQX_HOST" ]]; then
if [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"}
EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX"
elif [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'}
EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX")
if [[ "$EMQX_CLUSTER__DISCOVERY" == "dns" ]] && \
[[ "$EMQX_CLUSTER__DNS__TYPE" == "srv" ]] && \
grep -q "$(hostname).$EMQX_CLUSTER__DNS__NAME" /etc/hosts; then
EMQX_HOST="$(hostname).$EMQX_CLUSTER__DNS__NAME"
elif [[ "$EMQX_CLUSTER__DISCOVERY" == "k8s" ]] && \
[[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && \
[[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"}
EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX"
elif [[ "$EMQX_CLUSTER__DISCOVERY" == "k8s" ]] && \
[[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && \
[[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'}
EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX")
else
EMQX_HOST="$LOCAL_IP"
fi
@ -97,13 +105,13 @@ fill_tuples() {
local file=$1
local elements=${*:2}
for var in $elements; do
if grep -qE "\{\s*$var\s*,\s*(true|false)\s*\}\s*\." "$file"; then
sed -r "s/\{\s*($var)\s*,\s*(true|false)\s*\}\s*\./{\1, true}./1" "$file" > tmpfile && cat tmpfile > "$file"
elif grep -q "$var\s*\." "$file"; then
if grep -qE "\{\s*$var\s*,\s*(true|false)\s*\}\s*\." "$file" 2>/dev/null; then
sed -r "s/\{\s*($var)\s*,\s*(true|false)\s*\}\s*\./{\1, true}./1" "$file" 2>/dev/null > tmpfile && cat tmpfile > "$file"
elif grep -q "$var\s*\." "$file" 2>/dev/null; then
# backward compatible.
sed -r "s/($var)\s*\./{\1, true}./1" "$file" > tmpfile && cat tmpfile > "$file"
sed -r "s/($var)\s*\./{\1, true}./1" "$file" > tmpfile 2>/dev/null && cat tmpfile > "$file"
else
sed '$a'\\ "$file" > tmpfile && cat tmpfile > "$file"
sed '$a'\\ "$file" 2>/dev/null > tmpfile && cat tmpfile > "$file"
echo "{$var, true}." >> "$file"
fi
done

6
deploy/packages/deb/Makefile
View File

@ -1,4 +1,3 @@
ARCH ?= amd64
TOPDIR := /tmp/emqx
# Keep this short to avoid bloating beam files with long file path info
SRCDIR := $(TOPDIR)/$(PKG_VSN)
@ -8,11 +7,12 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH)
TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh)
.PHONY: all
all: | $(BUILT)
cp -r debian $(SRCDIR)/
cp emqx.service $(SRCDIR)/debian/${EMQX_NAME}.service
sed -i "s#<DATETIME>#$(shell date -u '+%a, %d %b %Y %T %z')#g" $(SRCDIR)/debian/changelog
sed -i "s#<VERSION>#$(PKG_VSN)#g" $(SRCDIR)/debian/changelog
sed -i "s/emqx-pkg/$(EMQX_NAME)-pkg/g" $(SRCDIR)/debian/rules; \
@ -21,6 +21,8 @@ all: | $(BUILT)
cd $(SRCDIR) && dpkg-buildpackage -us -uc
mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME)
cp $(SRCDIR)/../$(SOURCE_PKG).deb $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb
sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb | head -c 64 > \
$(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb.sha256
$(BUILT):
mkdir -p $(TOPDIR) $(SRCDIR)

2
deploy/packages/deb/debian/control
View File

@ -4,7 +4,7 @@ Priority: optional
Maintainer: emqx <contact@emqx.io>
Build-Depends: debhelper (>=9)
Standards-Version: 3.9.6
Homepage: https://www.emqx.io
Homepage: https://www.emqx.com
Package: emqx
Architecture: any

150
deploy/packages/deb/debian/init.script
View File

@ -1,150 +0,0 @@
#! /bin/bash
### BEGIN INIT INFO
# Provides: emqx
# Required-Start: $remote_fs $syslog
# Required-Stop: $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Erlang MQTT Broker
# Description: EMQX, a distributed, massively scalable, highly extensible MQTT message broker written in Erlang/OT
### END INIT INFO
NAME=emqx
DAEMON=/usr/bin/$NAME
SCRIPTNAME=/etc/init.d/$NAME
# Read configuration variable file if it is present
[ -r /etc/default/$NAME ] && . /etc/default/$NAME
# Load the VERBOSE setting and other rcS variables
. /lib/init/vars.sh
. /lib/lsb/init-functions
# `service` strips all environmental VARS so
# if no HOME was set in /etc/default/$NAME then set one here
# to the data directory for erlexec's sake
if [ -z "$HOME" ]; then
export HOME=/var/lib/emqx
fi
#
# Function that starts the daemon/service
#
do_start()
{
# Return
# 0 if daemon has been started
# 1 if daemon was already running
# 2 if daemon could not be started
# Startup with the appropriate user
start-stop-daemon --start \
--name emqx \
--user emqx \
--exec $DAEMON -- start \
|| return 2
}
#
# Function that stops the daemon/service
#
do_stop()
{
# Identify the erts directory
ERTS_PATH=`$DAEMON ertspath`
# Attempt a clean shutdown.
$DAEMON stop
# waiting stop done sleep 5
sleep 5
# Return
# 0 if daemon has been stopped
# 1 if daemon was already stopped
# 2 if daemon could not be stopped
# other if a failure occurred
# Make sure it's down by using a more direct approach
start-stop-daemon --stop \
--quiet \
--retry=TERM/30/KILL/5 \
--user emqx \
--exec $ERTS_PATH/run_erl
return $?
}
#
# Function that graceful reload the daemon/service
#
do_reload() {
# Restart the VM without exiting the process
$DAEMON restart && return $? || return 2
}
# Checks the status of a node
do_status() {
$DAEMON ping && echo $"$NAME is running" && return 0
echo $"$NAME is stopped" && return 2
}
case "$1" in
start)
log_daemon_msg "Starting $NAME"
$DAEMON ping >/dev/null 2>&1 && echo $"$NAME is already running" && exit 0
do_start
case "$?" in
0|1) log_end_msg 0 ;;
2) log_end_msg 1
exit 1
;;
esac
;;
stop)
log_daemon_msg "Stopping $NAME"
do_stop
case "$?" in
0|1) log_end_msg 0 ;;
2) log_end_msg 1
exit 1
;;
esac
;;
ping)
# See if the VM is alive
$DAEMON ping || exit $?
;;
reload|force-reload)
log_daemon_msg "Reloading $NAME"
do_reload
ES=$?
log_end_msg $ES
exit $ES
;;
restart)
log_daemon_msg "Restarting $NAME"
do_stop
case "$?" in
0|1)
do_start
case "$?" in
0) log_end_msg 0 ;;
1) log_end_msg 1 && exit 1 ;; # Old process is still running
*) log_end_msg 1 && exit 1 ;; # Failed to start
esac
;;
*)
# Failed to stop
log_end_msg 1 && exit 1
;;
esac
;;
status)
do_status && exit 0 || exit $?
;;
*)
echo "Usage: $SCRIPTNAME {start|stop|ping|restart|force-reload|status}" >&2
exit 3
;;
esac

9
deploy/packages/deb/debian/postinst
View File

@ -5,9 +5,6 @@
set -e
# install startup script
update-rc.d emqx defaults >/dev/null
# create group
if ! getent group emqx >/dev/null; then
addgroup --system emqx
@ -38,6 +35,12 @@ chmod -R 0755 /usr/lib/emqx/bin
ln -s /usr/lib/emqx/bin/emqx /usr/bin/emqx
ln -s /usr/lib/emqx/bin/emqx_ctl /usr/bin/emqx_ctl
if systemctl status --no-pager; then
systemctl enable emqx;
else
echo "systemd is not in use, skip enable emqx"
fi
case "$1" in
configure)
;;

8
deploy/packages/deb/debian/postrm
View File

@ -23,9 +23,6 @@ case "$1" in
purge)
rm -f /etc/default/emqx
# ensure we remove the rc.d scripts installed by postinst
update-rc.d emqx remove >/dev/null
if [ -d /var/lib/emqx ]; then
rm -r /var/lib/emqx
fi
@ -58,6 +55,11 @@ case "$1" in
if [ -d /usr/lib/emqx ]; then
rm -r /usr/lib/emqx
fi
if systemctl status --no-pager; then
systemctl disable emqx;
rm -f /etc/systemd/emqx.service
fi
;;
remove)

6
deploy/packages/deb/debian/rules
View File

@ -46,8 +46,10 @@ install: build
cp -R releases debian/emqx/usr/lib/emqx
cp -R etc/* debian/emqx/etc/emqx
cp -R data/* debian/emqx/var/lib/emqx
install -m755 debian/init.script debian/emqx/etc/init.d/emqx
install -d debian/emqx/lib/systemd/system
install -p -m0644 debian/emqx.service debian/emqx/lib/systemd/system/emqx.service
dh_shlibdeps
# We have nothing to do by default.

1
deploy/packages/deb/emqx.service Normal file
View File

@ -0,0 +1 @@
../emqx.service

13
deploy/packages/emqx.service
View File

@ -7,11 +7,18 @@ User=emqx
Group=emqx
Type=forking
Environment=HOME=/var/lib/emqx
ExecStart=/usr/bin/emqx start
# Must use a 'bash' wrap for some OS
# errno=13 'Permission denied'
# Cannot create FIFO ... for writing
ExecStart=/bin/bash /usr/bin/emqx start
LimitNOFILE=1048576
ExecStop=/usr/bin/emqx stop
ExecStop=/bin/bash /usr/bin/emqx stop
Restart=on-failure
RestartSec=5s
# When clustered, give the peers enough time to get this node's 'DOWN' event
RestartSec=60s
[Install]
WantedBy=multi-user.target

16
deploy/packages/rpm/Makefile
View File

@ -5,8 +5,9 @@ BUILT := $(SRCDIR)/BUILT
dash := -
none :=
space := $(none) $(none)
RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?")
RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]")
## RPM does not allow '-' in version nubmer and release string, replace with '_'
RPM_VSN := $(subst -,_,$(PKG_VSN))
RPM_REL := otp$(subst -,_,$(OTP_VSN))
ARCH ?= amd64
ifeq ($(ARCH),mips64)
@ -16,12 +17,8 @@ endif
EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH)
ifeq ($(RPM_REL),)
# no tail
RPM_REL := 1
endif
SOURCE_PKG := emqx-$(SYSTEM)-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh)
SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi)
# Not $(PWD) as it does not work for make -C
@ -47,11 +44,12 @@ all: | $(BUILT)
--define "_service_dst $(SERVICE_DST)" \
--define "_post_addition $(POST_ADDITION)" \
--define "_preun_addition $(PREUN_ADDITION)" \
--define "_ostype -$(SYSTEM)" \
--define "_sharedstatedir /var/lib" \
emqx.spec
mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME)
cp $(TOPDIR)/RPMS/$(shell uname -m)/$(SOURCE_PKG).rpm $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm
sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm | head -c 64 > \
$(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm.sha256
$(BUILT):
mkdir -p $(TOPDIR) $(SRCDIR) $(SRCDIR)/BUILT

16
deploy/packages/rpm/emqx.service
View File

@ -1,15 +1 @@
[Unit]
Description=emqx daemon
After=network.target
[Service]
User=emqx
Group=emqx
Type=forking
Environment=HOME=/var/lib/emqx
ExecStart=/bin/sh /usr/bin/emqx start
LimitNOFILE=1048576
ExecStop=/bin/sh /usr/bin/emqx stop
[Install]
WantedBy=multi-user.target
../emqx.service

8
deploy/packages/rpm/emqx.spec
View File

@ -5,7 +5,7 @@
%define _log_dir %{_var}/log/%{_name}
%define _lib_home /usr/lib/%{_name}
%define _var_home %{_sharedstatedir}/%{_name}
%define _build_name_fmt %{_arch}/%{_name}%{?_ostype}-%{_version}-%{_release}.%{_arch}.rpm
%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}.%{_arch}.rpm
%define _build_id_links none
Name: %{_package_name}
@ -19,6 +19,12 @@ BuildRoot: %{_tmppath}/%{_name}-%{_version}-root
Provides: %{_name}
AutoReq: 0
%if "%{_arch} %{?rhel}" == "x86_64 7"
Requires: openssl11 libatomic procps which findutils
%else
Requires: libatomic procps which findutils
%endif
%description
EMQX, a distributed, massively scalable, highly extensible MQTT message broker written in Erlang/OTP.

188
docker.mk
View File

@ -1,188 +0,0 @@
#!/usr/bin/make -f
# -*- makefile -*-
## default globals.
## when built with `make docker` command the default profile is either emqx or emqx-ee (for enterprise)
## or the TARGET varialbe can be set beforehand to force a different name
TARGET ?= emqx/$(PROFILE)
QEMU_ARCH ?= x86_64
ARCH ?= amd64
QEMU_VERSION ?= v5.0.0-2
OS ?= alpine
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
ifeq ($(findstring emqx-ee, $(TARGET)), emqx-ee)
ARCH_LIST := amd64 arm64v8 arm32v7
EMQX_NAME := emqx-ee
else ifeq ($(findstring emqx-edge, $(TARGET)), emqx-edge)
ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
EMQX_NAME := emqx-edge
else
ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
EMQX_NAME := emqx
endif
.PHONY: docker
docker: docker-build docker-tag docker-save
.PHONY: docker-prepare
docker-prepare:
## Prepare the machine before any code installation scripts
# @echo "PREPARE: Setting up dependencies."
# @apt update -y
# @apt install --only-upgrade docker-ce -y
## Update docker configuration to enable docker manifest command
@echo "PREPARE: Updating docker configuration"
@mkdir -p $$HOME/.docker
# enable experimental to use docker manifest command
@echo '{ "experimental": "enabled" }' | tee $$HOME/.docker/config.json
# enable experimental
@echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50 }' | tee /etc/docker/daemon.json
@service docker restart
.PHONY: docker-build
docker-build:
## Build Docker image
@echo "DOCKER BUILD: Build Docker image."
@echo "DOCKER BUILD: build version -> $(PKG_VSN)."
@echo "DOCKER BUILD: arch - $(ARCH)."
@echo "DOCKER BUILD: qemu arch - $(QEMU_ARCH)."
@echo "DOCKER BUILD: docker repo - $(TARGET) "
@echo "DOCKER BUILD: emqx name - $(EMQX_NAME)."
## Prepare qemu to build images other then x86_64 on travis
@echo "PREPARE: Qemu" \
&& docker run --rm --privileged multiarch/qemu-user-static:register --reset
@mkdir -p tmp \
&& cd tmp \
&& curl -L -o qemu-$(QEMU_ARCH)-static.tar.gz https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-$(QEMU_ARCH)-static.tar.gz \
&& tar xzf qemu-$(QEMU_ARCH)-static.tar.gz \
&& cd -
@docker build --no-cache \
--build-arg PKG_VSN=$(PKG_VSN) \
--build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine \
--build-arg RUN_FROM=$(ARCH)/alpine:3.12 \
--build-arg EMQX_NAME=$(EMQX_NAME) \
--build-arg QEMU_ARCH=$(QEMU_ARCH) \
--tag $(TARGET):build-$(OS)-$(ARCH) \
-f deploy/docker/Dockerfile .
.PHONY: docker-tag
docker-tag:
@echo "DOCKER TAG: Tag Docker image."
@for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):build-$(OS)-$${arch})" ]; then \
docker tag $(TARGET):build-$(OS)-$${arch} $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
echo "DOCKER TAG: $(TARGET):$(PKG_VSN)-$(OS)-$${arch}"; \
if [ $${arch} = amd64 ]; then \
docker tag $(TARGET):$(PKG_VSN)-$(OS)-amd64 $(TARGET):$(PKG_VSN); \
echo "DOCKER TAG: $(TARGET):$(PKG_VSN)"; \
fi; \
fi; \
done
.PHONY: docker-save
docker-save:
@echo "DOCKER SAVE: Save Docker image."
@mkdir -p _packages/$(EMQX_NAME)
@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
docker save $(TARGET):$(PKG_VSN) > $(EMQX_NAME)-docker-$(PKG_VSN); \
zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN).zip $(EMQX_NAME)-docker-$(PKG_VSN); \
mv ./$(EMQX_NAME)-docker-$(PKG_VSN).zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN).zip; \
fi
@for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker save $(TARGET):$(PKG_VSN)-$(OS)-$${arch} > $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
mv ./$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip; \
fi; \
done
.PHONY: docker-push
docker-push:
@echo "DOCKER PUSH: Push Docker image.";
@echo "DOCKER PUSH: pushing - $(TARGET):$(PKG_VSN).";
@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
docker push $(TARGET):$(PKG_VSN); \
docker tag $(TARGET):$(PKG_VSN) $(TARGET):latest; \
docker push $(TARGET):latest; \
fi;
@for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker push $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
fi; \
done
.PHONY: docker-manifest-list
docker-manifest-list:
version="docker manifest create --amend $(TARGET):$(PKG_VSN)"; \
latest="docker manifest create --amend $(TARGET):latest"; \
for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ];then \
version="$${version} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
latest="$${latest} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
fi; \
done; \
eval $$version; \
eval $$latest;
for arch in $(ARCH_LIST); do \
case $${arch} in \
"amd64") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
fi; \
;; \
"arm64v8") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
fi; \
;; \
"arm32v7") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
fi; \
;; \
"i386") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
fi; \
;; \
"s390x") \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
fi; \
;; \
esac; \
done;
docker manifest inspect $(TARGET):$(PKG_VSN)
docker manifest push $(TARGET):$(PKG_VSN);
docker manifest inspect $(TARGET):latest
docker manifest push $(TARGET):latest;
.PHONY: docker-clean
docker-clean:
@echo "DOCKER CLEAN: Clean Docker image."
@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then docker rmi -f $$(docker images -q $(TARGET):$(PKG_VSN)); fi
@for arch in $(ARCH_LIST); do \
if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
docker rmi -f $$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch}); \
fi \
done

47
elvis.config
View File

@ -1,47 +0,0 @@
%% -*-: erlang -*-
[
{
elvis,
[
{config,
[
#{dirs => ["src", "apps/**/src", "lib-ce/**/src", "lib-ee/**/src"],
filter => "*.erl",
ruleset => erl_files,
rules => [
{elvis_style, state_record_and_type, disable},
{elvis_style, no_common_caveats_call, #{}},
{elvis_style, no_debug_call, #{ debug_functions => [ {ct, pal}
, {ct, print}
]}},
{elvis_style, operator_spaces, #{rules => [{right, "|"},
{left, "|"},
{right, "||"},
{left, "||"}]}}
]
},
#{dirs => ["test", "apps/**/test", "lib-ce/**/src"],
filter => "*.erl",
rules => [
{elvis_text_style, line_length, #{ limit => 100
, skip_comments => false }},
{elvis_style, dont_repeat_yourself, #{ min_complexity => 100 }}
]
},
#{dirs => ["."],
filter => "Makefile",
ruleset => makefiles
},
#{dirs => ["."],
filter => "rebar.config",
ruleset => rebar_config
},
#{dirs => ["."],
filter => "elvis.config",
ruleset => elvis_config
}
]
}
]
}
].

268
etc/emqx.conf
View File

@ -1,4 +1,4 @@
## EMQ X Configuration 4.3
## EMQX Configuration
## NOTE: Do not change format of CONFIG_SECTION_{BGN,END} comments!
@ -101,6 +101,11 @@ cluster.autoclean = 5m
## Value: String
## cluster.dns.app = emqx
## Type of dns record.
##
## Value: Value: a | srv
## cluster.dns.type = a
##--------------------------------------------------------------------
## Cluster using etcd
@ -203,10 +208,10 @@ node.data_dir = {{ platform_data_dir }}
## the heartbeat pings.
##
## NOTE: When managed by systemd (or other supervision tools like systemd),
## heart will probably only cause EMQ X to stop, but restart or not will
## heart will probably only cause EMQX to stop, but restart or not will
## depend on systemd's restart strategy.
## NOTE: When running in docker, the container will die as soon as the the
## heart process kills EMQ X, but restart or not will depend on container
## heart process kills EMQX, but restart or not will depend on container
## supervision strategy, such as k8s restartPolicy.
##
## Value: on
@ -301,6 +306,17 @@ node.crash_dump = {{ platform_log_dir }}/crash.dump
## vm.args: -kernel net_ticktime Number
## node.dist_net_ticktime = 120
## If the host of an Erlang node has many network interfaces,
## this parameter specifies which one to listen on.
## For the type definition of ip_address(), see inet(3).
##
## See: http://www.erlang.org/doc/man/kernel_app.html
##
## Value: IP Address, [0-255].[0-255].[0-255].[0-255]
## Default: 0.0.0.0
## 0.0.0.0 means all network interfaces.
node.dist_use_interface = 0.0.0.0
## Sets the port range for the listener socket of a distributed Erlang node.
## Note that if there are firewalls between clustered nodes, this port segment
## for nodes’ communication should be allowed.
@ -341,6 +357,17 @@ rpc.async_batch_size = 256
## Defaults to `stateless`.
rpc.port_discovery = stateless
## If the host of an Erlang node has many network interfaces,
## this parameter specifies which one for RPC server to listen on.
## For the type definition of ip_address(), see inet(3).
##
## See: http://www.erlang.org/doc/man/kernel_app.html
##
## Value: IP Address, [0-255].[0-255].[0-255].[0-255]
## Default: 0.0.0.0
## 0.0.0.0 means all network interfaces.
#rpc.tcp_server_ip = 0.0.0.0
## TCP port number for RPC server to listen on.
##
## Only takes effect when `rpc.port_discovery` = `manual`.
@ -354,7 +381,7 @@ rpc.port_discovery = stateless
##
## Value: Interger [0-256]
## Default = 1
#rpc.tcp_client_num = 1
#rpc.tcp_client_num = 0
## RCP Client connect timeout.
##
@ -427,7 +454,7 @@ log.to = both
## this level will be logged.
##
## Default: warning
log.level = emergency
log.level = warning
## The dir for log files.
##
@ -460,6 +487,32 @@ log.file = emqx.log
## Value: text | json
log.formatter = json
## Format of the text logger.
##
## Value: rfc3339 | FORMAT
## Where FORMAT is the format string of the timestamp. Supported specifiers:
## %Y: year
## %m: month
## %d: day
## %H: hour
## %M: minute
## %S: second
## %N: nanoseconds (000000000 - 999999999)
## %6N: microseconds (00000 - 999999)
## %3N: milliseconds (000 - 999)
## %z: timezone, [+-]HHMM
## %:z: timezone, [+-]HH:MM
## %::z: timezone, [+-]HH:MM:SS
##
## For example:
## log.formatter.text.date.format = %Y-%m-%dT%H:%M:%S.%6N %:z
##
## Before 4.2, the default date format was:
## log.formatter.text.date.format = %Y-%m-%d %H:%M:%S.%3N
##
## Default: rfc3339
# log.formatter.text.date.format = rfc3339
## Log to single line
## Value: Boolean
#log.single_line = true
@ -600,7 +653,10 @@ log.rotation.count = 5
## Allow anonymous authentication by default if no auth plugins loaded.
## Notice: Disable the option in production deployment!
##
## Value: true | false
## Configured to `false_quick_deny` means quick deny of anonymous clients
## (without username). So the auth backend checks can be skipped.
##
## Value: true | false | false_quick_deny
allow_anonymous = true
## Allow or deny if no ACL rules matched.
@ -647,6 +703,45 @@ acl_deny_action = ignore
## Value: Integer,Duration,Duration
flapping_detect_policy = 30, 1m, 5m
## When using multiple authentication backends, this config can be used to define the order.
## Default value "none" means no explicit ordering.
## Use comma to separate the names or aliases, for example "jwt,http" means "jwt" authentication should be checked before "http".
## Supported names are:
## 'http': emqx_auth_http
## 'jwt': emqx_auth_jwt
## 'ldap': emqx_auth_ldap
## 'mnesia': emqx_auth_mnesia
## 'mongo' (or 'mongodb'): emqx_auth_mongo
## 'mysql': emqx_auth_mysql
## 'pgsql' (or 'postgres'): emqx_auth_pgsql
## 'redis': emqx_auth_redis
## The specified backends are ordered prior to the unspecified ones,
## for example when "mnesia", "jwt" and "http" are in use,
## if only "jwt,http" is configured here, then "mnesia" is ordered after the other two.
## The name can also be the specific callback module name, e.g. my_auth_plugin_module,
## but it is silently discarded if there is no such module found in the system.
auth_order = none
## When using multiple ACL backends, this config can be used to define the order.
## Default value "none" means no explicit ordering.
## Use comma to separate the names or aliases, for example "jwt,http" means "jwt" ACL should be checked before "http".
## Supported names are:
## 'http': emqx_acl_http
## 'internal' (or 'file'): emqx_mod_acl_internal
## 'jwt': emqx_auth_jwt
## 'ldap': emqx_acl_ldap
## 'mnesia': emqx_acl_mnesia
## 'mongo' (or 'mongodb'): emqx_acl_mongo
## 'mysql': emqx_acl_mysql
## 'pgsql' (or 'postgres'): emqx_acl_pgsql
## 'redis': emqx_acl_redis
## The specified backends are ordered prior to the unspecified ones,
## for example when "mnesia", "jwt" and "http" are in use,
## if only "jwt,http" is configured here, then "mnesia" is ordered after the other two.
## The name can also be the specific callback module name, e.g. my_auth_plugin_module,
## but it is silently discarded if there is no such module found in the system.'
acl_order = none
##--------------------------------------------------------------------
## MQTT Protocol
##--------------------------------------------------------------------
@ -694,6 +789,11 @@ mqtt.wildcard_subscription = true
## Value: boolean
mqtt.shared_subscription = true
## Whether the Server supports MQTT Exclusive Subscriptions.
##
## Value: boolean
mqtt.exclusive_subscription = false
## Whether to ignore loop delivery of messages.(for mqtt v3.1.1)
##
## Value: true | false
@ -799,6 +899,11 @@ zone.external.force_gc_policy = 16000|16MB
## Value: boolean
## zone.external.shared_subscription = false
## Whether the Server supports MQTT Exclusive Subscriptions.
##
## Value: boolean
## zone.external.exclusive_subscription = false
## Server Keep Alive
##
## Value: Number
@ -1001,6 +1106,11 @@ zone.internal.acl_deny_action = ignore
## Value: boolean
## zone.internal.shared_subscription = true
## Whether the Server supports MQTT Exclusive Subscriptions.
##
## Value: boolean
## zone.internal.exclusive_subscription = false
## See zone.$name.max_subscriptions.
##
## Value: Integer
@ -1118,7 +1228,7 @@ listener.tcp.external.zone = external
## Example: allow 192.168.0.0/24
listener.tcp.external.access.1 = allow all
## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed
## Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed
## behind HAProxy or Nginx.
##
## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/
@ -1126,7 +1236,7 @@ listener.tcp.external.access.1 = allow all
## Value: on | off
## listener.tcp.external.proxy_protocol = on
## Sets the timeout for proxy protocol. EMQ X will close the TCP connection
## Sets the timeout for proxy protocol. EMQX will close the TCP connection
## if no proxy protocol packet recevied within the timeout.
##
## Value: Duration
@ -1411,6 +1521,59 @@ listener.ssl.external.certfile = /etc/ssl/dgiot/server.pem
##
## Value: File
listener.ssl.external.cacertfile = /etc/ssl/dgiot/ca.pem
## Whether to enable OCSP stapling for the listener. If set to true,
## requires definining the OCSP responder URL.
##
## Value: boolean
## Default: false
## listener.ssl.external.enable_ocsp_stapling = true
## URL for the OCSP responder to check the server certificate against.
##
## Value: String
## listener.ssl.external.ocsp_responder_url = http://my.ocsp.responder.com
## Path to the file containing PEM-encoded certificate of the OCSP
## issuer for the server certificate.
##
## Value: File
## listener.ssl.external.ocsp_issuer_pem = {{ platform_etc_dir }}/certs/ocsp-issuer.pem
## The period to refresh the OCSP response for the server.
##
## Value: Duration
## listener.ssl.external.ocsp_refresh_interval = 5m
## The timeout for the HTTP request when checking OCSP responses.
##
## Value: Duration
## listener.ssl.external.ocsp_refresh_http_timeout = 15s
## Whether to enable CRL verification and caching for this listener.
##
## Value: boolean
## Default: false
## listener.ssl.external.enable_crl_check = true
## Comma-separated URL list for CRL servers to fetch and cache CRLs
## from. Must include the path to the CRL file(s).
##
## Value: String
## listener.ssl.external.crl_cache_urls = http://my.crl.server/intermediate.crl.pem, http://my.other.crl.server/another.crl.pem
## The timeout for the HTTP request when fetching CRLs. This is
## global for all listeners.
##
## Value: Duration
## Default: 15 s
crl_cache_http_timeout = 15s
## The period to refresh the CRLs from the servers. This is global
## for all URLs and listeners.
##
## Value: Duration
## Default: 15 m
crl_cache_refresh_interval = 15m
## The Ephemeral Diffie-Helman key exchange is a very effective way of
## ensuring Forward Secrecy by exchanging a set of keys that never hit
@ -1469,7 +1632,7 @@ listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL
## Note that 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot
## be configured at the same time.
## See 'https://tools.ietf.org/html/rfc4279#section-2'.
#listener.ssl.external.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA
#listener.ssl.external.psk_ciphers = RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384,RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256,RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA
## SSL parameter renegotiation is a feature that allows a client and a server
## to renegotiate the parameters of the SSL connection on the fly.
@ -1510,6 +1673,14 @@ listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL
## Value: cn | dn | crt | pem | md5
## listener.ssl.external.peer_cert_as_clientid = cn
## Default is 'notice', set 'debug' to inspect TLS handshake messaes.
## This log level is not related to EMQX's log level.
##
## NOTE: never set to 'debug' in production environemnts.
##
## Value: emergency | alert | critical | error | warning | notice | info | debug
## listener.ssl.external.log_level = notice
#
## TCP backlog for the SSL connection.
##
## See listener.tcp.$name.backlog
@ -1536,14 +1707,14 @@ listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL
## See: listener.tcp.$name.recbuf
##
## Value: Bytes
## listener.ssl.external.recbuf = 4KB
listener.ssl.external.recbuf = 4KB
## The TCP send buffer(os kernel) for internal MQTT connections.
##
## See: listener.tcp.$name.sndbuf
##
## Value: Bytes
## listener.ssl.external.sndbuf = 4KB
listener.ssl.external.sndbuf = 4KB
## The size of the user-level software buffer used by the driver.
##
@ -1571,6 +1742,19 @@ listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL
## Value: true | false
listener.ssl.external.reuseaddr = true
## Whether to perform GC after TLS/SSL handshake.
##
## Default: false
##
## Value: true | false
## listener.ssl.external.gc_after_handshake = false
## hibernate the SSL process after idling for amount of time
## Default: undefined (off)
##
## Value: Duration
## listener.ssl.external.hibernate_after = 5s
##--------------------------------------------------------------------
## External WebSocket listener for MQTT protocol
@ -1630,13 +1814,13 @@ listener.ws.external.access.1 = allow all
## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5
## listener.ws.external.supported_subprotocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5
## Specify which HTTP header for real source IP if the EMQ X cluster is
## Specify which HTTP header for real source IP if the EMQX cluster is
## deployed behind NGINX or HAProxy.
##
## Default: X-Forwarded-For
## listener.ws.external.proxy_address_header = X-Forwarded-For
## Specify which HTTP header for real source port if the EMQ X cluster is
## Specify which HTTP header for real source port if the EMQX cluster is
## deployed behind NGINX or HAProxy.
##
## Default: X-Forwarded-Port
@ -1994,7 +2178,7 @@ listener.wss.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL
## Note that 'listener.wss.external.ciphers' and 'listener.wss.external.psk_ciphers' cannot
## be configured at the same time.
## See 'https://tools.ietf.org/html/rfc4279#section-2'.
## listener.wss.external.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA
## listener.wss.external.psk_ciphers = RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384,RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256,RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA
## See: listener.ssl.$name.secure_renegotiate
##
@ -2021,6 +2205,10 @@ listener.wss.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL
## Value: cn | dn | crt | pem | md5
## listener.wss.external.peer_cert_as_clientid = cn
## See: listener.ssl.$name.log_level
## Value: emergency | alert | critical | error | warning | notice | info | debug
## listener.wss.external.log_level = notice
## TCP backlog for the WebSocket/SSL connection.
##
## See: listener.tcp.$name.backlog
@ -2214,6 +2402,29 @@ module.presence.qos = 1
## module.rewrite.pub.rule.1 = x/# ^x/y/(.+)$ z/y/$1
## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2
##--------------------------------------------------------------------
## Slow Subscribers Statistics Module
## the expire time of the record which in topk
##
## Value: 5 minutes
#module.slow_subs.expire_interval = 5m
## maximum number of Top-K record
##
## Defalut: 10
#module.slow_subs.top_k_num = 10
## Stats Type
##
## Default: whole
#module.slow_subs.stats_type = whole
## Stats Threshold
##
## Default: 500ms
#module.slow_subs.threshold = 500ms
## CONFIG_SECTION_END=modules ==================================================
##-------------------------------------------------------------------
@ -2265,14 +2476,27 @@ broker.session_locking_strategy = quorum
## Dispatch strategy for shared subscription
##
## Value: Enum
## - hash_clientid
## - hash # same as hash_clientid
## - hash_topic
## - local
## - random
## - round_robin
## - sticky
## - hash # same as hash_clientid
## - hash_clientid
## - hash_topic
broker.shared_subscription_strategy = random
## Per group dispatch strategy for shared subscription
##
## Value: Enum
## - hash_clientid
## - hash # same as hash_clientid
## - hash_topic
## - local
## - random
## - round_robin
## - sticky
#broker.sample_group.shared_subscription_strategy = local
## Enable/disable shared dispatch acknowledgement for QoS1 and QoS2 messages
## This should allow messages to be dispatched to a different subscriber in
## the group in case the picked (based on shared_subscription_strategy) one # is offline
@ -2313,6 +2537,16 @@ broker.route_batch_clean = off
## - false: disable trie path compaction
# broker.perf.trie_compaction = true
## Enable client disconnect event will be triggered by which reasons.
## Value: on | off
## `discarded`: session was discarded by another client with same client ID when new connection use `clean_session = true`.
## Default: off
## `takeover`: session was takeovered by another client with same client ID when new connection use `clean_session = false`.
## Default: off
##
# broker.client_disconnect_discarded = off
# broker.client_disconnect_takeovered = off
## CONFIG_SECTION_BGN=sys_mon ==================================================
## Enable Long GC monitoring. Disable if the value is 0.

4
etc/emqx_cloud/vm.args
View File

@ -1,5 +1,5 @@
######################################################################
## Erlang VM Args for EMQ X Broker
## Erlang VM Args
######################################################################
## NOTE:
@ -115,4 +115,4 @@
-shutdown_time 30000
## patches dir
-pa {{ platform_data_dir }}/patches
-pa "{{ platform_data_dir }}/patches"

4
etc/emqx_edge/vm.args
View File

@ -1,5 +1,5 @@
######################################################################
## Erlang VM Args for EMQ X Edge
## Erlang VM Args
######################################################################
## NOTE:
@ -113,4 +113,4 @@
-shutdown_time 10000
## patches dir
-pa {{ platform_data_dir }}/patches
-pa "{{ platform_data_dir }}/patches"

11
include/emqx.hrl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -85,9 +85,13 @@
%%--------------------------------------------------------------------
-record(route, {
topic :: binary(),
dest :: node() | {binary(), node()}
topic,
dest
}).
-type route() :: #route{
topic :: binary(),
dest :: node() | {binary(), node()}
}.
%%--------------------------------------------------------------------
%% Plugin
@ -132,4 +136,3 @@
}).
-endif.

20
include/emqx_mqtt.hrl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -542,4 +542,22 @@
-define(SHARE(Group, Topic), emqx_topic:join([<<?SHARE>>, Group, Topic])).
-define(IS_SHARE(Topic), case Topic of <<?SHARE, _/binary>> -> true; _ -> false end).
-define(TYPE_NAMES, {
'CONNECT'
, 'CONNACK'
, 'PUBLISH'
, 'PUBACK'
, 'PUBREC'
, 'PUBREL'
, 'PUBCOMP'
, 'SUBSCRIBE'
, 'SUBACK'
, 'UNSUBSCRIBE'
, 'UNSUBACK'
, 'PINGREQ'
, 'PINGRESP'
, 'DISCONNECT'
, 'AUTH'
}).
-endif.

2
include/emqx_release.hrl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.

12
include/logger.hrl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -49,3 +49,13 @@
line => ?LINE}))
end).
%% Copy-paste to avoid changing the old macro which may cause beam md5 changes in a lot of modules
%% i.e. hot-upgrade hell
-define(LOG_SENSITIVE(Level, Format, Args),
begin
(logger:log(Level,#{},#{report_cb => fun(_) -> {'$logger_header'()++(Format), emqx_misc:redact(Args)} end,
mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY},
line => ?LINE,
is_sensitive => true
}))
end).

2
include/types.hrl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.

58
init.sh
View File

@ -1,58 +0,0 @@
#!/usr/bin/env bash
flag=$1
cd ../dgiot-dashboard
git checkout master
git pull
# Judge whether there is a node_ Modules folder
if [ ! -d "node_modules" ]; then
echo "node_modules not found, install node_modules..."
pnpm install &> /dev/null
fi
# Judge whether dist directory is empty
if [ ! -d "dist" ]; then
echo "dist not found, build..."
pnpm build &> /dev/null
fi
cd ./dist/
# Judge apps /dgiot_ api/priv/www/dgiot_dgiot_dashboard Does the folder exist
if [ ! -d "../dgiot/apps/dgiot_api/priv/www/dgiot_dashboard" ]; then
echo "dgiot_dashboard not found, copy..."
wget https://dgiot-release-1306147891.cos.ap-nanjing.myqcloud.com/v4.4.0/dgiot_dashboard.tar.gz &> /dev/null
tar xf dgiot_dashboard.tar.gz &> /dev/null
fi
# Judge apps /dgiot_ api/priv/www/dgiot_file Does the folder exist
if [ ! -d "../dgiot/apps/dgiot_api/priv/www/dgiot_file" ]; then
echo "dgiot_file not found, copy..."
wget https://dgiot-release-1306147891.cos.ap-nanjing.myqcloud.com/v4.4.0/dgiot_file.tar.gz &> /dev/null
tar xf dgiot_file.tar.gz &> /dev/null
fi
# Judge apps /dgiot_api/priv/www/dgiot_dgiot_swagger Does the folder exist
if [ ! -d "../dgiot/apps/dgiot_api/priv/www/dgiot_swagger" ]; then
echo "dgiot_swagger not found, copy..."
wget https://dgiot-release-1306147891.cos.ap-nanjing.myqcloud.com/v4.4.0/dgiot_swagger.tar.gz &> /dev/null
tar xf dgiot_swagger.tar.gz &> /dev/null
fi
# Judge apps dgiot_swagger.tar.gz dgiot_file.tar.gz dgiot_dashboard.tar.gz Does the folder exist
if [ -f "dgiot_swagger.tar.gz" ]; then
rm -rf dgiot_swagger.tar.gz
fi
if [ -f "dgiot_file.tar.gz" ]; then
rm -rf dgiot_file.tar.gz
fi
if [ -f "dgiot_dashboard.tar.gz" ]; then
rm -rf dgiot_dashboard.tar.gz
fi
cd ../../dgiot
git checkout master
git pull
# Determine whether to use git page files
if [ -n "$flag" ] && [ "$flag" = "github" ]; then
echo "install git-page..."
wget https://github.com/dgiot/dgiot-dashboard/archive/refs/heads/gh-pages.zip
unzip gh-pages.zip
mv dgiot-dashboard-gh-pages/ www
fi
rm ./apps/dgiot_api/priv/www -rf
cp ../dgiot-dashboard/dist/ ./apps/dgiot_api/priv/www -rf
make run

2
lib-ce/emqx_dashboard/README.md
View File

@ -2,7 +2,7 @@
emqx-dashboard
==============
Dashboard for the EMQ X Broker.
Dashboard for the EMQX Broker.
REST API
--------

18
lib-ce/emqx_dashboard/etc/emqx_dashboard.conf
View File

@ -1,14 +1,19 @@
##--------------------------------------------------------------------
## EMQ X Dashboard
## EMQX Dashboard
##--------------------------------------------------------------------
## Default user's login name.
##
## For safety, it should be changed as soon as possible.
## Please use the './bin/emqx_ctl admins' CLI to change it.
## Then comment `dashboard.default_user.login/password` from here
## Value: String
dashboard.default_user.login = admin
## Default user's password.
##
## The initial default password for 'dashboard.default_user.login'"
## For safety, it should be changed as soon as possible.
## Please use the './bin/emqx_ctl admins' CLI to change it.
## Then comment `dashboard.default_user.login/password` from here
## Value: String
dashboard.default_user.password = public
@ -77,6 +82,12 @@ dashboard.listener.http.ipv6_v6only = false
## Value: File
## dashboard.listener.https.keyfile = etc/certs/key.pem
## String containing the private key file password. Only used if the
## private keyfile is password-protected.
##
## Value: String
## dashboard.listener.https.key_password = yourpass
## Path to a file containing the user certificate.
##
## Value: File
@ -127,4 +138,3 @@ dashboard.listener.http.ipv6_v6only = false
##
## Value: on | off
## dashboard.listener.https.honor_cipher_order = on

Some files were not shown because too many files have changed in this diff Show More