diff --git a/.ci/acl_migration_test/build.sh b/.ci/acl_migration_test/build.sh new file mode 100644 index 00000000..b7c779f1 --- /dev/null +++ b/.ci/acl_migration_test/build.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -xe + +cd "$EMQX_PATH" + +rm -rf _build _upgrade_base + +mkdir _upgrade_base +pushd _upgrade_base + wget "https://s3-us-west-2.amazonaws.com/packages.emqx/emqx-ce/v${EMQX_BASE}/emqx-ubuntu20.04-${EMQX_BASE}-amd64.zip" +popd + +make emqx-zip diff --git a/.ci/acl_migration_test/prepare.sh b/.ci/acl_migration_test/prepare.sh new file mode 100644 index 00000000..07706867 --- /dev/null +++ b/.ci/acl_migration_test/prepare.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -xe + +mkdir -p "$TEST_PATH" +cd "$TEST_PATH" + +cp ../"$EMQX_PATH"/_upgrade_base/*.zip ./ +unzip ./*.zip + +cp ../"$EMQX_PATH"/_packages/emqx/*.zip ./emqx/releases/ + +git clone --depth 1 https://github.com/terry-xiaoyu/one_more_emqx.git + +./one_more_emqx/one_more_emqx.sh emqx2 diff --git a/.ci/acl_migration_test/suite.sh b/.ci/acl_migration_test/suite.sh new file mode 100644 index 00000000..69c024c8 --- /dev/null +++ b/.ci/acl_migration_test/suite.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -xe + +export EMQX_PATH="$1" +export EMQX_BASE="$2" + +export TEST_PATH="emqx_test" + +./build.sh + +VERSION=$("$EMQX_PATH"/pkg-vsn.sh) +export VERSION + +./prepare.sh + +./test.sh diff --git a/.ci/acl_migration_test/test.sh b/.ci/acl_migration_test/test.sh new file mode 100644 index 00000000..b214a0a5 --- /dev/null +++ b/.ci/acl_migration_test/test.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +set -e + +EMQX_ENDPOINT="http://localhost:8081/api/v4/acl" +EMQX2_ENDPOINT="http://localhost:8917/api/v4/acl" + +function run() { + emqx="$1" + shift + + echo "[$emqx]" "$@" + + pushd "$TEST_PATH/$emqx" + "$@" + popd +} + +function post_rule() { + endpoint="$1" + rule="$2" + echo -n "->($endpoint) " + curl -s -u admin:public -X POST "$endpoint" -d "$rule" + echo +} + +function verify_clientid_rule() { + endpoint="$1" + id="$2" + echo -n "<-($endpoint) " + curl -s -u admin:public "$endpoint/clientid/$id" | grep "$id" || (echo "verify rule for client $id failed" && return 1) +} + +# Run nodes + +run emqx ./bin/emqx start +run emqx2 ./bin/emqx start + +run emqx ./bin/emqx_ctl plugins load emqx_auth_mnesia +run emqx2 ./bin/emqx_ctl plugins load emqx_auth_mnesia + +run emqx2 ./bin/emqx_ctl cluster join 'emqx@127.0.0.1' + +# Add ACL rule to unupgraded EMQX nodes + +post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT1_A","topic": "t", "action": "pub", "access": "allow"}' +post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT1_B","topic": "t", "action": "pub", "access": "allow"}' + +# Upgrade emqx2 node + +run emqx2 ./bin/emqx install "$VERSION" +sleep 60 + +# Verify upgrade blocked + +run emqx2 ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep false || (echo "emqx2 shouldn't have migrated" && exit 1) + +# Verify old rules on both nodes + +verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_A' +verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_A' + +verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_B' +verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_B' + +# Add ACL on OLD and NEW node, verify on all nodes + +post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT2_A","topic": "t", "action": "pub", "access": "allow"}' +post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT2_B","topic": "t", "action": "pub", "access": "allow"}' + +verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_A' +verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_A' + +verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_B' +verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_B' + +# Upgrade emqx node + +run emqx ./bin/emqx install "$VERSION" + +# Wait for upgrade + +sleep 60 + +# Verify if upgrade occured + +run emqx ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep true || (echo "emqx should have migrated" && exit 1) +run emqx2 ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep true || (echo "emqx2 should have migrated" && exit 1) + +# Verify rules are kept + +verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_A' +verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_A' + +verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_B' +verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_B' + +verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_A' +verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_A' + +verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_B' +verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_B' + +# Add ACL on OLD and NEW node, verify on all nodes + +post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT3_A","topic": "t", "action": "pub", "access": "allow"}' +post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT3_B","topic": "t", "action": "pub", "access": "allow"}' + +verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT3_A' +verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT3_A' + +verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT3_B' +verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT3_B' + +# Stop nodes + +run emqx ./bin/emqx stop +run emqx2 ./bin/emqx stop + +echo "Success!" + diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile deleted file mode 100644 index 6f9a1215..00000000 --- a/.ci/build_packages/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 -FROM ${BUILD_FROM} - -ARG EMQX_NAME=emqx - -COPY . /emqx - -WORKDIR /emqx - -RUN rm -rf _build/${EMQX_NAME}/lib _build/${EMQX_NAME}-pkg/lib - -RUN make ${EMQX_NAME}-zip || cat rebar3.crashdump - -RUN make ${EMQX_NAME}-pkg || cat rebar3.crashdump - -RUN /emqx/.ci/build_packages/tests.sh diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 68d8f2df..a51885c1 100644 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -1,28 +1,56 @@ -#!/bin/bash +#!/usr/bin/env bash + +## This script tests built package start/stop +## Accept 2 args PROFILE and PACKAGE_TYPE + set -x -e -u + +if [ -z "${1:-}" ]; then + echo "Usage $0 e.g. emqx, emqx-edge" + exit 1 +fi + +if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then + echo "Usage $0 zip|pkg" + exit 1 +fi + +PROFILE="${1}" +PACKAGE_TYPE="${2}" + export CODE_PATH=${CODE_PATH:-"/emqx"} -export EMQX_NAME=${EMQX_NAME:-"emqx"} -export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" +export PACKAGE_PATH="${CODE_PATH}/_packages/${PROFILE}" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" # export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1" # export EMQX_NODE_COOKIE=$(date +%s%N) -case "$(uname -m)" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH=arm - ;; -esac -export ARCH +SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)" + +if [ "$PACKAGE_TYPE" = 'zip' ]; then + PKG_SUFFIX="zip" +else + case "${SYSTEM:-}" in + ubuntu*|debian*|raspbian*) + PKG_SUFFIX='deb' + ;; + *) + PKG_SUFFIX='rpm' + ;; + esac +fi + +PACKAGE_NAME="${PROFILE}-$("$CODE_PATH"/scripts/pkg-full-vsn.sh)" +OLD_PACKAGE_PATTERN="${PROFILE}-$("$CODE_PATH"/scripts/pkg-full-vsn.sh 'vsn_matcher')" +PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" + +PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" +if ! [ -f "$PACKAGE_FILE" ]; then + echo "$PACKAGE_FILE is not a file" + exit 1 +fi emqx_prepare(){ mkdir -p "${PACKAGE_PATH}" - if [ ! -d "/paho-mqtt-testing" ]; then git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho-mqtt-testing fi @@ -31,82 +59,80 @@ emqx_prepare(){ emqx_test(){ cd "${PACKAGE_PATH}" + local packagename="${PACKAGE_FILE_NAME}" + case "$PKG_SUFFIX" in + "zip") + unzip -q "${PACKAGE_PATH}/${packagename}" + export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \ + EMQX_MQTT__MAX_TOPIC_ALIAS=10 + sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins - for var in "$PACKAGE_PATH"/"${EMQX_NAME}"-*;do - case ${var##*.} in - "zip") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip) - unzip -q "${PACKAGE_PATH}/${packagename}" - export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \ - EMQX_MQTT__MAX_TOPIC_ALIAS=10 - sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins - - echo "running ${packagename} start" - "${PACKAGE_PATH}"/emqx/bin/emqx start || ( tail "${PACKAGE_PATH}"/emqx/log/emqx.log.1 && exit 1 ) - IDLE_TIME=0 - while ! "${PACKAGE_PATH}"/emqx/bin/emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' - do - if [ $IDLE_TIME -gt 10 ] - then - echo "emqx running error" - exit 1 - fi - sleep 10 - IDLE_TIME=$((IDLE_TIME+1)) - done - pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic - "${PACKAGE_PATH}"/emqx/bin/emqx stop - echo "running ${packagename} stop" - rm -rf "${PACKAGE_PATH}"/emqx - ;; - "deb") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.deb) - dpkg -i "${PACKAGE_PATH}/${packagename}" - if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ] + echo "running ${packagename} start" + if ! "${PACKAGE_PATH}"/emqx/bin/emqx start; then + cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true + cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true + exit 1 + fi + IDLE_TIME=0 + while ! "${PACKAGE_PATH}"/emqx/bin/emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' + do + if [ $IDLE_TIME -gt 10 ] then - echo "package install error" + echo "emqx running error" exit 1 fi + sleep 10 + IDLE_TIME=$((IDLE_TIME+1)) + done + pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic + "${PACKAGE_PATH}"/emqx/bin/emqx stop + echo "running ${packagename} stop" + rm -rf "${PACKAGE_PATH}"/emqx + ;; + "deb") + dpkg -i "${PACKAGE_PATH}/${packagename}" + if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ] + then + echo "package install error" + exit 1 + fi - echo "running ${packagename} start" - running_test - echo "running ${packagename} stop" + echo "running ${packagename} start" + running_test + echo "running ${packagename} stop" - dpkg -r "${EMQX_NAME}" - if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "rc" ] - then - echo "package remove error" - exit 1 - fi + dpkg -r "${PROFILE}" + if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "rc" ] + then + echo "package remove error" + exit 1 + fi - dpkg -P "${EMQX_NAME}" - if dpkg -l |grep -q emqx - then - echo "package uninstall error" - exit 1 - fi - ;; - "rpm") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm) - rpm -ivh "${PACKAGE_PATH}/${packagename}" - if ! rpm -q emqx | grep -q emqx; then - echo "package install error" - exit 1 - fi + dpkg -P "${PROFILE}" + if dpkg -l |grep -q emqx + then + echo "package uninstall error" + exit 1 + fi + ;; + "rpm") + yum install -y "${PACKAGE_PATH}/${packagename}" + if ! rpm -q "${PROFILE}" | grep -q "${PROFILE}"; then + echo "package install error" + exit 1 + fi - echo "running ${packagename} start" - running_test - echo "running ${packagename} stop" + echo "running ${packagename} start" + running_test + echo "running ${packagename} stop" - rpm -e "${EMQX_NAME}" - if [ "$(rpm -q emqx)" != "package emqx is not installed" ];then - echo "package uninstall error" - exit 1 - fi - ;; - - esac - done + rpm -e "${PROFILE}" + if [ "$(rpm -q emqx)" != "package emqx is not installed" ];then + echo "package uninstall error" + exit 1 + fi + ;; + esac } running_test(){ @@ -114,7 +140,11 @@ running_test(){ EMQX_MQTT__MAX_TOPIC_ALIAS=10 sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins - emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 ) + if ! emqx start; then + cat /var/log/emqx/erlang.log.1 || true + cat /var/log/emqx/emqx.log.1 || true + exit 1 + fi IDLE_TIME=0 while ! emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' do @@ -129,45 +159,47 @@ running_test(){ pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions emqx stop || kill "$(ps -ef | grep -E '\-progname\s.+emqx\s' |awk '{print $2}')" - - if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \ - || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then - service emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 ) - IDLE_TIME=0 - while ! emqx_ctl status | grep -E 'Node\s.*@.*\sis\sstarted' - do - if [ $IDLE_TIME -gt 10 ] - then - echo "emqx service error" - exit 1 - fi - sleep 10 - IDLE_TIME=$((IDLE_TIME+1)) - done - service emqx stop - fi } relup_test(){ TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh)" - if [ -d "${RELUP_PACKAGE_PATH}" ];then - cd "${RELUP_PACKAGE_PATH}" + if [ ! -d "${RELUP_PACKAGE_PATH}" ];then + return 0 + fi + cd "${RELUP_PACKAGE_PATH}" + while read -r pkg; do + packagename=$(basename "${pkg}") + unzip -q "$packagename" + if ! ./emqx/bin/emqx start; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + exit 1 + fi + ./emqx/bin/emqx_ctl status + ./emqx/bin/emqx versions + OldVsn="$(./emqx/bin/emqx eval 'Versions=[{S, V} || {_,V,_, S} <- release_handler:which_releases()], + Current = proplists:get_value(current, Versions, proplists:get_value(permanent, Versions)), + io:format("~s", [Current])')" + cp "${PACKAGE_PATH}/${PROFILE}-${TARGET_VERSION}"-*.zip ./emqx/releases/ + ./emqx/bin/emqx install "${TARGET_VERSION}" + [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 + export EMQX_WAIT_FOR_STOP=300 + ./emqx/bin/emqx_ctl status - find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" | - while read -r pkg; do - packagename=$(basename "${pkg}") - unzip "$packagename" - ./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 ) - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx versions - cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases - ./emqx/bin/emqx install "${TARGET_VERSION}" - [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx stop - rm -rf emqx - done - fi + # also test remove old rel + ./emqx/bin/emqx uninstall "$OldVsn" + + # check emqx still runs + ./emqx/bin/emqx ping + + if ! ./emqx/bin/emqx stop; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + echo "failed to stop emqx" + exit 1 + fi + rm -rf emqx + done < <(find . -maxdepth 1 -name "${OLD_PACKAGE_PATTERN}.zip") } emqx_prepare diff --git a/.ci/docker-compose-file/conf.env b/.ci/docker-compose-file/conf.env index 93dfecd2..0141a6b4 100644 --- a/.ci/docker-compose-file/conf.env +++ b/.ci/docker-compose-file/conf.env @@ -1,5 +1,5 @@ EMQX_AUTH__LDAP__SERVERS=ldap_server -EMQX_AUTH__MONGO__SERVER=mongo_server:27017 +EMQX_AUTH__MONGO__SERVER=toxiproxy:27017 EMQX_AUTH__MYSQL__SERVER=mysql_server:3306 EMQX_AUTH__MYSQL__USERNAME=root EMQX_AUTH__MYSQL__PASSWORD=public diff --git a/.ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml b/.ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml new file mode 100644 index 00000000..5d21010b --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml @@ -0,0 +1,99 @@ +version: '3.9' + +services: + haproxy: + container_name: haproxy + image: haproxy:2.3 + depends_on: + - emqx1 + - emqx2 + volumes: + - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg + - ../../etc/certs:/usr/local/etc/haproxy/certs + ports: + - "18083:18083" + # - "1883:1883" + # - "8883:8883" + # - "8083:8083" + # - "5683:5683/udp" + # - "9999:9999" + # - "8084:8084" + networks: + - emqx_bridge + working_dir: /usr/local/etc/haproxy + command: + - bash + - -c + - | + cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /usr/local/etc/haproxy/certs/emqx.pem + haproxy -f /usr/local/etc/haproxy/haproxy.cfg + + emqx1: + restart: always + container_name: node1.emqx.io + image: $TARGET:$EMQX_TAG + env_file: + - conf.cluster.env + volumes: + - etc:/opt/emqx/etc + environment: + - "EMQX_HOST=node1.emqx.io" + ports: + - "11881:18083" +# - "1883:1883" + command: + - /bin/sh + - -c + - | + sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf + sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins + /opt/emqx/bin/emqx foreground + healthcheck: + test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"] + interval: 5s + timeout: 25s + retries: 5 + networks: + emqx_bridge: + aliases: + - node1.emqx.io + + emqx2: + restart: always + container_name: node2.emqx.io + image: $TARGET:$EMQX_TAG + env_file: + - conf.cluster.env + volumes: + - etc:/opt/emqx/etc + environment: + - "EMQX_HOST=node2.emqx.io" + ports: + - "11882:18083" + command: + - /bin/sh + - -c + - | + sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf + sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins + /opt/emqx/bin/emqx foreground + healthcheck: + test: ["CMD", "/opt/emqx/bin/emqx", "ping"] + interval: 5s + timeout: 25s + retries: 5 + networks: + emqx_bridge: + aliases: + - node2.emqx.io +volumes: + etc: +networks: + emqx_bridge: + driver: bridge + name: emqx_bridge + ipam: + driver: default + config: + - subnet: 172.100.239.0/24 + gateway: 172.100.239.1 diff --git a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml index 18e1bb6c..3655928e 100644 --- a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml @@ -27,6 +27,7 @@ services: haproxy -f /usr/local/etc/haproxy/haproxy.cfg emqx1: + restart: always container_name: node1.emqx.io image: $TARGET:$EMQX_TAG env_file: @@ -51,6 +52,7 @@ services: - node1.emqx.io emqx2: + restart: always container_name: node2.emqx.io image: $TARGET:$EMQX_TAG env_file: diff --git a/.ci/docker-compose-file/docker-compose-enterprise-tomcat-tcp.yaml b/.ci/docker-compose-file/docker-compose-enterprise-tomcat-tcp.yaml new file mode 100644 index 00000000..90306919 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-enterprise-tomcat-tcp.yaml @@ -0,0 +1,10 @@ +version: '3.9' + +services: + web_server: + container_name: Tomcat + build: + context: ./http-service + image: web-server + networks: + - emqx_bridge diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml new file mode 100644 index 00000000..005ac40d --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -0,0 +1,16 @@ +version: '3.9' + +services: + toxiproxy: + container_name: toxiproxy + image: ghcr.io/shopify/toxiproxy:2.5.0 + restart: always + networks: + - emqx_bridge + volumes: + - "./toxiproxy.json:/config/toxiproxy.json" + ports: + - 8474:8474 + command: + - "-host=0.0.0.0" + - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index a502aeb6..e5270263 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 env_file: - conf.env environment: diff --git a/.ci/docker-compose-file/http-service/Dockerfile b/.ci/docker-compose-file/http-service/Dockerfile new file mode 100644 index 00000000..6601841e --- /dev/null +++ b/.ci/docker-compose-file/http-service/Dockerfile @@ -0,0 +1,15 @@ +FROM tomcat:10.0.5 + +RUN wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip \ + && unzip -q apache-maven-3.6.3-bin.zip \ + && mv apache-maven-3.6.3 /opt/apache-maven-3.6.3/ \ + && ln -s /opt/apache-maven-3.6.3/ /opt/maven +ENV M2_HOME=/opt/maven +ENV M2=$M2_HOME/bin +ENV PATH=$M2:$PATH +COPY ./web-server /code +WORKDIR /code +RUN mvn package -Dmaven.skip.test=true +RUN mv ./target/emqx-web-0.0.1.war /usr/local/tomcat/webapps/emqx-web.war +EXPOSE 8080 +CMD ["/usr/local/tomcat/bin/catalina.sh","run"] diff --git a/.ci/docker-compose-file/http-service/web-server/pom.xml b/.ci/docker-compose-file/http-service/web-server/pom.xml new file mode 100644 index 00000000..7dfd4135 --- /dev/null +++ b/.ci/docker-compose-file/http-service/web-server/pom.xml @@ -0,0 +1,65 @@ + + 4.0.0 + emqx-web + emqx-web + 0.0.1 + war + + + mysql + mysql-connector-java + 8.0.16 + + + commons-dbutils + commons-dbutils + 1.7 + + + commons-logging + commons-logging + 1.2 + + + commons-dbcp + commons-dbcp + 1.4 + + + commons-pool + commons-pool + 1.6 + + + jakarta.servlet + jakarta.servlet-api + 5.0.0 + provided + + + + + + src/main/reousrce + + **/*.java + + + + + + maven-compiler-plugin + 3.8.1 + + 1.8 + 1.8 + + + + maven-war-plugin + 3.2.3 + + + + + \ No newline at end of file diff --git a/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/AuthDAO.java b/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/AuthDAO.java new file mode 100644 index 00000000..61340df4 --- /dev/null +++ b/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/AuthDAO.java @@ -0,0 +1,54 @@ +package com.emqx.dao; + +import java.io.IOException; +import java.sql.SQLException; + +import org.apache.commons.dbutils.QueryRunner; +import org.apache.commons.dbutils.handlers.ScalarHandler; + +import com.emqx.util.EmqxDatabaseUtil; + +public class AuthDAO { + + public String getUserName(String userName) throws IOException, SQLException { + QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource()); + String sql = "select password from http_user where username='"+userName+"'"; + String password =runner.query(sql, new ScalarHandler()); + return password; + } + + public String getClient(String clientid) throws IOException, SQLException { + QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource()); + String sql = "select password from http_user where clientid='"+clientid+"'"; + String password =runner.query(sql, new ScalarHandler()); + return password; + } + + public String getUserAccess(String userName) throws IOException, SQLException { + QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource()); + String sql = "select access from http_acl where username='"+userName+"'"; + String access =runner.query(sql, new ScalarHandler()); + return access; + } + + public String getUserTopic(String userName) throws IOException, SQLException { + QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource()); + String sql = "select topic from http_acl where username='"+userName+"'"; + String topic =runner.query(sql, new ScalarHandler()); + return topic; + } + + public String getClientAccess(String clientid) throws IOException, SQLException { + QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource()); + String sql = "select access from http_acl where clientid='"+clientid+"'"; + String access =runner.query(sql, new ScalarHandler()); + return access; + } + + public String getClientTopic(String clientid) throws IOException, SQLException { + QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource()); + String sql = "select topic from http_acl where clientid='"+clientid+"'"; + String topic =runner.query(sql, new ScalarHandler()); + return topic; + } +} diff --git a/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/DBUtilsTest.java b/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/DBUtilsTest.java new file mode 100644 index 00000000..9836d4b1 --- /dev/null +++ b/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/DBUtilsTest.java @@ -0,0 +1,45 @@ +package com.emqx.dao; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.sql.SQLException; +import java.util.Properties; + +import org.apache.commons.dbcp.BasicDataSource; +import org.apache.commons.dbutils.QueryRunner; +import org.apache.commons.dbutils.handlers.ColumnListHandler; +import org.apache.commons.dbutils.handlers.ScalarHandler; +import org.apache.commons.dbutils.handlers.columns.StringColumnHandler; + + +public class DBUtilsTest { + + public static void main(String args[]) throws FileNotFoundException, IOException, SQLException { + Properties property = new Properties();//流文件 + + property.load(DBUtilsTest.class.getClassLoader().getResourceAsStream("database.properties")); + + BasicDataSource dataSource = new BasicDataSource(); + dataSource.setDriverClassName(property.getProperty("jdbc.driver")); + dataSource.setUrl(property.getProperty("jdbc.url")); + dataSource.setUsername(property.getProperty("jdbc.username")); + dataSource.setPassword(property.getProperty("jdbc.password")); + + // 初始化连接数 if(initialSize!=null) + //dataSource.setInitialSize(Integer.parseInt(initialSize)); + + // 最小空闲连接 if(minIdle!=null) + //dataSource.setMinIdle(Integer.parseInt(minIdle)); + + // 最大空闲连接 if(maxIdle!=null) + //dataSource.setMaxIdle(Integer.parseInt(maxIdle)); + + QueryRunner runner = new QueryRunner(dataSource); + String sql="select username from mqtt_user where id=1"; + String result = runner.query(sql, new ScalarHandler()); + + System.out.println(result); + + } +} diff --git a/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AclServlet.java b/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AclServlet.java new file mode 100644 index 00000000..85915d55 --- /dev/null +++ b/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AclServlet.java @@ -0,0 +1,103 @@ +package com.emqx.servlet; + +import java.io.IOException; +import java.sql.SQLException; + +import com.emqx.dao.AuthDAO; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class AclServlet extends HttpServlet { + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // TODO Auto-generated method stub + doPost(req, resp); + } + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + String clientid = req.getParameter("clientid"); + String username = req.getParameter("username"); + String access = req.getParameter("access"); + String topic = req.getParameter("topic"); + //String password = req.getParameter("password"); + + //step0: password is not null, or not pass. + + AuthDAO dao = new AuthDAO(); + try { + //step1: check username access&topic + if(username != null) { + String access_1 = dao.getUserAccess(username); + String topic_1 = dao.getUserTopic(username); + + if(access.equals(access_1)) { + if(topic.equals(topic_1)) { + resp.setStatus(200); + } + else { + if(clientid != null){ + String access_2 = dao.getClientAccess(clientid); + String topic_2 = dao.getClientTopic(clientid); + if(access.equals(access_2)) { + if(topic.equals(topic_2)) { + resp.setStatus(200); + } + else { + resp.setStatus(400); + } + }else { + resp.setStatus(400); + } + }else { + resp.setStatus(400); + } + } + }else {//step2.1: username password is not match, then check clientid password + if(clientid != null){ + String access_3 = dao.getClientAccess(clientid); + String topic_3 = dao.getClientTopic(clientid); + if(access.equals(access_3)) { + if(topic.equals(topic_3)) { + resp.setStatus(200); + } + else { + resp.setStatus(400); + } + }else { + resp.setStatus(400); + } + }else { + resp.setStatus(400); + } + } + }else {//step2.2: username is null, then check clientid password + if(clientid != null){ + String access_4 = dao.getClientAccess(clientid); + String topic_4 = dao.getClientTopic(clientid); + if(access.equals(access_4)) { + if(topic.equals(topic_4)) { + resp.setStatus(200); + } + else { + resp.setStatus(400); + } + }else { + resp.setStatus(400); + } + }else { + resp.setStatus(400); + } + } + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } +} diff --git a/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AuthServlet.java b/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AuthServlet.java new file mode 100644 index 00000000..a59ca756 --- /dev/null +++ b/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AuthServlet.java @@ -0,0 +1,72 @@ +package com.emqx.servlet; + +import java.io.IOException; +import java.sql.SQLException; + +import com.emqx.dao.AuthDAO; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class AuthServlet extends HttpServlet { + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // TODO Auto-generated method stub + doPost(req, resp); + } + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + String clientid = req.getParameter("clientid"); + String username =req.getParameter("username"); + String password = req.getParameter("password"); + + //step0: password is not null, or not pass. + if(password == null) { + resp.setStatus(400); + return; + } + AuthDAO dao = new AuthDAO(); + try { + //step1: check username password + if(username != null) { + String password_d = dao.getUserName(username); + + if(password.equals(password_d)) { + resp.setStatus(200); + //200 + }else {//step2.1: username password is not match, then check clientid password + if(clientid != null){ + String password_c = dao.getClient(clientid); + if(password.equals(password_c)) { + resp.setStatus(200); + }else { + resp.setStatus(400); + } + }else { + resp.setStatus(400); + } + } + }else {//step2.2: username is null, then check clientid password + if(clientid != null){ + String password_c = dao.getClient(clientid); + if(password.equals(password_c)) { + resp.setStatus(200); + }else { + resp.setStatus(400); + } + }else { + resp.setStatus(400); + } + } + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } +} diff --git a/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/util/EmqxDatabaseUtil.java b/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/util/EmqxDatabaseUtil.java new file mode 100644 index 00000000..b8fb0f22 --- /dev/null +++ b/.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/util/EmqxDatabaseUtil.java @@ -0,0 +1,27 @@ +package com.emqx.util; + +import java.io.IOException; +import java.util.Properties; + +import javax.sql.DataSource; + +import org.apache.commons.dbcp.BasicDataSource; + +import com.emqx.dao.DBUtilsTest; + +public class EmqxDatabaseUtil { + + public static DataSource getDataSource() throws IOException { + Properties property = new Properties();// 流文件 + + property.load(EmqxDatabaseUtil.class.getClassLoader().getResourceAsStream("database.properties")); + + BasicDataSource dataSource = new BasicDataSource(); + dataSource.setDriverClassName(property.getProperty("jdbc.driver")); + dataSource.setUrl(property.getProperty("jdbc.url")); + dataSource.setUsername(property.getProperty("jdbc.username")); + dataSource.setPassword(property.getProperty("jdbc.password")); + + return dataSource; + } +} diff --git a/.ci/docker-compose-file/http-service/web-server/src/main/reousrce/database.properties b/.ci/docker-compose-file/http-service/web-server/src/main/reousrce/database.properties new file mode 100644 index 00000000..11886f34 --- /dev/null +++ b/.ci/docker-compose-file/http-service/web-server/src/main/reousrce/database.properties @@ -0,0 +1,4 @@ +jdbc.driver= com.mysql.jdbc.Driver +jdbc.url= jdbc:mysql://mysql_server:3306/mqtt +jdbc.username= root +jdbc.password= public \ No newline at end of file diff --git a/.ci/docker-compose-file/http-service/web-server/src/main/webapp/META-INF/MANIFEST.MF b/.ci/docker-compose-file/http-service/web-server/src/main/webapp/META-INF/MANIFEST.MF new file mode 100644 index 00000000..254272e1 --- /dev/null +++ b/.ci/docker-compose-file/http-service/web-server/src/main/webapp/META-INF/MANIFEST.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Class-Path: + diff --git a/.ci/docker-compose-file/http-service/web-server/src/main/webapp/WEB-INF/web.xml b/.ci/docker-compose-file/http-service/web-server/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 00000000..e779a454 --- /dev/null +++ b/.ci/docker-compose-file/http-service/web-server/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,31 @@ + + + emqx-web + + Auth + com.emqx.servlet.AuthServlet + + + Acl + com.emqx.servlet.AclServlet + + + Auth + /auth + + + Acl + /acl + + + index.html + index.htm + index.jsp + default.html + default.htm + default.jsp + + \ No newline at end of file diff --git a/.ci/docker-compose-file/http-service/web-server/src/main/webapp/index.html b/.ci/docker-compose-file/http-service/web-server/src/main/webapp/index.html new file mode 100644 index 00000000..2db63b2e --- /dev/null +++ b/.ci/docker-compose-file/http-service/web-server/src/main/webapp/index.html @@ -0,0 +1,10 @@ + + + + +love + + +It's lucky, jiabanxiang. + + \ No newline at end of file diff --git a/.ci/docker-compose-file/python/pytest.sh b/.ci/docker-compose-file/python/pytest.sh index eacbecc3..0189c86b 100644 --- a/.ci/docker-compose-file/python/pytest.sh +++ b/.ci/docker-compose-file/python/pytest.sh @@ -10,7 +10,7 @@ LB="haproxy" apk update && apk add git curl git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing -pip install pytest +pip install pytest==6.2.5 pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$LB" RESULT=$? diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json new file mode 100644 index 00000000..8bc77bdb --- /dev/null +++ b/.ci/docker-compose-file/toxiproxy.json @@ -0,0 +1,8 @@ +[ + { + "name": "mongo_single", + "listen": "0.0.0.0:27017", + "upstream": "mongo:27017", + "enabled": true + } +] diff --git a/.ci/fvt_tests/http_server/rebar.config b/.ci/fvt_tests/http_server/rebar.config index 259f2352..31467926 100644 --- a/.ci/fvt_tests/http_server/rebar.config +++ b/.ci/fvt_tests/http_server/rebar.config @@ -1,7 +1,7 @@ {erl_opts, [debug_info]}. -{deps, +{deps, [ - {minirest, {git, "https://github.com.cnpmjs.org/emqx/minirest.git", {tag, "0.3.1"}}} + {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.6"}}} ]}. {shell, [ diff --git a/.ci/fvt_tests/local_relup_test_run.sh b/.ci/fvt_tests/local_relup_test_run.sh new file mode 100644 index 00000000..0344823c --- /dev/null +++ b/.ci/fvt_tests/local_relup_test_run.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +USAGE="$0 profile vsn old_vsn package_path" +EXAMPLE="$0 emqx 4.3.8-b3bb6075 v4.3.2 /home/alice/relup_dubug/downloaded_packages" + +if [[ $# -ne 4 ]]; then + echo "$USAGE" + echo "$EXAMPLE" + exit 1 +fi + +set -ex + +PROFILE="$1" +VSN="$2" +OLD_VSN="$3" +PACKAGE_PATH="$4" +FROM_OTP_VSN="${5:-24.3.4.2-1}" +TO_OTP_VSN="${6:-24.3.4.2-1}" + +TEMPDIR=$(mktemp -d) +trap '{ rm -rf -- "$TEMPDIR"; }' EXIT + +git clone --branch=master "https://github.com/terry-xiaoyu/one_more_emqx.git" "$TEMPDIR/one_more_emqx" +cp -r "$PACKAGE_PATH" "$TEMPDIR/packages" +cp relup.lux "$TEMPDIR/" +cp -r http_server "$TEMPDIR/http_server" + +exec docker run \ + -v "$TEMPDIR:/relup_test" \ + -w "/relup_test" \ + -e REBAR_COLOR=none \ + -it emqx/relup-test-env:erl23.2.7.2-emqx-3-ubuntu20.04 \ + lux \ + --progress verbose \ + --case_timeout infinity \ + --var PROFILE="$PROFILE" \ + --var PACKAGE_PATH="/relup_test/packages" \ + --var ONE_MORE_EMQX_PATH="/relup_test/one_more_emqx" \ + --var VSN="$VSN" \ + --var OLD_VSN="$OLD_VSN" \ + --var FROM_OTP_VSN="$FROM_OTP_VSN" \ + --var TO_OTP_VSN="$TO_OTP_VSN" \ + relup.lux diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index 462da9ea..d7ed957d 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -1,15 +1,14 @@ [config var=PROFILE] [config var=PACKAGE_PATH] -[config var=BENCH_PATH] [config var=ONE_MORE_EMQX_PATH] [config var=VSN] -[config var=OLD_VSNS] +[config var=OLD_VSN] +[config var=FROM_OTP_VSN] +[config var=TO_OTP_VSN] [config shell_cmd=/bin/bash] [config timeout=600000] -[loop old_vsn $OLD_VSNS] - [shell http_server] !cd http_server !rebar3 shell @@ -21,30 +20,27 @@ ?> [shell emqx] + !OLD_VSN=$(echo $OLD_VSN | sed -r 's/[v|e]//g') !cd $PACKAGE_PATH - !unzip -q -o $PROFILE-ubuntu20.04-$(echo $old_vsn | sed -r 's/[v|e]//g')-amd64.zip + !unzip -q -o $PROFILE-$(echo $OLD_VSN | sed -r 's/[v|e]//g')-otp${FROM_OTP_VSN}-ubuntu20.04-amd64.zip ?SH-PROMPT !cd emqx - !export EMQX_LOG__CONSOLE_HANDLER__ENABLE=true - !export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug - !export EMQX_LOG__PRIMARY_LEVEL=debug - !export EMQX_ZONES__DEFAULT__LISTENERS__MQTT_WSS__BIND="0.0.0.0:8085" + !export EMQX_LOG__LEVEL=debug !./bin/emqx start ?EMQ X .* is started successfully! ?SH-PROMPT [shell emqx2] + !OLD_VSN=$(echo $OLD_VSN | sed -r 's/[v|e]//g') !cd $PACKAGE_PATH !cp -f $ONE_MORE_EMQX_PATH/one_more_$(echo $PROFILE | sed 's/-/_/g').sh . !./one_more_$(echo $PROFILE | sed 's/-/_/g').sh emqx2 ?SH-PROMPT !cd emqx2 - !export EMQX_LOG__CONSOLE_HANDLER__ENABLE=true - !export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug - !export EMQX_LOG__PRIMARY_LEVEL=debug + !export EMQX_LOG__LEVEL=debug !./bin/emqx start ?EMQ X .* is started successfully! @@ -67,6 +63,8 @@ !./bin/emqx_ctl rules create 'SELECT * FROM "t/#"' '[{"name":"data_to_webserver", "params": {"$$resource": "resource:691c29ba"}}]' ?created ?SH-PROMPT + !sleep 5 + ?SH-PROMPT [shell emqx] !./bin/emqx_ctl resources list @@ -75,19 +73,41 @@ !./bin/emqx_ctl rules list ?691c29ba ?SH-PROMPT + !./bin/emqx_ctl broker metrics | grep "messages.publish" + ???SH-PROMPT [shell bench] - !cd $BENCH_PATH - - !./emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300 - ???sent + !emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300 + # e.g. Start with 20 workers, addrs pool size: 1 and req interval: 200 ms + ?^Start [shell emqx] !echo "" > log/emqx.log.1 ?SH-PROMPT - !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/ + ## upgrade to the new version + !./bin/emqx install $VSN + ?Made release permanent: "$VSN" + ?SH-PROMPT + + !./bin/emqx versions |grep permanent + ?(.*)$VSN + ?SH-PROMPT + + ## downgrade to the old version + !./bin/emqx install $${OLD_VSN} + ?Made release permanent:.* + ?SH-PROMPT + + !./bin/emqx versions |grep permanent | grep -qs "$${OLD_VSN}" + ?SH-PROMPT: + !echo ==$$?== + ?^==0== + ?SH-PROMPT: + + ## again, upgrade to the new version !./bin/emqx install $VSN ?Made release permanent: "$VSN" ?SH-PROMPT @@ -103,7 +123,7 @@ """ ?SH-PROMPT - !./bin/emqx_ctl plugins list | grep emqx_management + !./bin/emqx_ctl plugins list | grep --color=never emqx_management ?Plugin\(emqx_management.*active=true\) ?SH-PROMPT @@ -111,8 +131,29 @@ !echo "" > log/emqx.log.1 ?SH-PROMPT - !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/ + ## upgrade to the new version + !./bin/emqx install $VSN + ?Made release permanent: "$VSN" + ?SH-PROMPT + + !./bin/emqx versions |grep permanent + ?(.*)$VSN + ?SH-PROMPT + + ## downgrade to the old version + !./bin/emqx install $${OLD_VSN} + ?Made release permanent:.* + ?SH-PROMPT + + !./bin/emqx versions |grep permanent | grep -qs "$${OLD_VSN}" + ?SH-PROMPT: + !echo ==$$?== + ?^==0== + ?SH-PROMPT: + + ## again, upgrade to the new version !./bin/emqx install $VSN ?Made release permanent: "$VSN" ?SH-PROMPT @@ -128,26 +169,34 @@ """ ?SH-PROMPT - !./bin/emqx_ctl plugins list | grep emqx_management + !./bin/emqx_ctl plugins list | grep --color=never emqx_management ?Plugin\(emqx_management.*active=true\) ?SH-PROMPT [shell bench] - ???publish complete - ??SH-PROMPT: + ?publish complete + ?SH-PROMPT !sleep 30 ?SH-PROMPT - !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq --raw-output ".data[0].metrics[] | select(.node==\"emqx@127.0.0.1\").matched" - ?300 +[shell emqx] + !./bin/emqx_ctl broker metrics | grep "messages.publish" + ???SH-PROMPT + +## We don't guarantee not to lose a single message! +## So even if we received 290~300 messages, we consider it as success +[shell bench] + !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq -M --raw-output ".data[0].metrics[] | select(.node==\"emqx@127.0.0.1\").matched" + ?(29[0-9])|(300) ?SH-PROMPT - !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq --raw-output ".data[0].actions[0].metrics[] | select(.node==\"emqx@127.0.0.1\").success" - ?300 + !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq -M --raw-output ".data[0].actions[0].metrics[] | select(.node==\"emqx@127.0.0.1\").success" + ?(29[0-9])|(300) ?SH-PROMPT + ## The /counter API is provided by .ci/fvt_test/http_server !curl http://127.0.0.1:8080/counter - ???{"data":300,"code":0} + ?\{"data":(29[0-9])|(300),"code":0\} ?SH-PROMPT [shell emqx2] @@ -181,8 +230,6 @@ !halt(3). ?SH-PROMPT: -[endloop] - [cleanup] !echo ==$$?== ?==0== diff --git a/.gitattributes b/.gitattributes index 4ed73da9..8ecb2ae1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ +build text eol=lf * text=auto *.* text eol=lf *.jpg -text diff --git a/.gitignore b/.gitignore index 3af834c3..e6febf20 100644 --- a/.gitignore +++ b/.gitignore @@ -39,8 +39,6 @@ etc/emqx.conf.rendered Mnesia.*/ *.DS_Store _checkouts -./rebar.config -./rebar.config.erl rebar.config.rendered /rebar3 rebar.lock @@ -90,6 +88,9 @@ erlang_ls.config *~ # Emacs temporary files .#* +*# # For direnv .envrc +mix.lock +.gitconfig.tmp *.log diff --git a/.tool-versions b/.tool-versions index 0b6392b9..9ff4a119 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 23.2.7.2-emqx-3 +erlang 24.3.4.2-1 diff --git a/Makefile b/Makefile index 7303a3c2..3d9cda94 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,19 @@ $(shell $(CURDIR)/scripts/git-hooks-init.sh) -REBAR_VERSION = 3.14.3-emqx-7 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts +export EMQX_RELUP ?= true +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-alpine3.15.1 +export EMQX_DEFAULT_RUNNER = alpine:3.15.1 +export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) -export EMQX_DESC ?= EMQ X -export EMQX_CE_DASHBOARD_VERSION ?= v4.3.8 +export DOCKERFILE := deploy/docker/Dockerfile +export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) export REBAR_COLOR=none + FIND=/usr/bin/find +else + FIND=find endif GET_DASHBOARD=$(SCRIPTS)/get-dashboard.sh @@ -16,6 +22,7 @@ PROFILE ?= emqx REL_PROFILES := emqx emqx-edge PKG_PROFILES := emqx-pkg emqx-edge-pkg PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default +CT_READABLE ?= true ifeq ($(OS),Windows_NT) QUIKRUN=$(CURDIR)/_build/$(PROFILE)/rel/emqx/bin/emqx.cmd console @@ -34,7 +41,7 @@ all: $(REBAR) $(PROFILES) .PHONY: ensure-rebar3 ensure-rebar3: @$(SCRIPTS)/fail-on-old-otp-version.escript - @$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION) + @$(SCRIPTS)/ensure-rebar3.sh $(REBAR): ensure-rebar3 @@ -59,11 +66,19 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh) ## app/name-ct targets are intended for local tests hence cover is not enabled .PHONY: $(APPS:%=%-ct) define gen-app-ct-target -$1-ct: - $(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) +$1-ct: $(REBAR) + $(REBAR) ct --name 'test@127.0.0.1' -v --readable $(CT_READABLE) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) +## app/name-ct-pipeline targets are used in pipeline -> make cover data for each app +.PHONY: $(APPS:%=%-ct-pipeline) +define gen-app-ct-target-pipeline +$1-ct-pipeline: $(REBAR) + $(REBAR) ct --name 'test@127.0.0.1' -c -v --readable $(CT_READABLE) --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) +endef +$(foreach app,$(APPS),$(eval $(call gen-app-ct-target-pipeline,$(app)))) + ## apps/name-prop targets .PHONY: $(APPS:%=%-prop) define gen-app-prop-target @@ -81,6 +96,7 @@ coveralls: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send .PHONY: $(REL_PROFILES) + $(REL_PROFILES:%=%): $(REBAR) get-dashboard @$(REBAR) as $(@) do compile,release @@ -93,15 +109,17 @@ $(REL_PROFILES:%=%): $(REBAR) get-dashboard clean: $(PROFILES:%=clean-%) $(PROFILES:%=clean-%): @if [ -d _build/$(@:clean-%=%) ]; then \ - rm rebar.lock \ + rm -f rebar.lock; \ rm -rf _build/$(@:clean-%=%)/rel; \ - find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ - find _build/$(@:clean-%=%) -type l -delete; \ + $(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ + $(FIND) _build/$(@:clean-%=%) -type l -delete; \ fi .PHONY: clean-all clean-all: + @rm -f rebar.lock @rm -rf _build + @rm -f rebar.lock .PHONY: deps-all deps-all: $(REBAR) $(PROFILES:%=deps-%) @@ -116,8 +134,9 @@ $(PROFILES:%=deps-%): $(REBAR) get-dashboard @rm -f rebar.lock .PHONY: xref -xref: $(REBAR) +xref: $(REBAR) $(REL_PROFILES:%=%-rel) @$(REBAR) as check xref + @scripts/xref-check.escript .PHONY: dialyzer dialyzer: $(REBAR) @@ -130,10 +149,19 @@ COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS) $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS) @$(BUILD) $(subst -rel,,$(@)) rel +## download relup base packages +.PHONY: $(REL_PROFILES:%=%-relup-downloads) +define download-relup-packages +$1-relup-downloads: + @if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-base-packages.sh $1; fi +endef +ALL_ZIPS = $(REL_PROFILES) +$(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt)))) + ## relup target is to create relup instructions .PHONY: $(REL_PROFILES:%=%-relup) define gen-relup-target -$1-relup: $(COMMON_DEPS) +$1-relup: $1-relup-downloads $(COMMON_DEPS) @$(BUILD) $1 relup endef ALL_ZIPS = $(REL_PROFILES) @@ -156,6 +184,27 @@ $1: $1-rel endef $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt)))) +## docker target is to create docker instructions +.PHONY: $(REL_PROFILES:%=%-docker) +define gen-docker-target +$1-docker: $(COMMON_DEPS) + @$(BUILD) $1 docker +endef +ALL_ZIPS = $(REL_PROFILES) +$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) + +## emqx-docker-testing +## emqx-ee-docker-testing +## is to directly copy a unzipped zip-package to a +## base image such as ubuntu20.04. Mostly for testing +.PHONY: $(REL_PROFILES:%=%-docker-testing) +define gen-docker-target-testing +$1-docker-testing: $(COMMON_DEPS) + @$(BUILD) $1 docker-testing +endef +ALL_ZIPS = $(REL_PROFILES) +$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt)))) + .PHONY: run run: $(PROFILE) quickrun @@ -166,4 +215,3 @@ ci: $(REBAR) $(PROFILE) .PHONY: quickrun quickrun: @$(QUIKRUN) -include docker.mk diff --git a/Windows.md b/Windows.md index 5e947a22..22794578 100644 --- a/Windows.md +++ b/Windows.md @@ -6,7 +6,7 @@ NOTE: The instructions and examples are based on Windows 10. ### Visual studio for C/C++ compile and link -EMQ X includes Erlang NIF (Native Implmented Function) components, implemented +EMQX includes Erlang NIF (Native Implmented Function) components, implemented in C/C++. To compile and link C/C++ libraries, the easiest way is perhaps to install Visual Studio. @@ -25,17 +25,17 @@ C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build Depending on your visual studio version and OS, the paths may differ. The first path is for rebar3 port compiler to find `cl.exe` and `link.exe` -The second path is for Powershell or CMD to setup environment variables. +The second path is for CMD to setup environment variables. ### Erlang/OTP -Install Erlang/OTP 23.2 from https://www.erlang.org/downloads +Install Erlang/OTP 24.2.1 from https://www.erlang.org/downloads You may need to edit the `Path` environment variable to allow running -Erlang commands such as `erl` from powershell. +Erlang commands such as `erl` from CMD. -To validate Erlang installation in CMD or powershell: +To validate Erlang installation in CMD : -* Start (or restart) CMD or powershell +* Start (or restart) CMD * Execute `erl` command to enter Erlang shell @@ -45,13 +45,13 @@ e.g. ``` PS C:\Users\zmsto> erl -Eshell V11.1.4 (abort with ^G) +Eshell V12.2.1 (abort with ^G) 1> halt(). ``` ### bash -All EMQ X build/run scripts are either in `bash` or `escript`. +All EMQX build/run scripts are either in `bash` or `escript`. `escript` is installed as a part of Erlang. To install a `bash` environment in Windows, there are quite a few options. @@ -63,12 +63,12 @@ Cygwin is what we tested with. to `Path` list. * Validate installation. - Start (restart) CMD or powershell console and execute `which bash`, it should + Start (restart) CMD console and execute `which bash`, it should print out `/usr/bin/bash` ### Other tools -Some of the unix world tools are required to build EMQ X. Including: +Some of the unix world tools are required to build EMQX. Including: * git * curl @@ -84,11 +84,11 @@ When using scoop: scoop install git curl make jq zip unzip ``` -## Build EMQ X source code +## Build EMQX source code * Clone the repo: `git clone https://github.com/emqx/emqx.git` -* Start CMD or Powershell +* Start CMD * Execute `vcvarsall.bat x86_amd64` to load environment variables @@ -112,11 +112,11 @@ scoop install git curl make jq zip unzip To fix it, Visual Studio's bin paths should be ordered prior to Cygwin's (or similar installation's) bin paths in `Path` environment variable. -## Run EMQ X +## Run EMQX -To start EMQ X broker. +To start EMQX broker. -Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQ X. +Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQX. Then execute `_build\emqx\rel\emqx>.\bin\emqx_ctl status` to check status. If everything works fine, it should print out diff --git a/apps/dgiot/conf/rebar.config b/apps/dgiot/conf/rebar.config index 8a5c5b81..1b3a053a 100644 --- a/apps/dgiot/conf/rebar.config +++ b/apps/dgiot/conf/rebar.config @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- %% This config file is the very basic config to compile emqx %% This allows emqx to be used as a dependency for other applications %% such as emqx module/plugin develpments and tests. @@ -6,23 +7,24 @@ %% with rebar.config.erl module. Final result is written to %% rebar.config.rendered if environment DEBUG is set. +{minimum_otp_vsn, "23"}. {edoc_opts, [{preprocess,true}]}. {erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import, - warn_obsolete_guard,compressed, - {d, snk_kind, msg}]}. + warn_obsolete_guard,compressed, + {d, snk_kind, msg}]}. {extra_src_dirs, [{"etc", [{recursive,true}]}]}. {xref_checks,[undefined_function_calls,undefined_functions,locals_not_used, - deprecated_function_calls,warnings_as_errors,deprecated_functions]}. + deprecated_function_calls,warnings_as_errors,deprecated_functions]}. {dialyzer, [ - {warnings, [unmatched_returns, error_handling, race_conditions]}, + {warnings, [unmatched_returns, error_handling]}, {plt_location, "."}, {plt_prefix, "emqx_dialyzer"}, {plt_apps, all_apps}, {statistics, true} -] + ] }. {cover_opts, [verbose]}. @@ -37,39 +39,40 @@ {deps, [ - {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps - , {ehttpc, {git, "https://gitee.com/fastdgiot/ehttpc", {tag, "0.1.14"}}} - , {gproc, {git, "https://gitee.com/fastdgiot/gproc", {tag, "0.8.0"}}} - , {jiffy, {git, "https://gitee.com/fastdgiot/jiffy", {tag, "1.0.5"}}} - , {cowboy, {git, "https://gitee.com/fastdgiot/cowboy", {tag, "2.8.2"}}} - , {esockd, {git, "https://gitee.com/fastdgiot/esockd", {tag, "5.8.0"}}} - , {ekka, {git, "https://gitee.com/fastdgiot/ekka", {tag, "0.8.1.7"}}} - , {gen_rpc, {git, "https://gitee.com/fastdgiot/gen_rpc", {tag, "2.5.1"}}} - , {cuttlefish, {git, "https://gitee.com/fastdgiot/cuttlefish", {tag, "v4.3.7"}}} - , {minirest, {git, "https://gitee.com/fastdgiot/minirest", {tag, "0.3.7"}}} - , {ecpool, {git, "https://gitee.com/fastdgiot/ecpool", {tag, "0.5.2"}}} - , {replayq, {git, "https://gitee.com/fastdgiot/replayq", {tag, "0.3.2"}}} - , {pbkdf2, {git, "https://gitee.com/fastdgiot/erlang-pbkdf2.git", {branch, "2.0.4"}}} - , {emqtt, {git, "https://gitee.com/fastdgiot/emqtt", {tag, "1.2.3.1"}}} - , {rulesql, {git, "https://gitee.com/fastdgiot/rulesql", {tag, "0.1.2"}}} - , {recon, {git, "https://gitee.com/fastdgiot/recon", {tag, "2.5.1"}}} - , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 - , {getopt, "1.0.1"} - , {snabbkaffe, {git, "https://gitee.com/fastdgiot/snabbkaffe.git", {tag, "0.12.2"}}} - , {ejdbc, {git, "https://gitee.com/fastdgiot/ejdbc", {tag, "1.0.1"}}} - , {gun, {git, "https://gitee.com/fastdgiot/gun", {tag, "1.3.5"}}} - , {gen_smtp, {git, "https://gitee.com/fastdgiot/gen_smtp", "1.1.2"}} - , {srly, {git, "https://gitee.com/fastdgiot/srly.git", "1.0.0"}} + {erlfmt, {git, "https://gitee.com/fastdgiot/erlfmt", {tag, "v1.1.0"}}} + , {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps + , {redbug, {git, "https://gitee.com/fastdgiot/redbug.git", {tag, "2.0.7"}}} + , {ehttpc, {git, "https://gitee.com/fastdgiot/ehttpc", {tag, "0.3.0"}}} + , {gun, {git, "https://gitee.com/fastdgiot/gun", {tag, "1.3.7"}}} + , {gproc, {git, "https://gitee.com/fastdgiot/gproc", {tag, "0.8.0"}}} + , {jiffy, {git, "https://gitee.com/fastdgiot/jiffy", {tag, "1.0.5"}}} + , {cowboy, {git, "https://gitee.com/fastdgiot/cowboy", {tag, "2.9.0"}}} + , {esockd, {git, "https://gitee.com/fastdgiot/esockd", {tag, "5.8.7"}}} + , {ekka, {git, "https://gitee.com/fastdgiot/ekka", {tag, "0.8.1.11"}}} + , {gen_rpc, {git, "https://gitee.com/fastdgiot/gen_rpc", {tag, "2.8.1"}}} + , {cuttlefish, {git, "https://gitee.com/fastdgiot/cuttlefish", {tag, "v4.3.7"}}} + , {minirest, {git, "https://gitee.com/fastdgiot/minirest", {tag, "0.3.10"}}} + , {ecpool, {git, "https://gitee.com/fastdgiot/ecpool", {tag, "0.5.2"}}} + , {replayq, {git, "https://gitee.com/fastdgiot/replayq", {tag, "0.3.4"}}} + , {pbkdf2, {git, "https://gitee.com/fastdgiot/erlang-pbkdf2.git", {branch, "2.0.4"}}} + , {emqtt, {git, "https://gitee.com/fastdgiot/emqtt", {tag, "1.2.3.1"}}} + , {rulesql, {git, "https://gitee.com/fastdgiot/rulesql", {tag, "0.1.5"}}} + , {recon, {git, "https://gitee.com/fastdgiot/recon", {tag, "2.5.1"}}} + , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 + , {getopt, "1.0.1"} + , {snabbkaffe, {git, "https://gitee.com/fastdgiot/snabbkaffe.git", {tag, "1.0.1"}}} + , {lc, {git, "https://gitee.com/fastdgiot/lc.git", {tag, "0.3.2"}}} + , {grpc, {git, "https://gitee.com/fastdgiot/grpc-erl", {tag, "0.6.7"}}} ]}. {xref_ignores, - [ %% schema registry is for enterprise - {emqx_schema_registry,get_all_schemas,0}, - {emqx_schema_api,format_schema,1}, - {emqx_schema_api,make_schema_params,1}, - {emqx_schema_parser,decode,3}, - {emqx_schema_parser,encode,3}, - {emqx_schema_registry,add_schema,1}, - emqx_exhook_pb, % generated code for protobuf - emqx_exproto_pb % generated code for protobuf - ]}. + [ %% schema registry is for enterprise + {emqx_schema_registry,get_all_schemas,0}, + {emqx_schema_api,format_schema,1}, + {emqx_schema_api,make_schema_params,1}, + {emqx_schema_parser,decode,3}, + {emqx_schema_parser,encode,3}, + {emqx_schema_registry,add_schema,1}, + emqx_exhook_pb, % generated code for protobuf + emqx_exproto_pb % generated code for protobuf +]}. diff --git a/apps/dgiot/conf/rebar.config.erl b/apps/dgiot/conf/rebar.config.erl index 09215614..aa2ed2ea 100644 --- a/apps/dgiot/conf/rebar.config.erl +++ b/apps/dgiot/conf/rebar.config.erl @@ -18,9 +18,9 @@ bcrypt() -> deps(Config) -> {deps, OldDeps} = lists:keyfind(deps, 1, Config), MoreDeps = case provide_bcrypt_dep() of - true -> [bcrypt()]; - false -> [] - end, + true -> [bcrypt()]; + false -> [] + end, {HasElixir, ExtraDeps} = extra_deps(), {HasElixir, lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps ++ ExtraDeps})}. @@ -48,29 +48,31 @@ filter_extra_deps([{Plugin, _} = P | More], Filter, Acc) -> end. overrides() -> - [{add, [{extra_src_dirs, [{"etc", [{recursive, true}]}]} - , {erl_opts, [{compile_info, [{emqx_vsn, get_vsn()}]}]} - ]} - , {add, snabbkaffe, - [{erl_opts, common_compile_opts()}]} + [ {add, [ {extra_src_dirs, [{"etc", [{recursive,true}]}]} + , {erl_opts, [{compile_info, [{emqx_vsn, get_vsn()}]}]} + ]} + + , {add, relx, [{erl_opts, [{d, 'RLX_LOG', rlx_log}]}]} + , {add, snabbkaffe, + [{erl_opts, common_compile_opts()}]} ] ++ community_plugin_overrides(). community_plugin_overrides() -> - [{add, App, [{erl_opts, [{i, "include"}]}]} || App <- relx_plugin_apps_extra()]. + [{add, App, [ {erl_opts, [{i, "include"}]}]} || App <- relx_plugin_apps_extra()]. config(HasElixir) -> - [{cover_enabled, is_cover_enabled()} - , {profiles, profiles()} - , {project_app_dirs, project_app_dirs()} - , {plugins, plugins(HasElixir)} - | [{provider_hooks, [{pre, [{compile, {mix, find_elixir_libs}}]} - , {post, [{compile, {mix, consolidate_protocols}}]} - ]} || HasElixir] + [ {cover_enabled, is_cover_enabled()} + , {profiles, profiles()} + , {project_app_dirs, project_app_dirs()} + , {plugins, plugins(HasElixir)} + | [ {provider_hooks, [ {pre, [{compile, {mix, find_elixir_libs}}]} + , {post, [{compile, {mix, consolidate_protocols}}]} + ]} || HasElixir ] ]. is_cover_enabled() -> case os:getenv("ENABLE_COVER_COMPILE") of - "1" -> true; + "1"-> true; "true" -> true; _ -> false end. @@ -88,12 +90,11 @@ project_app_dirs() -> ["apps/*", alternative_lib_dir() ++ "/*", "."]. plugins(HasElixir) -> - [{relup_helper, {git, "https://gitee.com/fastdgiot/relup_helper", {tag, "2.0.0"}}} - , {er_coap_client, {git, "https://gitee.com/fastdgiot/er_coap_client", {tag, "v1.0"}}} + [{relup_helper, {git, "https://gitee.com/fastdgiot/relup_helper", {tag, "2.1.0"}}} %% emqx main project does not require port-compiler %% pin at root level for deterministic , {pc, {git, "https://gitee.com/fastdgiot/port_compiler.git", {tag, "v1.11.1"}}} - | [rebar_mix || HasElixir] + | [ {rebar_mix, "v0.4.0"} || HasElixir ] ] %% test plugins are concatenated to default profile plugins %% otherwise rebar3 test profile runs are super slow @@ -101,18 +102,18 @@ plugins(HasElixir) -> test_plugins() -> [rebar3_proper, - {coveralls, {git, "https://gitee.com/fastdgiot/coveralls-erl", {branch, "fix-git-info"}}} + {coveralls, {git, "https://gitee.com/fastdgiot/coveralls-erl", {tag, "v2.2.0-emqx-1"}}} ]. test_deps() -> [{bbmustache, "1.10.0"} - , {emqx_ct_helpers, {git, "https://gitee.com/fastdgiot/emqx-ct-helpers", {tag, "1.3.9"}}} + , {emqx_ct_helpers, {git, "https://gitee.com/fastdgiot/emqx-ct-helpers", {tag, "1.3.11"}}} , meck ]. common_compile_opts() -> [debug_info % alwyas include debug_info - , {compile_info, [{emqx_vsn, get_vsn()}]} + , {compile_info, [{emqx_vsn, get_vsn()}]} , {d, snk_kind, msg} ] ++ [{d, 'EMQX_ENTERPRISE'} || is_enterprise()] ++ @@ -126,58 +127,64 @@ prod_compile_opts() -> ]. prod_overrides() -> - [{add, [{erl_opts, [deterministic]}]}]. - -relup_deps(Profile) -> - {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "scripts/inject-deps.escript " ++ atom_to_list(Profile)}]}. + [{add, [ {erl_opts, [deterministic]}]}]. profiles() -> Vsn = get_vsn(), - [{'emqx', [{erl_opts, prod_compile_opts()} - , {relx, relx(Vsn, cloud, bin)} - , {overrides, prod_overrides()} - , relup_deps('emqx') - ]} - , {'emqx-pkg', [{erl_opts, prod_compile_opts()} - , {relx, relx(Vsn, cloud, pkg)} - , {overrides, prod_overrides()} - , relup_deps('emqx-pkg') - ]} - , {'emqx-edge', [{erl_opts, prod_compile_opts()} - , {relx, relx(Vsn, edge, bin)} - , {overrides, prod_overrides()} - , relup_deps('emqx-edge') - ]} - , {'emqx-edge-pkg', [{erl_opts, prod_compile_opts()} - , {relx, relx(Vsn, edge, pkg)} - , {overrides, prod_overrides()} - , relup_deps('emqx-edge-pkg') - ]} - , {check, [{erl_opts, common_compile_opts()} - ]} - , {test, [{deps, test_deps()} - , {erl_opts, common_compile_opts() ++ erl_opts_i()} - , {extra_src_dirs, [{"test", [{recursive, true}]}]} - ]} + [ {'emqx', [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, cloud, bin)} + , {overrides, prod_overrides()} + ]} + , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, cloud, pkg)} + , {overrides, prod_overrides()} + ]} + , {'emqx-edge', [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, edge, bin)} + , {overrides, prod_overrides()} + ]} + , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, edge, pkg)} + , {overrides, prod_overrides()} + ]} + , {check, [ {erl_opts, common_compile_opts()} + ]} + , {test, [ {deps, test_deps()} + , {erl_opts, common_compile_opts() ++ erl_opts_i()} + , {extra_src_dirs, [{"test", [{recursive,true}]}]} + ]} ] ++ ee_profiles(Vsn). %% RelType: cloud (full size) | edge (slim size) %% PkgType: bin | pkg relx(Vsn, RelType, PkgType) -> IsEnterprise = is_enterprise(), - [{include_src, false} - , {include_erts, true} - , {extended_start_script, false} - , {generate_start_script, false} - , {sys_config, false} - , {vm_args, false} - , {release, {emqx, Vsn}, relx_apps(RelType)} - , {overlay, relx_overlay(RelType)} - , {overlay_vars, [{built_on_arch, rebar_utils:get_arch()} - , {emqx_description, emqx_description(RelType, IsEnterprise)} - | overlay_vars(RelType, PkgType, IsEnterprise)]} + [ {include_src,false} + , {include_erts, true} + , {extended_start_script,false} + , {generate_start_script,false} + , {sys_config,false} + , {vm_args,false} + , {release, {emqx, Vsn}, relx_apps(RelType)} + , {overlay, relx_overlay(RelType)} + , {overlay_vars, [ {built_on_platform, built_on()} + , {emqx_description, emqx_description(RelType, IsEnterprise)} + | overlay_vars(RelType, PkgType, IsEnterprise)]} ]. +built_on() -> + On = rebar_utils:get_arch(), + case distro() of + false -> On; + Distro -> On ++ "-" ++ Distro + end. + +distro() -> + case os:type() of + {unix, _} -> string:strip(os:cmd("scripts/get-distro.sh"), both, $\n); + _ -> false + end. + emqx_description(cloud, true) -> "EMQ X Enterprise"; emqx_description(cloud, false) -> "EMQ X Broker"; emqx_description(edge, _) -> "EMQ X Edge". @@ -198,7 +205,6 @@ overlay_vars_rel(RelType) -> , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce , {enable_plugin_emqx_recon, true} , {enable_plugin_emqx_retainer, true} - , {enable_plugin_emqx_web_hook, true} , {enable_plugin_emqx_telemetry, true} %% dgiot base plugin , {enable_plugin_dgiot, true} @@ -228,54 +234,42 @@ overlay_vars_rel(RelType) -> %% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb) overlay_vars_pkg(bin) -> - [{platform_bin_dir, "bin"} - , {platform_data_dir, "data"} - , {platform_etc_dir, "etc"} - , {platform_lib_dir, "lib"} - , {platform_log_dir, "log"} - , {platform_plugins_dir, "etc/plugins"} - , {runner_root_dir, "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)"} - , {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"} - , {runner_etc_dir, "$RUNNER_ROOT_DIR/etc"} - , {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"} - , {runner_log_dir, "$RUNNER_ROOT_DIR/log"} - , {runner_data_dir, "$RUNNER_ROOT_DIR/data"} - , {runner_user, ""} + [ {platform_bin_dir, "bin"} + , {platform_data_dir, "data"} + , {platform_etc_dir, "etc"} + , {platform_lib_dir, "lib"} + , {platform_log_dir, "log"} + , {platform_plugins_dir, "etc/plugins"} + , {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"} + , {runner_etc_dir, "$RUNNER_ROOT_DIR/etc"} + , {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"} + , {runner_log_dir, "$RUNNER_ROOT_DIR/log"} + , {runner_data_dir, "$RUNNER_ROOT_DIR/data"} + , {runner_user, ""} ]; overlay_vars_pkg(pkg) -> - [{platform_bin_dir, ""} - , {platform_data_dir, "/var/lib/emqx"} - , {platform_etc_dir, "/etc/emqx"} - , {platform_lib_dir, ""} - , {platform_log_dir, "/var/log/emqx"} - , {platform_plugins_dir, "/var/lib/emqx/plugins"} - , {runner_root_dir, "/usr/lib/emqx"} - , {runner_bin_dir, "/usr/bin"} - , {runner_etc_dir, "/etc/emqx"} - , {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"} - , {runner_log_dir, "/var/log/emqx"} - , {runner_data_dir, "/var/lib/emqx"} - , {runner_user, "emqx"} + [ {platform_bin_dir, ""} + , {platform_data_dir, "/var/lib/emqx"} + , {platform_etc_dir, "/etc/emqx"} + , {platform_lib_dir, ""} + , {platform_log_dir, "/var/log/emqx"} + , {platform_plugins_dir, "/var/lib/emqx/plugins"} + , {runner_bin_dir, "/usr/bin"} + , {runner_etc_dir, "/etc/emqx"} + , {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"} + , {runner_log_dir, "/var/log/emqx"} + , {runner_data_dir, "/var/lib/emqx"} + , {runner_user, "emqx"} ]. relx_apps(ReleaseType) -> - [kernel - , sasl - , crypto - , public_key - , asn1 - , syntax_tools - , ssl - , os_mon - , inets - , compiler - , runtime_tools + relx_otp_apps() ++ + [ redbug , cuttlefish , jsx , jesse , jwerl , odbc - , srly , erlydtl , erlport , ecpool @@ -283,7 +277,7 @@ relx_apps(ReleaseType) -> , gpb , poolboy , ibrowse - , gen_smtp + , emqx , {mnesia, load} , {ekka, load} @@ -291,15 +285,19 @@ relx_apps(ReleaseType) -> , observer_cli ] ++ [emqx_modules || not is_enterprise()] - ++ [emqx_license || is_enterprise()] - ++ [bcrypt || provide_bcrypt_release(ReleaseType)] - ++ relx_apps_per_rel(ReleaseType) - ++ [{N, load} || N <- relx_plugin_apps(ReleaseType)]. + ++ [emqx_license || is_enterprise()] + ++ [bcrypt || provide_bcrypt_release(ReleaseType)] + ++ relx_apps_per_rel(ReleaseType) + ++ [{N, load} || N <- relx_plugin_apps(ReleaseType)]. + +relx_otp_apps() -> + {ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"), + true = is_list(Apps), + Apps. relx_apps_per_rel(cloud) -> - [luerl - , xmerl - | [{observer, load} || is_app(observer)] + [ + {observer, load} || is_app(observer) ]; relx_apps_per_rel(edge) -> []. @@ -307,31 +305,28 @@ relx_apps_per_rel(edge) -> is_app(Name) -> case application:load(Name) of ok -> true; - {error, {already_loaded, _}} -> true; + {error,{already_loaded, _}} -> true; _ -> false end. relx_plugin_apps(ReleaseType) -> - [emqx_retainer - , emqx_management - , emqx_dashboard - , emqx_bridge_mqtt - , emqx_recon - , emqx_rule_engine - , emqx_sasl - , emqx_exhook - , emqx_web_hook - , emqx_auth_mnesia + [ emqx_retainer + , emqx_management + , emqx_dashboard + , emqx_bridge_mqtt + , emqx_recon + , emqx_rule_engine + , emqx_sasl + , emqx_auth_mnesia ] ++ [emqx_telemetry || not is_enterprise()] - ++ relx_plugin_apps_per_rel(ReleaseType) - ++ relx_plugin_apps_enterprise(is_enterprise()) - ++ relx_plugin_apps_extra(). + ++ relx_plugin_apps_per_rel(ReleaseType) + ++ relx_plugin_apps_enterprise(is_enterprise()) + ++ relx_plugin_apps_extra(). relx_plugin_apps_per_rel(cloud) -> - [emqx_lwm2m - , emqx_lua_hook - , emqx_exhook + [ + emqx_exhook , emqx_prometheus , emqx_psk_file , emqx_auth_mnesia @@ -362,7 +357,7 @@ relx_plugin_apps_per_rel(edge) -> relx_plugin_apps_enterprise(true) -> [list_to_atom(A) || A <- filelib:wildcard("*", "lib-ee"), - filelib:is_dir(filename:join(["lib-ee", A]))]; + filelib:is_dir(filename:join(["lib-ee", A]))]; relx_plugin_apps_enterprise(false) -> []. relx_plugin_apps_extra() -> @@ -370,29 +365,31 @@ relx_plugin_apps_extra() -> [Plugin || {Plugin, _} <- ExtraDeps]. relx_overlay(ReleaseType) -> - [{mkdir, "log/"} - , {mkdir, "data/"} - , {mkdir, "data/mnesia"} - , {mkdir, "data/configs"} - , {mkdir, "data/patches"} - , {mkdir, "data/scripts"} - , {template, "data/loaded_plugins.tmpl", "data/loaded_plugins"} - , {template, "data/loaded_modules.tmpl", "data/loaded_modules"} - , {template, "data/emqx_vars", "releases/emqx_vars"} - , {copy, "bin/emqx", "bin/emqx"} - , {copy, "bin/emqx_ctl", "bin/emqx_ctl"} - , {copy, "bin/node_dump", "bin/node_dump"} - , {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"} - , {copy, "bin/emqx", "bin/emqx-{{release_version}}"} %% for relup - , {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"} %% for relup - , {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"} %% for relup - , {template, "bin/emqx.cmd", "bin/emqx.cmd"} - , {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"} - , {copy, "bin/nodetool", "bin/nodetool"} - , {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"} - , {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish"} - , {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish-{{release_version}}"} - , {copy, "priv/emqx.schema", "releases/{{release_version}}/"} + [ {mkdir, "log/"} + , {mkdir, "data/"} + , {mkdir, "data/mnesia"} + , {mkdir, "data/configs"} + , {mkdir, "data/patches"} + , {mkdir, "data/scripts"} + , {mkdir, "data/backup"} + , {template, "data/loaded_plugins.tmpl", "data/loaded_plugins"} + , {template, "data/loaded_modules.tmpl", "data/loaded_modules"} + , {template, "data/emqx_vars", "releases/emqx_vars"} + , {copy, "bin/emqx", "bin/emqx"} + , {copy, "bin/emqx_ctl", "bin/emqx_ctl"} + , {copy, "bin/emqx_cluster_rescue", "bin/emqx_cluster_rescue"} + , {copy, "bin/node_dump", "bin/node_dump"} + , {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"} + , {copy, "bin/emqx", "bin/emqx-{{release_version}}"} %% for relup + , {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"} %% for relup + , {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"} %% for relup + , {template, "bin/emqx.cmd", "bin/emqx.cmd"} + , {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"} + , {copy, "bin/nodetool", "bin/nodetool"} + , {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"} + , {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish"} + , {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish-{{release_version}}"} + , {copy, "priv/emqx.schema", "releases/{{release_version}}/"} ] ++ case is_enterprise() of true -> ee_etc_overlay(ReleaseType); false -> etc_overlay(ReleaseType) @@ -401,50 +398,50 @@ relx_overlay(ReleaseType) -> etc_overlay(ReleaseType) -> PluginApps = relx_plugin_apps(ReleaseType), Templates = emqx_etc_overlay(ReleaseType) ++ - lists:append([plugin_etc_overlays(App) || App <- PluginApps]) ++ - [community_plugin_etc_overlays(App) || App <- relx_plugin_apps_extra()], - [{mkdir, "etc/"} - , {mkdir, "etc/plugins"} - , {template, "etc/BUILT_ON", "releases/{{release_version}}/BUILT_ON"} - , {copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"} + lists:append([plugin_etc_overlays(App) || App <- PluginApps]) ++ + [community_plugin_etc_overlays(App) || App <- relx_plugin_apps_extra()], + [ {mkdir, "etc/"} + , {mkdir, "etc/plugins"} + , {template, "etc/BUILT_ON", "releases/{{release_version}}/BUILT_ON"} + , {copy, "{{base_dir}}/lib/emqx/etc/certs","etc/"} ] ++ - lists:map( - fun({From, To}) -> {template, From, To}; - (FromTo) -> {template, FromTo, FromTo} - end, Templates) - ++ extra_overlay(ReleaseType). + lists:map( + fun({From, To}) -> {template, From, To}; + (FromTo) -> {template, FromTo, FromTo} + end, Templates) + ++ extra_overlay(ReleaseType). extra_overlay(cloud) -> - [{copy, "{{base_dir}}/lib/emqx_lwm2m/lwm2m_xml", "etc/"} - , {copy, "{{base_dir}}/lib/emqx_psk_file/etc/psk.txt", "etc/psk.txt"} + [ + {copy, "{{base_dir}}/lib/emqx_psk_file/etc/psk.txt", "etc/psk.txt"} ]; extra_overlay(edge) -> []. emqx_etc_overlay(cloud) -> emqx_etc_overlay_common() ++ - [{"etc/emqx_cloud/vm.args", "etc/vm.args"} + [ {"etc/emqx_cloud/vm.args","etc/vm.args"} ]; emqx_etc_overlay(edge) -> emqx_etc_overlay_common() ++ - [{"etc/emqx_edge/vm.args", "etc/vm.args"} + [ {"etc/emqx_edge/vm.args","etc/vm.args"} ]. emqx_etc_overlay_common() -> ["etc/acl.conf", "etc/emqx.conf", "etc/ssl_dist.conf", - %% TODO: check why it has to end with .paho - %% and why it is put to etc/plugins dir - {"etc/acl.conf.paho", "etc/plugins/acl.conf.paho"}]. + %% TODO: check why it has to end with .paho + %% and why it is put to etc/plugins dir + {"etc/acl.conf.paho", "etc/plugins/acl.conf.paho"}]. plugin_etc_overlays(App0) -> App = atom_to_list(App0), ConfFiles = find_conf_files(App), %% NOTE: not filename:join here since relx translates it for windows - [{"{{base_dir}}/lib/" ++ App ++ "/etc/" ++ F, "etc/plugins/" ++ F} - || F <- ConfFiles]. + [{"{{base_dir}}/lib/"++ App ++"/etc/" ++ F, "etc/plugins/" ++ F} + || F <- ConfFiles]. community_plugin_etc_overlays(App0) -> App = atom_to_list(App0), - {"{{base_dir}}/lib/" ++ App ++ "/etc/" ++ App ++ ".conf", "etc/plugins/" ++ App ++ ".conf"}. + {"{{base_dir}}/lib/"++ App ++"/etc/" ++ App ++ ".conf", "etc/plugins/" ++ App ++ ".conf"}. %% NOTE: for apps fetched as rebar dependency (there is so far no such an app) %% the overlay should be hand-coded but not to rely on build-time wildcards. @@ -465,7 +462,7 @@ get_vsn() -> false -> os:cmd("./pkg-vsn.sh"); Vsn -> Vsn end, - re:replace(PkgVsn, "\n", "", [{return, list}]). + re:replace(PkgVsn, "\n", "", [{return ,list}]). maybe_dump(Config) -> is_debug() andalso file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]), @@ -491,22 +488,22 @@ provide_bcrypt_release(ReleaseType) -> erl_opts_i() -> [{i, "apps"}] ++ - [{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++ - [{i, Dir} || Dir <- filelib:wildcard(filename:join([alternative_lib_dir(), "*", "include"]))]. + [{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++ + [{i, Dir} || Dir <- filelib:wildcard(filename:join([alternative_lib_dir(), "*", "include"]))]. dialyzer(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), AppsToAnalyse = case os:getenv("DIALYZER_ANALYSE_APP") of - false -> - []; - Value -> - [list_to_atom(App) || App <- string:tokens(Value, ",")] - end, + false -> + []; + Value -> + [ list_to_atom(App) || App <- string:tokens(Value, ",")] + end, AppNames = [emqx | list_dir("apps")] ++ list_dir(alternative_lib_dir()), - KnownApps = [Name || Name <- AppsToAnalyse, lists:member(Name, AppNames)], + KnownApps = [Name || Name <- AppsToAnalyse, lists:member(Name, AppNames)], AppsToExclude = AppNames -- KnownApps, @@ -519,21 +516,21 @@ dialyzer(Config) -> coveralls() -> case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of - {"true", Token} when is_list(Token) -> - Cfgs = [{coveralls_repo_token, Token}, + {"true", Token} when is_list(Token) -> + Cfgs = [{coveralls_repo_token, Token}, {coveralls_service_job_id, os:getenv("GITHUB_RUN_ID")}, {coveralls_commit_sha, os:getenv("GITHUB_SHA")}, {coveralls_coverdata, "_build/test/cover/*.coverdata"}, {coveralls_service_name, "github"}], - case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request" - andalso string:tokens(os:getenv("GITHUB_REF"), "/") of - [_, "pull", PRNO, _] -> - [{coveralls_service_pull_request, PRNO} | Cfgs]; - _ -> - Cfgs - end; - _ -> - [] + case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request" + andalso string:tokens(os:getenv("GITHUB_REF"), "/") of + [_, "pull", PRNO, _] -> + [{coveralls_service_pull_request, PRNO} | Cfgs]; + _ -> + Cfgs + end; + _ -> + [] end. list_dir(Dir) -> diff --git a/apps/dgiot/include/logger.hrl b/apps/dgiot/include/logger.hrl index 43fb769c..0799e3d4 100644 --- a/apps/dgiot/include/logger.hrl +++ b/apps/dgiot/include/logger.hrl @@ -1,5 +1,3 @@ --compile({parse_transform, emqx_logger}). - -define(DEBUG(Format), ?LOG(debug, Format, [])). -define(DEBUG(Format, Args), ?LOG(debug, Format, Args)). @@ -26,7 +24,7 @@ -define(LOG(Level, Format, Args), begin (logger:log(Level, #{}, #{ - report_cb => fun(_) -> {'$logger_header'() ++ (Format), (Args)} end, + report_cb => fun(_) -> { (Format), (Args)} end, domain => [dgiot_public], mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}, line => ?LINE})) @@ -35,7 +33,7 @@ -define(LOG(Level, Format, Args, ACL), begin (logger:log(Level, #{}, #{ - report_cb => fun(_) -> {'$logger_header'() ++ (Format), (Args)} end, + report_cb => fun(_) -> {(Format), (Args)} end, domain => ACL, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}, line => ?LINE})) @@ -59,9 +57,10 @@ line => ?LINE})) end). + -define(PLOG(Level, Map), begin - (dgiot_parse:log(#{ + (dgiot_parse_log:log(#{ <<"pid">> => erlang:pid_to_list(self()), <<"time">> => dgiot_datetime:now_microsecs(), <<"node">> => node(), diff --git a/apps/dgiot/include/rule_actions.hrl b/apps/dgiot/include/rule_actions.hrl index 47df33c3..cd47e212 100644 --- a/apps/dgiot/include/rule_actions.hrl +++ b/apps/dgiot/include/rule_actions.hrl @@ -1,4 +1,20 @@ --compile({parse_transform, dgiot_rule_actions_trans}). +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-compile({parse_transform, emqx_rule_actions_trans}). -type selected_data() :: map(). -type env_vars() :: map(). @@ -6,6 +22,11 @@ -define(BINDING_KEYS, '__bindings__'). +-define(LOG_RULE_ACTION(Level, Metadata, Fmt, Args), + emqx_rule_utils:log_action(Level, Metadata, Fmt, Args)). + -define(bound_v(Key, ENVS0), maps:get(Key, maps:get(?BINDING_KEYS, ENVS0, #{}))). + +-define(JWT_TABLE, emqx_rule_engine_jwt_table). diff --git a/apps/dgiot/src/transport/dgiot_mqtt.erl b/apps/dgiot/src/transport/dgiot_mqtt.erl index ef2df8cd..25582350 100644 --- a/apps/dgiot/src/transport/dgiot_mqtt.erl +++ b/apps/dgiot/src/transport/dgiot_mqtt.erl @@ -20,7 +20,16 @@ -include("dgiot_mqtt.hrl"). -include_lib("dgiot/include/logger.hrl"). -include_lib("emqx_rule_engine/include/rule_engine.hrl"). --include_lib("emqx_rule_engine/include/rule_actions.hrl"). +%%-include_lib("emqx_rule_engine/include/rule_actions.hrl"). + +-define(LOG_RULE_ACTION(Level, Metadata, Fmt, Args), + emqx_rule_utils:log_action(Level, Metadata, Fmt, Args)). + +-define(bound_v(Key, ENVS0), + maps:get(Key, + maps:get(?BINDING_KEYS, ENVS0, #{}))). + +-define(BINDING_KEYS, '__bindings__'). %% ETS tables for PubSub -define(SUBOPTION, emqx_suboption). diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 82937d03..2b59d095 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.3.1"}, + {vsn, "4.4.6"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index 9cd66269..b1042d28 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -1,16 +1,44 @@ -%% -*-: erlang -*- - +%% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, [ - {<<"4.3.0">>, [ - {load_module, emqx_plugin_libs_ssl, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ], + {<<"4\\.4\\.[3-5]">>, + [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.2",[ + {load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.0", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}, + {update,emqx_slow_subs,{advanced,["4.4.0"]}}, + {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], [ - {<<"4.3.0">>, [ - {load_module, emqx_plugin_libs_ssl, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ] -}. + {<<"4\\.4\\.[3-5]">>, + [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.3", + [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.2", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.0", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}, + {update,emqx_slow_subs,{advanced,["4.4.0"]}}, + {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl index 2c80dfcf..1026c830 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -58,21 +58,31 @@ save_files_return_opts(Options, Dir) -> KeyFile = Get(<<"keyfile">>), CertFile = Get(<<"certfile">>), CAFile = GetD(<<"cacertfile">>, Get(<<"cafile">>)), - Key = do_save_file(KeyFile, Dir), - Cert = do_save_file(CertFile, Dir), - CA = do_save_file(CAFile, Dir), + Key = maybe_save_file(KeyFile, Dir), + Cert = maybe_save_file(CertFile, Dir), + CA = maybe_save_file(CAFile, Dir), Verify = case GetD(<<"verify">>, false) of false -> verify_none; _ -> verify_peer end, SNI = case Get(<<"server_name_indication">>) of + <<"disable">> -> disable; + "disable" -> disable; + "" -> undefined; + <<>> -> undefined; undefined -> undefined; SNI0 -> ensure_str(SNI0) end, Versions = emqx_tls_lib:integral_versions(Get(<<"tls_versions">>)), Ciphers = emqx_tls_lib:integral_ciphers(Versions, Get(<<"ciphers">>)), - filter([{keyfile, Key}, {certfile, Cert}, {cacertfile, CA}, - {verify, Verify}, {server_name_indication, SNI}, {versions, Versions}, {ciphers, Ciphers}]). + filter([ {keyfile, Key} + , {certfile, Cert} + , {cacertfile, CA} + , {verify, Verify} + , {server_name_indication, SNI} + , {versions, Versions} + , {ciphers, Ciphers} + ]). %% @doc Save a key or certificate file in data dir, %% and return path of the saved file. @@ -80,25 +90,47 @@ save_files_return_opts(Options, Dir) -> -spec save_file(file_input(), atom() | string() | binary()) -> string(). save_file(Param, SubDir) -> Dir = filename:join([emqx:get_env(data_dir), SubDir]), - do_save_file( Param, Dir). + maybe_save_file(Param, Dir). filter([]) -> []; filter([{_, ""} | T]) -> filter(T); filter([{_, undefined} | T]) -> filter(T); filter([H | T]) -> [H | filter(T)]. -do_save_file(#{<<"filename">> := FileName, <<"file">> := Content}, Dir) +maybe_save_file(#{<<"filename">> := FileName, <<"file">> := Content}, Dir) when FileName =/= undefined andalso Content =/= undefined -> - do_save_file(ensure_str(FileName), iolist_to_binary(Content), Dir); -do_save_file(FilePath, _) when is_binary(FilePath) -> + maybe_save_file(ensure_str(FileName), iolist_to_binary(Content), Dir); +maybe_save_file(FilePath, _) when is_binary(FilePath) -> ensure_str(FilePath); -do_save_file(FilePath, _) when is_list(FilePath) -> +maybe_save_file(FilePath, _) when is_list(FilePath) -> FilePath; -do_save_file(_, _) -> "". +maybe_save_file(_, _) -> "". -do_save_file("", _, _Dir) -> ""; %% ignore -do_save_file(_, <<>>, _Dir) -> ""; %% ignore -do_save_file(FileName, Content, Dir) -> +maybe_save_file("", _, _Dir) -> ""; %% no filename, ignore +maybe_save_file(FileName, <<>>, Dir) -> %% no content, see if file exists + {ok, Cwd} = file:get_cwd(), + %% NOTE: when FileName is an absolute path, filename:join has no effect + CwdFile = ensure_str(filename:join([Cwd, FileName])), + DataDirFile = ensure_str(filename:join([Dir, FileName])), + Possibles0 = case CwdFile =:= DataDirFile of + true -> [CwdFile]; + false -> [CwdFile, DataDirFile] + end, + Possibles = Possibles0 ++ + case FileName of + "etc/certs/" ++ Path -> + %% this is the dir hard-coded in rule-engine resources as + %% default, unfortunatly we cannot change the deaults + %% due to compatibilty reasons, so we have to make a guess + ["/etc/emqx/certs/" ++ Path]; + _ -> + [] + end, + case find_exist_file(FileName, Possibles) of + false -> erlang:throw({bad_cert_file, Possibles}); + Found -> Found + end; +maybe_save_file(FileName, Content, Dir) -> FullFilename = filename:join([Dir, FileName]), ok = filelib:ensure_dir(FullFilename), case file:write_file(FullFilename, Content) of @@ -112,3 +144,9 @@ do_save_file(FileName, Content, Dir) -> ensure_str(L) when is_list(L) -> L; ensure_str(B) when is_binary(B) -> unicode:characters_to_list(B, utf8). +find_exist_file(_Name, []) -> false; +find_exist_file(Name, [F | Rest]) -> + case filelib:is_regular(F) of + true -> F; + false -> find_exist_file(Name, Rest) + end. diff --git a/baiduMap_upgrade.sh b/baiduMap_upgrade.sh deleted file mode 100644 index 54019a72..00000000 --- a/baiduMap_upgrade.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -#wget -qO baiduMap_upgrade.sh https://gitee.com/dgiiot/dgiot/raw/master/baiduMap_upgrade.sh && chmod 777 baiduMap_upgrade.sh && sh baiduMap_upgrade.sh 'WpeAb6pL4tsX2ZVd11156GHbO9Ut6c4HZhG' - -#if [ ! -d "dgiot-dashboard" ]; then -echo "当前进度: 安装node" -mkdir /usr/local/node/ -cd /usr/local/node/ -wget https://npm.taobao.org/mirrors/node/v14.17.6/node-v14.17.6-linux-x64.tar.gz -tar -zxvf node-v14.17.6-linux-x64.tar.gz -rm -rf node-v14.17.6-linux-x64.tar.gz -ln -s /usr/local/node/node-v14.17.6-linux-x64/bin/npm /usr/local/bin/npm -ln -s /usr/local/node/node-v14.17.6-linux-x64/bin/node /usr/local/bin/node -node -v -#fi - -mapKey=$1 -echo "你输入的百度地图key为: ${mapKey}" -cd /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv - -if [ ! -d "dgiot-dashboard" ]; then - echo "当前进度: 克隆dgiot-dashboard" - git clone --depth 1 https://gitee.com/dgiiot/dgiot-dashboard.git dgiot-dashboard -fi - -cd dgiot-dashboard -if [ ! -d "node_modules" ]; then - echo "进度: 安装依赖" - echo "node_modules not found, install node_modules..." - npm install -fi -echo "进度: 替换key" -sed -i "s/'WpeAb6pL4tsX2ZVd56GHbO9Ut6c4HZhG'/${mapKey}/g" /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/dgiot-dashboard/src/config/secret.config.js -echo "进度: 打包编译" -npm build -echo "进度: 拷贝文件夹dist" -mv /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/www /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/wwwback -mkdir /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/www -cp -r /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/dgiot-dashboard/dist/* /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/www/ -echo "进度: 删除下载文件dgiot-dashboard" -#rm -rf /data/dgiot/dgiot/lib/dgiot_api-4.3.0/priv/dgiot-dashboard -echo '替换百度地图key完成' diff --git a/bin/cuttlefish-4.3.10 b/bin/cuttlefish-4.3.10 deleted file mode 100644 index 6748a22e..00000000 Binary files a/bin/cuttlefish-4.3.10 and /dev/null differ diff --git a/bin/emqx b/bin/emqx index e68c1f8c..95840a1c 100755 --- a/bin/emqx +++ b/bin/emqx @@ -1,12 +1,22 @@ -#!/bin/bash +#!/usr/bin/env bash # -*- tab-width:4;indent-tabs-mode:nil -*- # ex: ts=4 sw=4 et set -e -ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" +DEBUG="${DEBUG:-0}" +if [ "$DEBUG" -eq 1 ]; then + set -x +fi + +RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" # shellcheck disable=SC1090 -. "$ROOT_DIR"/releases/emqx_vars +. "$RUNNER_ROOT_DIR"/releases/emqx_vars + +EMQX_LICENSE_CONF='' +REL_NAME="emqx" +ERTS_PATH="$RUNNER_ROOT_DIR/erts-$ERTS_VSN/bin" +export EMQX_DESCRIPTION RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME" CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}" @@ -28,91 +38,166 @@ export PROGNAME="erl" DYNLIBS_DIR="$RUNNER_ROOT_DIR/dynlibs" ERTS_LIB_DIR="$ERTS_DIR/../lib" -# Echo to stderr on errors -echoerr() { echo "$*" 1>&2; } - -check_eralng_start() { - "$BINDIR/$PROGNAME" -noshell -boot "$REL_DIR/start_clean" -s crypto start -s init stop -} - -if ! check_eralng_start >/dev/null 2>&1; then - BUILT_ON="$(head -1 "${REL_DIR}/BUILT_ON")" - ## failed to start, might be due to missing libs, try to be portable - export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH" - if ! check_eralng_start; then - ## it's hopeless - echoerr "FATAL: Unable to start Erlang (with libcrypto)." - echoerr "Please make sure it's running on the correct platform with all required dependencies." - echoerr "This EMQ X release is built for $BUILT_ON" - exit 1 - fi - echoerr "WARNING: There seem to be missing dynamic libs from the OS. Using libs from ${DYNLIBS_DIR}" -fi - -## backward compatible -if [ -d "$ERTS_DIR/lib" ]; then - export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH" -fi +# Fix bin permission for all erts bin files +# the 'x' attributes may get lost if the files are extracted from a relup package +find "$BINDIR" -exec chmod a+x {} \; # cuttlefish try to read environment variables starting with "EMQX_" export CUTTLEFISH_ENV_OVERRIDE_PREFIX='EMQX_' -relx_usage() { - command="$1" +usage() { + local command="$1" case "$command" in - unpack) - echo "Usage: $REL_NAME unpack [VERSION]" - echo "Unpacks a release package VERSION, it assumes that this" - echo "release package tarball has already been deployed at one" - echo "of the following locations:" - echo " releases/-.tar.gz" - echo " releases/-.zip" - ;; - install) - echo "Usage: $REL_NAME install [VERSION]" - echo "Installs a release package VERSION, it assumes that this" - echo "release package tarball has already been deployed at one" - echo "of the following locations:" - echo " releases/-.tar.gz" - echo " releases/-.zip" - echo "" - echo " --no-permanent Install release package VERSION but" - echo " don't make it permanent" - ;; - uninstall) - echo "Usage: $REL_NAME uninstall [VERSION]" - echo "Uninstalls a release VERSION, it will only accept" - echo "versions that are not currently in use" - ;; - upgrade) - echo "Usage: $REL_NAME upgrade [VERSION]" - echo "Upgrades the currently running release to VERSION, it assumes" - echo "that a release package tarball has already been deployed at one" - echo "of the following locations:" - echo " releases/-.tar.gz" - echo " releases/-.zip" - echo "" - echo " --no-permanent Install release package VERSION but" - echo " don't make it permanent" - ;; - downgrade) - echo "Usage: $REL_NAME downgrade [VERSION]" - echo "Downgrades the currently running release to VERSION, it assumes" - echo "that a release package tarball has already been deployed at one" - echo "of the following locations:" - echo " releases/-.tar.gz" - echo " releases/-.zip" - echo "" - echo " --no-permanent Install release package VERSION but" - echo " don't make it permanent" - ;; - *) - echo "Usage: $REL_NAME {start|start_boot |ertspath|foreground|stop|restart|reboot|pid|ping|console|console_clean|console_boot |attach|remote_console|upgrade|downgrade|install|uninstall|versions|escript|rpc|rpcterms|eval|root_dir}" - ;; + start) + echo "Start EMQX service in daemon mode" + ;; + stop) + echo "Stop the running EMQX program" + ;; + console) + echo "Boot up EMQX service in an interactive Erlang shell" + echo "This command needs a tty" + ;; + console_clean) + echo "This command does NOT boot up the EMQX service" + echo "It only starts an interactive Erlang shell with all the" + echo "EMQX code available" + ;; + foreground) + echo "Start EMQX in foreground mode without an interactive shell" + ;; + pid) + echo "Print out EMQX process identifier" + ;; + ping) + echo "Check if the EMQX node is up and running" + echo "This command exit with 0 silently if node is running" + ;; + escript) + echo "Execute a escript using the Erlang runtime from EMQX package installation" + echo "For example $REL_NAME escript /path/to/my/escript my_arg1 my_arg2" + ;; + attach) + echo "This command is applicable when EMQX is started in daemon mode." + echo "It attaches the current shell to EMQX's control console" + echo "through a named pipe." + echo "WARNING: try to use the safer alternative, remote_console command." + ;; + remote_console) + echo "Start an interactive shell running an Erlang node which " + echo "hidden-connects to the running EMQX node". + echo "This command is mostly used for troubleshooting." + ;; + ertspath) + echo "Print path to Erlang runtime dir" + ;; + rpc) + echo "Usge $REL_NAME rpc MODULE FUNCTION [ARGS, ...]" + echo "Connect to the EMQX node and make an Erlang RPC" + echo "This command blocks for at most 60 seconds." + echo "It exits with non-zero code in case of any RPC failure" + echo "including connection error and runtime exception" + ;; + rpcterms) + echo "Usge $REL_NAME rpcterms MODULE FUNCTION [ARGS, ...]" + echo "Connect to the EMQX node and make an Erlang RPC" + echo "The result of the RPC call is pretty-printed as an " + echo "Erlang term" + ;; + root_dir) + echo "Print EMQX installation root dir" + ;; + eval) + echo "Evaluate an Erlang expression in the EMQX node" + ;; + versions) + echo "List installed EMQX versions and their status" + ;; + unpack) + echo "Usage: $REL_NAME unpack [VERSION]" + echo "Unpacks a release package VERSION, it assumes that this" + echo "release package tarball has already been deployed at one" + echo "of the following locations:" + echo " releases/-.zip" + ;; + install) + echo "Usage: $REL_NAME install [VERSION]" + echo "Installs a release package VERSION, it assumes that this" + echo "release package tarball has already been deployed at one" + echo "of the following locations:" + echo " releases/-.zip" + echo "" + echo " --no-permanent Install release package VERSION but" + echo " don't make it permanent" + ;; + uninstall) + echo "Usage: $REL_NAME uninstall [VERSION]" + echo "Uninstalls a release VERSION, it will only accept" + echo "versions that are not currently in use" + ;; + upgrade) + echo "Usage: $REL_NAME upgrade [VERSION]" + echo "Upgrades the currently running release to VERSION, it assumes" + echo "that a release package tarball has already been deployed at one" + echo "of the following locations:" + echo " releases/-.zip" + echo "" + echo " --no-permanent Install release package VERSION but" + echo " don't make it permanent" + ;; + downgrade) + echo "Usage: $REL_NAME downgrade [VERSION]" + echo "Downgrades the currently running release to VERSION, it assumes" + echo "that a release package tarball has already been deployed at one" + echo "of the following locations:" + echo " releases/-.zip" + echo "" + echo " --no-permanent Install release package VERSION but" + echo " don't make it permanent" + ;; + *) + echo "Usage: $REL_NAME COMMAND [help]" + echo '' + echo "Commonly used COMMANDs:" + echo " start: Start EMQX in daemon mode" + echo " console: Start EMQX in an interactive Erlang shell" + echo " foreground: Start EMQX in foreground mode without an interactive shell" + echo " stop: Stop the running EMQX node" + echo " restart: Restart the running EMQX node" + echo " ctl: Administration commands, execute '$REL_NAME ctl help' for more details" + echo '' + echo "More:" + echo " Shell attach: remote_console | attach" + echo " Up/Down-grade: upgrade | downgrade | install | uninstall" + echo " Install info: ertspath | root_dir | versions" + echo " Runtime info: pid | ping | versions" + echo " Config check: check_conf" + echo " Advanced: console_clean | escript | rpc | rpcterms | eval" + echo '' + echo "Execute '$REL_NAME COMMAND help' for more information" + ;; esac } +COMMAND="${1:-}" + +if [ -z "$COMMAND" ]; then + usage 'help' + exit 1 +elif [ "$COMMAND" = 'help' ]; then + usage 'help' + exit 0 +fi + +if [ "${2:-}" = 'help' ]; then + ## 'ctl' command has its own usage info + if [ "$COMMAND" != 'ctl' ]; then + usage "$COMMAND" + exit 0 + fi +fi + # Simple way to check the correct user and fail early check_user() { # Validate that the user running the script is the owner of the @@ -132,7 +217,6 @@ check_user() { fi } - # Make sure the user running this script is the owner and/or su to that user check_user "$@" ES=$? @@ -140,10 +224,54 @@ if [ "$ES" -ne 0 ]; then exit $ES fi +# Echo to stderr on errors +echoerr() { echo "$*" 1>&2; } + +die() { + set +x + echoerr "ERROR: $1" + errno=${2:-1} + exit "$errno" +} + +assert_node_alive() { + if ! relx_nodetool "ping" > /dev/null; then + die "node_is_not_running!" 1 + fi +} + +check_erlang_start() { + # set ERL_CRASH_DUMP_BYTES to zero so it will not write a crash dump file + env ERL_CRASH_DUMP_BYTES=0 "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()" +} + +if ! check_erlang_start >/dev/null 2>&1; then + BUILT_ON="$(head -1 "${REL_DIR}/BUILT_ON")" + ## failed to start, might be due to missing libs, try to be portable + export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH" + if ! check_erlang_start; then + ## it's hopeless + echoerr "FATAL: Unable to start Erlang." + echoerr "Please make sure openssl-1.1.1 (libcrypto) and libncurses are installed." + echoerr "Also ensure it's running on the correct platform," + echoerr "this EMQX release is built for $BUILT_ON" + exit 1 + fi + echoerr "There seem to be missing dynamic libs from the OS." + echoerr "Using libs from ${DYNLIBS_DIR} instead." + echoerr "NOTE: EMQX's rpm or deb package installation is recommended!" +fi + +## backward compatible (old EMQX versions does this) +if [ -d "$ERTS_DIR/lib" ]; then + export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH" +fi + if [ -z "$WITH_EPMD" ]; then EPMD_ARG="-start_epmd false -epmd_module ekka_epmd -proto_dist ekka" else - EPMD_ARG="-start_epmd true $PROTO_DIST_ARG" + PROTO_DIST=$(grep -E '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}') + EPMD_ARG="-start_epmd true -proto_dist $PROTO_DIST" fi # Warn the user if ulimit -n is less than 1024 @@ -154,9 +282,6 @@ if [ "$ULIMIT_F" -lt 1024 ]; then echo "!!!!" fi -# By default, use cuttlefish to generate app.config and vm.args -CUTTLEFISH="${USE_CUTTLEFISH:-yes}" - SED_REPLACE="sed -i " case $(sed --help 2>&1) in *GNU*) SED_REPLACE="sed -i ";; @@ -227,75 +352,119 @@ relx_start_command() { "$START_OPTION" } +trim() { + echo -e "${1}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' +} + # Function to generate app.config and vm.args generate_config() { - ## Delete the *.siz files first or it cann't start after - ## changing the config 'log.rotation.size' - rm -rf "${RUNNER_LOG_DIR}"/*.siz + check_only="$1" + if [ "$check_only" != "check_only" ]; then + ## Delete the *.siz files first or it cann't start after + ## changing the config 'log.rotation.size' + rm -rf "${RUNNER_LOG_DIR}"/*.siz + fi - if [ "$CUTTLEFISH" != "yes" ]; then - # Note: we have added a parameter '-vm_args' to this. It - # appears redundant but it is not! the erlang vm allows us to - # access all arguments to the erl command EXCEPT '-args_file', - # so in order to get access to this file location from within - # the vm, we need to pass it in twice. - CONFIG_ARGS=" -config $RUNNER_ETC_DIR/app.config -args_file $RUNNER_ETC_DIR/vm.args -vm_args $RUNNER_ETC_DIR/vm.args " + set +e + if [ "${EMQX_LICENSE_CONF:-}" = "" ]; then + CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" else - EMQX_LICENSE_CONF_OPTION="" - if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then - EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}" - fi + CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -i "${EMQX_LICENSE_CONF}" -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + fi + # shellcheck disable=SC2181 + RESULT=$? + set -e + if [ $RESULT -gt 0 ]; then + echo "$CUTTLEFISH_OUTPUT" + exit $RESULT + fi + ## transform a single line args list like '-config ... -args_file ... -vm_args ...' to lines and get path for each file respectively + ## NOTE: the -args_file and -vm_args are the same file passed twice because args_file is used by beam, but not possible to get at runtime + ## by calling init:get_arguments/0 + lines="$(echo "$CUTTLEFISH_OUTPUT" | tail -1 \ + | sed -e $'s/-config/\\\nconfig=/g' \ + | sed -e $'s/-args_file/\\\nargs_file=/g' \ + | sed -e $'s/-vm_args/\\\nvm_args=/g')" + CONFIG_FILE="$(trim "$(echo -e "$lines" | grep 'config=' | sed 's/config=//g')")" + CUTTLE_GEN_ARG_FILE="$(trim "$(echo -e "$lines" | grep 'vm_args=' | sed 's/vm_args=//g')")" - set +e - # shellcheck disable=SC2086 - CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" - # shellcheck disable=SC2181 - RESULT=$? - set -e - if [ $RESULT -gt 0 ]; then - echo "$CUTTLEFISH_OUTPUT" - exit $RESULT - fi - # print override from environment variables (EMQX_*) - echo "$CUTTLEFISH_OUTPUT" | sed -e '$d' - CONFIG_ARGS=$(echo "$CUTTLEFISH_OUTPUT" | tail -n 1) - - ## Merge cuttlefish generated *.args into the vm.args - CUTTLE_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}') - TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp" - cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE" - echo "" >> "$TMP_ARG_FILE" - echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE" - sed '/^#/d' "$CUTTLE_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do - ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') - ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') - if [ "$ARG_KEY" = '' ]; then - ## for the flags, e.g. -heart -emu_args etc - ARG_KEY=$(echo "$ARG_LINE" | awk '{print $1}') - ARG_VALUE='' - TMP_ARG_KEY=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $1}') - if [ "$TMP_ARG_KEY" = '' ]; then - echo "$ARG_KEY" >> "$TMP_ARG_FILE" - fi - else - TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}') - if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then - if [ -n "$TMP_ARG_VALUE" ]; then - sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' $TMP_ARG_FILE" - else - echo "$ARG_LINE" >> "$TMP_ARG_FILE" - fi + ## Merge cuttlefish generated *.args into the vm.args + TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp" + cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE" + echo "" >> "$TMP_ARG_FILE" + echo "-pa \"${REL_DIR}/consolidated\"" >> "$TMP_ARG_FILE" + sed '/^#/d' "$CUTTLE_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do + ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') + ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') + if [ "$ARG_KEY" = '' ]; then + ## for the flags, e.g. -heart -emu_args etc + ARG_KEY=$(echo "$ARG_LINE" | awk '{print $1}') + ARG_VALUE='' + TMP_ARG_KEY=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $1}') + if [ "$TMP_ARG_KEY" = '' ]; then + echo "$ARG_KEY" >> "$TMP_ARG_FILE" + fi + else + TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}') + if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then + if [ -n "$TMP_ARG_VALUE" ]; then + sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' \"$TMP_ARG_FILE\"" + else + echo "$ARG_LINE" >> "$TMP_ARG_FILE" fi fi - done - mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE" - fi + fi + done - # shellcheck disable=SC2086 - if ! relx_nodetool chkconfig $CONFIG_ARGS; then - echoerr "Error reading $CONFIG_ARGS" + if ! relx_nodetool chkconfig -config "$CONFIG_FILE"; then + echoerr "Error reading $CONFIG_FILE" exit 1 fi + + if [ "$check_only" = "check_only" ]; then + rm -f "$TMP_ARG_FILE" + rm -f "$CUTTLE_GEN_ARG_FILE" + rm -f "$CONFIG_FILE" + else + mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE" + fi +} + +# check if a PID is down +is_down() { + PID="$1" + if ps -p "$PID" >/dev/null; then + # still around + # shellcheck disable=SC2009 # this grep pattern is not a part of the progra names + if ps -p "$PID" | grep -q 'defunct'; then + # zombie state, print parent pid + parent="$(ps -o ppid= -p "$PID" | tr -d ' ')" + echo "WARN: $PID is marked , parent:" + ps -p "$parent" + return 0 + fi + return 1 + fi + # it's gone + return 0 +} + +wait_for() { + local WAIT_TIME + local CMD + WAIT_TIME="$1" + shift + CMD="$*" + while true; do + if $CMD >/dev/null 2>&1; then + return 0 + fi + if [ "$WAIT_TIME" -le 0 ]; then + return 1 + fi + WAIT_TIME=$((WAIT_TIME - 1)) + sleep 1 + done } # Call bootstrapd for daemon commands like start/stop/console @@ -305,14 +474,38 @@ bootstrapd() { fi } -# Use $CWD/etc/sys.config if exists -if [ -z "$RELX_CONFIG_PATH" ]; then - if [ -f "$RUNNER_ETC_DIR/sys.config" ]; then - RELX_CONFIG_PATH="-config $RUNNER_ETC_DIR/sys.config" - else - RELX_CONFIG_PATH="" +# check if a PID is down +is_down() { + PID="$1" + if ps -p "$PID" >/dev/null; then + # still around + # shellcheck disable=SC2009 # this grep pattern is not a part of the progra names + if ps -p "$PID" | grep -q 'defunct'; then + return 0 + fi + return 1 fi -fi + # it's gone + return 0 +} + +wait_for() { + local WAIT_TIME + local CMD + WAIT_TIME="$1" + shift + CMD="$*" + while true; do + if $CMD >/dev/null 2>&1; then + return 0 + fi + if [ "$WAIT_TIME" -le 0 ]; then + return 1 + fi + WAIT_TIME=$((WAIT_TIME - 1)) + sleep 1 + done +} IS_BOOT_COMMAND='no' case "$1" in @@ -325,8 +518,21 @@ case "$1" in foreground) IS_BOOT_COMMAND='yes' ;; + check_conf) + IS_BOOT_COMMAND='yes' + ;; esac +if [ "$IS_BOOT_COMMAND" = 'no' ]; then + # for non-boot commands, inspect vm.