From cba13a1fee97971dc5753dadb852ea659d8dc611 Mon Sep 17 00:00:00 2001 From: baoliang Date: Wed, 10 Apr 2019 16:21:58 +0800 Subject: [PATCH 01/18] fix bug, update action would change the user of definition. --- docs/zh_CN/后端部署文档.md | 431 +++---------------------------- install.sh | 2 +- 2 files changed, 38 insertions(+), 395 deletions(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index 4559e4b817..a07effbf49 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -86,370 +86,11 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL #Default requiretty ``` -## 配置文件说明 - -``` -说明：配置文件位于 target/escheduler-{version}/conf 下面 -``` - -### escheduler-alert - -配置邮件告警信息 - - -* alert.properties - -``` -#以qq邮箱为例，如果是别的邮箱，请更改对应配置 -#alert type is EMAIL/SMS -alert.type=EMAIL - -# mail server configuration -mail.protocol=SMTP -mail.server.host=smtp.exmail.qq.com -mail.server.port=25 -mail.sender=xxxxxxx@qq.com -mail.passwd=xxxxxxx - -# xls file path, need manually create it before use if not exist -xls.file.path=/opt/xls -``` - - - - -### escheduler-common - -通用配置文件配置，队列选择及地址配置，通用文件目录配置 - -- common/common.properties - -``` -#task queue implementation, default "zookeeper" -escheduler.queue.impl=zookeeper - -# user data directory path, self configuration, please make sure the directory exists and have read write permissions -data.basedir.path=/tmp/escheduler - -# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions -data.download.basedir.path=/tmp/escheduler/download - -# process execute directory. self configuration, please make sure the directory exists and have read write permissions -process.exec.basepath=/tmp/escheduler/exec - -# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/escheduler" is recommended -data.store2hdfs.basepath=/escheduler - -# whether hdfs starts -hdfs.startup.state=true - -# system env path. self configuration, please make sure the directory and file exists and have read write execute permissions -escheduler.env.path=/opt/.escheduler_env.sh -escheduler.env.py=/opt/escheduler_env.py - -#resource.view.suffixs -resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml - -# is development state? default "false" -development.state=false -``` - - - -SHELL任务 环境变量配置 - -``` -说明：配置文件位于 target/escheduler-{version}/conf/env 下面，这个会是Worker执行任务时加载的环境 -``` - -.escheduler_env.sh -``` -export HADOOP_HOME=/opt/soft/hadoop -export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop -export SPARK_HOME1=/opt/soft/spark1 -export SPARK_HOME2=/opt/soft/spark2 -export PYTHON_HOME=/opt/soft/python -export JAVA_HOME=/opt/soft/java -export HIVE_HOME=/opt/soft/hive - -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH -``` - - -​ - -Python任务 环境变量配置 - -``` -说明：配置文件位于 target/escheduler-{version}/conf/env 下面 -``` - -escheduler_env.py -``` -import os - -HADOOP_HOME="/opt/soft/hadoop" -SPARK_HOME1="/opt/soft/spark1" -SPARK_HOME2="/opt/soft/spark2" -PYTHON_HOME="/opt/soft/python" -JAVA_HOME="/opt/soft/java" -HIVE_HOME="/opt/soft/hive" -PATH=os.environ['PATH'] -PATH="%s/bin:%s/bin:%s/bin:%s/bin:%s/bin:%s/bin:%s"%(HIVE_HOME,HADOOP_HOME,SPARK_HOME1,SPARK_HOME2,JAVA_HOME,PYTHON_HOME,PATH) - -os.putenv('PATH','%s'%PATH) -``` - - - -hadoop 配置文件 - -- common/hadoop/hadoop.properties - -``` -# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory -fs.defaultFS=hdfs://mycluster:8020 - -#resourcemanager ha note this need ips , this empty if single -yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx - -# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine -yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s - -``` - - - -定时器配置文件 - -- quartz.properties - -``` -#============================================================================ -# Configure Main Scheduler Properties -#============================================================================ -org.quartz.scheduler.instanceName = EasyScheduler -org.quartz.scheduler.instanceId = AUTO -org.quartz.scheduler.makeSchedulerThreadDaemon = true -org.quartz.jobStore.useProperties = false - -#============================================================================ -# Configure ThreadPool -#============================================================================ - -org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool -org.quartz.threadPool.makeThreadsDaemons = true -org.quartz.threadPool.threadCount = 25 -org.quartz.threadPool.threadPriority = 5 - -#============================================================================ -# Configure JobStore -#============================================================================ - -org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX -org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate -org.quartz.jobStore.tablePrefix = QRTZ_ -org.quartz.jobStore.isClustered = true -org.quartz.jobStore.misfireThreshold = 60000 -org.quartz.jobStore.clusterCheckinInterval = 5000 -org.quartz.jobStore.dataSource = myDs - -#============================================================================ -# Configure Datasources -#============================================================================ - -org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver -org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/escheduler?characterEncoding=utf8&useSSL=false -org.quartz.dataSource.myDs.user = xx -org.quartz.dataSource.myDs.password = xx -org.quartz.dataSource.myDs.maxConnections = 10 -org.quartz.dataSource.myDs.validationQuery = select 1 -``` - - - -zookeeper 配置文件 - - -- zookeeper.properties - -``` -#zookeeper cluster -zookeeper.quorum=192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 - -#escheduler root directory -zookeeper.escheduler.root=/escheduler - -#zookeeper server dirctory -zookeeper.escheduler.dead.servers=/escheduler/dead-servers -zookeeper.escheduler.masters=/escheduler/masters -zookeeper.escheduler.workers=/escheduler/workers - -#zookeeper lock dirctory -zookeeper.escheduler.lock.masters=/escheduler/lock/masters -zookeeper.escheduler.lock.workers=/escheduler/lock/workers - -#escheduler failover directory -zookeeper.escheduler.lock.masters.failover=/escheduler/lock/failover/masters -zookeeper.escheduler.lock.workers.failover=/escheduler/lock/failover/workers - -#escheduler failover directory -zookeeper.session.timeout=300 -zookeeper.connection.timeout=300 -zookeeper.retry.sleep=1000 -zookeeper.retry.maxtime=5 - -``` - - - -### escheduler-dao - -dao数据源配置 - -- dao/data_source.properties - -``` -# base spring data source configuration -spring.datasource.type=com.alibaba.druid.pool.DruidDataSource -spring.datasource.driver-class-name=com.mysql.jdbc.Driver -spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/escheduler?characterEncoding=UTF-8 -spring.datasource.username=xx -spring.datasource.password=xx - -# connection configuration -spring.datasource.initialSize=5 -# min connection number -spring.datasource.minIdle=5 -# max connection number -spring.datasource.maxActive=50 - -# max wait time for get a connection in milliseconds. if configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. -# If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. -spring.datasource.maxWait=60000 - -# milliseconds for check to close free connections -spring.datasource.timeBetweenEvictionRunsMillis=60000 - -# the Destroy thread detects the connection interval and closes the physical connection in milliseconds if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. -spring.datasource.timeBetweenConnectErrorMillis=60000 - -# the longest time a connection remains idle without being evicted, in milliseconds -spring.datasource.minEvictableIdleTimeMillis=300000 - -#the SQL used to check whether the connection is valid requires a query statement. If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. -spring.datasource.validationQuery=SELECT 1 -#check whether the connection is valid for timeout, in seconds -spring.datasource.validationQueryTimeout=3 - -# when applying for a connection, if it is detected that the connection is idle longer than time Between Eviction Runs Millis, -# validation Query is performed to check whether the connection is valid -spring.datasource.testWhileIdle=true - -#execute validation to check if the connection is valid when applying for a connection -spring.datasource.testOnBorrow=true -#execute validation to check if the connection is valid when the connection is returned -spring.datasource.testOnReturn=false -spring.datasource.defaultAutoCommit=true -spring.datasource.keepAlive=true - -# open PSCache, specify count PSCache for every connection -spring.datasource.poolPreparedStatements=true -spring.datasource.maxPoolPreparedStatementPerConnectionSize=20 -``` - - - -### escheduler-server - -master配置文件 - -- master.properties - -``` -# master execute thread num -master.exec.threads=100 - -# master execute task number in parallel -master.exec.task.number=20 - -# master heartbeat interval -master.heartbeat.interval=10 - -# master commit task retry times -master.task.commit.retryTimes=5 - -# master commit task interval -master.task.commit.interval=100 - - -# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 -master.max.cpuload.avg=10 - -# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. -master.reserved.memory=1 -``` - - - -worker配置文件 - -- worker.properties - -``` -# worker execute thread num -worker.exec.threads=100 - -# worker heartbeat interval -worker.heartbeat.interval=10 - -# submit the number of tasks at a time -worker.fetch.task.num = 10 - - -# only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 -worker.max.cpuload.avg=10 - -# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G. -worker.reserved.memory=1 -``` - - - -### escheduler-api - -web配置文件 - -- application.properties - -``` -# server port -server.port=12345 - -# session config -server.session.timeout=7200 - -server.context-path=/escheduler/ - -# file size limit for upload -spring.http.multipart.max-file-size=1024MB -spring.http.multipart.max-request-size=1024MB - -# post content -server.max-http-post-size=5000000 -``` - - - ## 伪分布式部署 -### 1，创建部署用户 - -​ 如上 **创建部署用户** - ### 2，根据实际需求来创建HDFS根路径 -​ 根据 **common/common.properties** 中 **hdfs.startup.state** 的配置来判断是否启动HDFS，如果启动，则需要创建HDFS根路径，并将 **owner** 修改为**部署用户**，否则忽略此步骤 +​ 根据 **common/common.properties** 中 **hdf.startup.states** 的配置来判断是否启动HDFS，如果启动，则需要创建HDFS根路径，并将 **owner** 修改为**部署用户**，否则忽略此步骤 ### 3，项目编译 @@ -465,40 +106,6 @@ server.max-http-post-size=5000000 - 将**.escheduler_env.sh** 和 **escheduler_env.py** 两个环境变量文件复制到 **common/common.properties**配置的**escheduler.env.path** 和 **escheduler.env.py** 的目录下，并将 **owner** 修改为**部署用户** -### 6，启停服务 - -* 启停Master - -```启动master -sh ./bin/escheduler-daemon.sh start master-server -sh ./bin/escheduler-daemon.sh stop master-server -``` - -* 启停Worker - -``` -sh ./bin/escheduler-daemon.sh start worker-server -sh ./bin/escheduler-daemon.sh stop worker-server -``` - -* 启停Api - -``` -sh ./bin/escheduler-daemon.sh start api-server -sh ./bin/escheduler-daemon.sh stop api-server -``` -* 启停Logger - -``` -sh ./bin/escheduler-daemon.sh start logger-server -sh ./bin/escheduler-daemon.sh stop logger-server -``` -* 启停Alert - -``` -sh ./bin/escheduler-daemon.sh start alert-server -sh ./bin/escheduler-daemon.sh stop alert-server -``` @@ -546,6 +153,42 @@ sh ./bin/escheduler-daemon.sh stop alert-server - 注意：scp_hosts.sh 里 `tar -zxvf $workDir/../escheduler-1.0.0.tar.gz -C $installPath` 中的版本号(1.0.0)需要执行前手动替换成对应的版本号 + +### 7，启停服务 + +* 启停Master + +```启动master +sh ./bin/escheduler-daemon.sh start master-server +sh ./bin/escheduler-daemon.sh stop master-server +``` + +* 启停Worker + +``` +sh ./bin/escheduler-daemon.sh start worker-server +sh ./bin/escheduler-daemon.sh stop worker-server +``` + +* 启停Api + +``` +sh ./bin/escheduler-daemon.sh start api-server +sh ./bin/escheduler-daemon.sh stop api-server +``` +* 启停Logger + +``` +sh ./bin/escheduler-daemon.sh start logger-server +sh ./bin/escheduler-daemon.sh stop logger-server +``` +* 启停Alert + +``` +sh ./bin/escheduler-daemon.sh start alert-server +sh ./bin/escheduler-daemon.sh stop alert-server +``` + ## 服务监控 monitor_server.py 脚本是监听，master和worker服务挂掉重启的脚本 diff --git a/install.sh b/install.sh index 81772729de..515dfe20ea 100644 --- a/install.sh +++ b/install.sh @@ -211,7 +211,7 @@ mailPassword="xxxxxxxxxx" xlsFilePath="/opt/xls" # conf/config/install_config.conf配置 -# 安装路径 +# 安装路径,不要当前路径(pwd)一样 installPath="/data1_1T/escheduler" # 部署用户 From c075d26affa5add464f1ee7a2c8b43ca3f48d699 Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 11 Apr 2019 11:17:40 +0800 Subject: [PATCH 02/18] fix bug, update action would change the user of definition. --- docs/zh_CN/后端部署文档.md | 67 +++++++++++--------------------- 1 file changed, 22 insertions(+), 45 deletions(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index a07effbf49..fa1259d6de 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -74,7 +74,7 @@ mysql -h {host} -u {user} -p{password} -D {db} < quartz.sql ## 创建部署用户 -因为escheduler worker都是以 sudo -u {linux-user} 方式来执行作业，所以部署用户需要有 sudo 权限，而且是免密的。 +- 在所有需要部署调度的机器上创建部署用户，因为worker是以 sudo -u {linux-user} 方式来执行作业，所以部署用户需要有 sudo 权限，而且是免密的。 ```部署账号 vi /etc/sudoers @@ -86,61 +86,30 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL #Default requiretty ``` -## 伪分布式部署 +## ssh免密配置 + 在部署机和其他安装机器上配置ssh免密登录，如果要在部署机上安装调度，需要配置本机免密登录自己 + +- [将 **主机器** 和各个其它机器SSH打通](http://geek.analysys.cn/topic/113) -### 2，根据实际需求来创建HDFS根路径 +## 产品部署 -​ 根据 **common/common.properties** 中 **hdf.startup.states** 的配置来判断是否启动HDFS，如果启动，则需要创建HDFS根路径，并将 **owner** 修改为**部署用户**，否则忽略此步骤 +### 1. 修改环境变量文件 -### 3，项目编译 +- 根据业务需求，修改conf/env/目录下的**escheduler_env.py**，**.escheduler_env.sh**两个文件中的环境变量 -​ 如上进行 **项目编译** +### 2. 修改部署参数 -### 4，修改配置文件 +​- 修改 install.sh 中变量的值，替换成自身业务所需的值 -​ 根据 **配置文件说明** 修改配置文件和 **环境变量** 文件 - -### 5，创建目录并将环境变量文件复制到指定目录 - -- 创建 **common/common.properties** 下的data.basedir.path、data.download.basedir.path和process.exec.basepath路径 - -- 将**.escheduler_env.sh** 和 **escheduler_env.py** 两个环境变量文件复制到 **common/common.properties**配置的**escheduler.env.path** 和 **escheduler.env.py** 的目录下，并将 **owner** 修改为**部署用户** - - - - -## 分布式部署 - -### 1，创建部署用户 - -- 在需要部署调度的机器上如上 **创建部署用户** -- [将 **主机器** 和各个其它机器SSH打通](https://blog.csdn.net/thinkmore1314/article/details/22489203) - -### 2，根据实际需求来创建HDFS根路径 - -​ 根据 **common/common.properties** 中 **hdfs.startup.state** 的配置来判断是否启动HDFS，如果启动，则需要创建HDFS根路径，并将 **owner** 修改为**部署用户**，否则忽略此步骤 - -### 3，项目编译 - -​ 如上进行 **项目编译** - -### 4，将环境变量文件复制到指定目录 - -​ 将**.escheduler_env.sh** 和 **escheduler_env.py** 两个环境变量文件复制到 **common/common.properties**配置的**escheduler.env.path** 和 **escheduler.env.py** 的目录下，并将 **owner** 修改为**部署用户** - -### 5，修改 install.sh - -​ 修改 install.sh 中变量的值，替换成自身业务所需的值 - -### 6，一键部署 +### 3. 一键部署 - 安装 pip install kazoo + - 安装目录如下： ``` bin conf - escheduler-1.0.0-SNAPSHOT.tar.gz install.sh lib monitor_server.py @@ -151,10 +120,18 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL - 使用部署用户 sh install.sh 一键部署 - - 注意：scp_hosts.sh 里 `tar -zxvf $workDir/../escheduler-1.0.0.tar.gz -C $installPath` 中的版本号(1.0.0)需要执行前手动替换成对应的版本号 +- jps查看服务是否启动 + +```aidl + MasterServer ----- master服务 + WorkerServer ----- worker服务 + LoggerServer ----- logger服务 + ApiApplicationServer ----- api服务 + AlertServer ----- alert服务 +``` -### 7，启停服务 +## 启停服务 * 启停Master From f4640e87b92b20a42e861e2c9b5499a945ba7182 Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 11 Apr 2019 11:20:21 +0800 Subject: [PATCH 03/18] update install document --- docs/zh_CN/后端部署文档.md | 2 +- install.sh | 115 ++++++++++++++----------------- 2 files changed, 54 insertions(+), 63 deletions(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index fa1259d6de..c380d96bb1 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -99,7 +99,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ### 2. 修改部署参数 -​- 修改 install.sh 中变量的值，替换成自身业务所需的值 +​- 修改 **install.sh**中的参数，替换成自身业务所需的值 ### 3. 一键部署 diff --git a/install.sh b/install.sh index 515dfe20ea..c6c734078f 100644 --- a/install.sh +++ b/install.sh @@ -47,8 +47,57 @@ mysqlUserName="xx" # mysql 密码 mysqlPassword="xx" +# conf/config/install_config.conf配置 +# 安装路径,不要当前路径(pwd)一样 +installPath="/data1_1T/escheduler" + +# 部署用户 +deployUser="escheduler" + +# zk集群 +zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" + +# 安装hosts +ips="ark0,ark1,ark2,ark3,ark4" + +# conf/config/run_config.conf配置 +# 运行Master的机器 +masters="ark0,ark1" + +# 运行Worker的机器 +workers="ark2,ark3,ark4" + +# 运行Alert的机器 +alertServer="ark3" + +# 运行Api的机器 +apiServers="ark1" + +# alert配置 +# 邮件协议 +mailProtocol="SMTP" + +# 邮件服务host +mailServerHost="smtp.exmail.qq.com" + +# 邮件服务端口 +mailServerPort="25" + +# 发送人 +mailSender="xxxxxxxxxx" + +# 发送人密码 +mailPassword="xxxxxxxxxx" + +# 下载Excel路径 +xlsFilePath="/tmp/xls" + # hadoop 配置 +# 是否启动hdfs,如果启动则为true,需要配置以下hadoop相关参数; +# 不启动设置为false,如果为false,以下配置不需要修改 +hdfsStartupSate="false" + # namenode地址,支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下 namenodeFs="hdfs://mycluster:8020" @@ -58,6 +107,8 @@ yarnHaIps="192.168.xx.xx,192.168.xx.xx" # 如果是单 resourcemanager,只需要配置一个主机名称,如果是resourcemanager HA,则默认配置就好 singleYarnIp="ark1" +# hdfs根路径,根路径的owner必须是部署用户 +hdfsPath="/escheduler" # common 配置 # 程序路径 @@ -69,17 +120,11 @@ downloadPath="/tmp/escheduler/download" # 任务执行路径 execPath="/tmp/escheduler/exec" -# hdfs根路径 -hdfsPath="/escheduler" - -# 是否启动hdfs,如果启动则为true,不启动设置为false -hdfsStartupSate="true" - # SHELL环境变量路径 -shellEnvPath="/opt/.escheduler_env.sh" +shellEnvPath="$installPath/conf/env/.escheduler_env.sh" # Python换将变量路径 -pythonEnvPath="/opt/escheduler_env.py" +pythonEnvPath="$installPath/conf/env/escheduler_env.py" # 资源文件的后缀 resSuffixs="txt,log,sh,conf,cfg,py,java,sql,hql,xml" @@ -87,11 +132,7 @@ resSuffixs="txt,log,sh,conf,cfg,py,java,sql,hql,xml" # 开发状态,如果是true,对于SHELL脚本可以在execPath目录下查看封装后的SHELL脚本,如果是false则执行完成直接删除 devState="true" - # zk 配置 -# zk集群 -zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" - # zk根目录 zkRoot="/escheduler" @@ -168,7 +209,6 @@ workerMaxCupLoadAvg="10" # worker预留内存,用来判断master是否还有执行能力 workerReservedMemory="1" - # api 配置 # api 服务端口 apiServerPort="12345" @@ -188,53 +228,6 @@ springMaxRequestSize="1024MB" # api 最大post请求大小 apiMaxHttpPostSize="5000000" - - -# alert配置 - -# 邮件协议 -mailProtocol="SMTP" - -# 邮件服务host -mailServerHost="smtp.exmail.qq.com" - -# 邮件服务端口 -mailServerPort="25" - -# 发送人 -mailSender="xxxxxxxxxx" - -# 发送人密码 -mailPassword="xxxxxxxxxx" - -# 下载Excel路径 -xlsFilePath="/opt/xls" - -# conf/config/install_config.conf配置 -# 安装路径,不要当前路径(pwd)一样 -installPath="/data1_1T/escheduler" - -# 部署用户 -deployUser="escheduler" - -# 安装hosts -ips="ark0,ark1,ark2,ark3,ark4" - - -# conf/config/run_config.conf配置 -# 运行Master的机器 -masters="ark0,ark1" - -# 运行Worker的机器 -workers="ark2,ark3,ark4" - -# 运行Alert的机器 -alertServer="ark3" - -# 运行Api的机器 -apiServers="ark1" - - # 1,替换文件 echo "1,替换文件" sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${mysqlHost}/${mysqlDb}?characterEncoding=UTF-8#g" conf/dao/data_source.properties @@ -317,8 +310,6 @@ sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/run_con sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/run_config.conf - - # 2,创建目录 echo "2,创建目录" From 8641c8b10bff0d587451c93340b0a2986d0b215b Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 11 Apr 2019 11:22:04 +0800 Subject: [PATCH 04/18] fix bug, update action would change the user of definition. --- docs/zh_CN/后端部署文档.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index c380d96bb1..85d3239c69 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -103,7 +103,9 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ### 3. 一键部署 -- 安装 pip install kazoo +- 安装zookeeper工具 + + `pip install kazoo` - 安装目录如下： From cbcffb33df85338854811d1f422b6df3dd4a54f0 Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 11 Apr 2019 11:23:43 +0800 Subject: [PATCH 05/18] fix bug, update action would change the user of definition. --- docs/zh_CN/后端部署文档.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index 85d3239c69..38ccbb17eb 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -103,10 +103,6 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ### 3. 一键部署 -- 安装zookeeper工具 - - `pip install kazoo` - - 安装目录如下： ``` @@ -120,7 +116,11 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ``` -- 使用部署用户 sh install.sh 一键部署 +- 安装zookeeper工具 + + `pip install kazoo` + +- 使用部署用户 `sh install.sh` 一键部署 - jps查看服务是否启动 From 7e65dda7dbffee54b4513676576cde97e8d06429 Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 11 Apr 2019 11:55:12 +0800 Subject: [PATCH 06/18] fix bug, update action would change the user of definition. --- docs/zh_CN/后端部署文档.md | 37 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index 38ccbb17eb..b03cac9462 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -93,15 +93,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ## 产品部署 -### 1. 修改环境变量文件 - -- 根据业务需求，修改conf/env/目录下的**escheduler_env.py**，**.escheduler_env.sh**两个文件中的环境变量 - -### 2. 修改部署参数 - -​- 修改 **install.sh**中的参数，替换成自身业务所需的值 - -### 3. 一键部署 +### 1. 切换到部署用户，修改安装目录权限 - 安装目录如下： @@ -115,12 +107,27 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL sql ``` +- 修改权限 + + `sudo chown -R escheduler:escheduler *` + +### 2. 修改环境变量文件 + +- 根据业务需求，修改conf/env/目录下的**escheduler_env.py**，**.escheduler_env.sh**两个文件中的环境变量 + +### 3. 修改部署参数 + +​- 修改 **install.sh**中的参数，替换成自身业务所需的值 + +### 4. 一键部署 - 安装zookeeper工具 `pip install kazoo` -- 使用部署用户 `sh install.sh` 一键部署 +- 一键部署 + + `sh install.sh` - jps查看服务是否启动 @@ -167,14 +174,6 @@ sh ./bin/escheduler-daemon.sh stop logger-server sh ./bin/escheduler-daemon.sh start alert-server sh ./bin/escheduler-daemon.sh stop alert-server ``` - -## 服务监控 - -monitor_server.py 脚本是监听，master和worker服务挂掉重启的脚本 - -注意：在全部服务都启动之后启动 - -nohup python -u monitor_server.py > nohup.out 2>&1 & ## 日志查看 日志统一存放于指定文件夹内 @@ -186,4 +185,4 @@ nohup python -u monitor_server.py > nohup.out 2>&1 & |—— escheduler-worker-server.log |—— escheduler-api-server.log |—— escheduler-logger-server.log -``` \ No newline at end of file +``` From 0dc1f45a16e8fe52008cb165e27e241d980b03ac Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 11 Apr 2019 13:39:21 +0800 Subject: [PATCH 07/18] delete tar update install document --- docs/zh_CN/后端部署文档.md | 6 +++--- script/scp_hosts.sh | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index b03cac9462..306afb9ab5 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -93,7 +93,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ## 产品部署 -### 1. 切换到部署用户，修改安装目录权限 +### 1. 修改安装目录权限 - 安装目录如下： @@ -109,7 +109,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ``` - 修改权限 - `sudo chown -R escheduler:escheduler *` + `chown -R deployUser:deployUser *` ### 2. 修改环境变量文件 @@ -125,7 +125,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL `pip install kazoo` -- 一键部署 +- 切换到部署用户，一键部署 `sh install.sh` diff --git a/script/scp_hosts.sh b/script/scp_hosts.sh index 3884a72d80..74db0c14d9 100755 --- a/script/scp_hosts.sh +++ b/script/scp_hosts.sh @@ -5,8 +5,6 @@ workDir=`cd ${workDir};pwd` source $workDir/../conf/config/run_config.conf source $workDir/../conf/config/install_config.conf -tar -zxvf $workDir/../EasyScheduler-1.0.0.tar.gz -C $installPath - hostsArr=(${ips//,/ }) for host in ${hostsArr[@]} do From 170a625ded4f881ca736821679cc1216c0488ca0 Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 11 Apr 2019 14:42:38 +0800 Subject: [PATCH 08/18] update install document --- docs/zh_CN/后端部署文档.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index 306afb9ab5..1e210783bf 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -109,7 +109,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ``` - 修改权限 - `chown -R deployUser:deployUser *` + `sudo chown -R deployUser:deployUser *` ### 2. 修改环境变量文件 @@ -119,6 +119,8 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ​- 修改 **install.sh**中的参数，替换成自身业务所需的值 +- 如果使用hdfs相关功能，需要拷贝 + ### 4. 一键部署 - 安装zookeeper工具 @@ -138,7 +140,18 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ApiApplicationServer ----- api服务 AlertServer ----- alert服务 ``` - + +## 日志查看 +日志统一存放于指定文件夹内 + +```日志路径 + logs/ + ├── escheduler-alert-server.log + ├── escheduler-master-server.log + |—— escheduler-worker-server.log + |—— escheduler-api-server.log + |—— escheduler-logger-server.log +``` ## 启停服务 @@ -175,14 +188,3 @@ sh ./bin/escheduler-daemon.sh start alert-server sh ./bin/escheduler-daemon.sh stop alert-server ``` -## 日志查看 -日志统一存放于指定文件夹内 - -```日志路径 - logs/ - ├── escheduler-alert-server.log - ├── escheduler-master-server.log - |—— escheduler-worker-server.log - |—— escheduler-api-server.log - |—— escheduler-logger-server.log -``` From 36eca53c124e17e6c040bcfc779a5eaa3cede176 Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 11 Apr 2019 14:46:45 +0800 Subject: [PATCH 09/18] update install document --- docs/zh_CN/后端部署文档.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index 1e210783bf..b6cea5988e 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -117,9 +117,9 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ### 3. 修改部署参数 -​- 修改 **install.sh**中的参数，替换成自身业务所需的值 +​* 修改 **install.sh**中的参数，替换成自身业务所需的值 -- 如果使用hdfs相关功能，需要拷贝 +* 如果使用hdfs相关功能，需要拷贝**hdfs-site.xml**和**core-site.xml**到conf目录下 ### 4. 一键部署 From d654c29c4d88f26a49d1aa4b8504aacbf2dda0b2 Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 11 Apr 2019 14:48:29 +0800 Subject: [PATCH 10/18] update install document --- docs/zh_CN/后端部署文档.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index b6cea5988e..165f4c2fb5 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -117,9 +117,10 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ### 3. 修改部署参数 -​* 修改 **install.sh**中的参数，替换成自身业务所需的值 -* 如果使用hdfs相关功能，需要拷贝**hdfs-site.xml**和**core-site.xml**到conf目录下 + - 修改 **install.sh**中的参数，替换成自身业务所需的值 + + - 如果使用hdfs相关功能，需要拷贝**hdfs-site.xml**和**core-site.xml**到conf目录下 ### 4. 一键部署 From 5037bb631c30639772392a702fd0740cd606669f Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 11 Apr 2019 15:04:44 +0800 Subject: [PATCH 11/18] update install document --- docs/zh_CN/后端部署文档.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index 165f4c2fb5..266ea0c2e2 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -27,15 +27,6 @@ 正常编译完后，会在当前目录生成 target/escheduler-{version}/ -``` - bin - conf - lib - script - sql - install.sh -``` - - 说明 ``` @@ -111,7 +102,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL `sudo chown -R deployUser:deployUser *` -### 2. 修改环境变量文件 +### 2. 修改环境变量文件core-site.xml - 根据业务需求，修改conf/env/目录下的**escheduler_env.py**，**.escheduler_env.sh**两个文件中的环境变量 From 2d49c5098a50224765197627120fca63cf2732b6 Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 11 Apr 2019 15:24:50 +0800 Subject: [PATCH 12/18] fix bug, update action would change the user of definition. --- docs/zh_CN/后端部署文档.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index 266ea0c2e2..db4f85fa29 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -82,7 +82,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL - [将 **主机器** 和各个其它机器SSH打通](http://geek.analysys.cn/topic/113) -## 产品部署 +## 部署 ### 1. 修改安装目录权限 From f6ac53b85d601999656603a65198438b3c298324 Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 11 Apr 2019 15:45:50 +0800 Subject: [PATCH 13/18] update install document --- docs/zh_CN/后端部署文档.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index db4f85fa29..68412f7eab 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -93,7 +93,6 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL conf install.sh lib - monitor_server.py script sql @@ -102,7 +101,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL `sudo chown -R deployUser:deployUser *` -### 2. 修改环境变量文件core-site.xml +### 2. 修改环境变量文件 - 根据业务需求，修改conf/env/目录下的**escheduler_env.py**，**.escheduler_env.sh**两个文件中的环境变量 From 2f9505474b2bfd032fa4931bb12c4ff040ea4ded Mon Sep 17 00:00:00 2001 From: baoliang Date: Fri, 12 Apr 2019 10:07:27 +0800 Subject: [PATCH 14/18] update install document --- docs/zh_CN/前端部署文档.md | 154 ++----------------------------- 1 file changed, 6 insertions(+), 148 deletions(-) diff --git a/docs/zh_CN/前端部署文档.md b/docs/zh_CN/前端部署文档.md index af703ae7ba..5ca9c360e3 100644 --- a/docs/zh_CN/前端部署文档.md +++ b/docs/zh_CN/前端部署文档.md @@ -1,20 +1,11 @@ # 前端部署文档 -- ##### 1. 开发环境搭建 - -- ##### 2. 自动化部署 - -- ##### 3. 手动部署 - -- ##### 4. Liunx下使用node启动并且守护进程 - - ### 1.开发环境搭建 -- #### node安装 +- #### 安装node Node包下载 (注意版本 8.9.4) `https://nodejs.org/download/release/v8.9.4/` -- #### 前端项目构建 +- #### 构建项目 用命令行模式 `cd` 进入 `escheduler-ui`项目目录并执行 `npm install` 拉取项目依赖包 > 如果 `npm install` 速度非常慢 @@ -23,8 +14,6 @@ Node包下载 (注意版本 8.9.4) `https://nodejs.org/download/release/v8.9.4/` > 运行 `cnpm install` - - > ##### ！！！这里特别注意 项目如果在拉取依赖包的过程中报 " node-sass error " 错误，请在执行完后再次执行以下命令 ``` npm install node-sass --unsafe-perm //单独安装node-sass依赖 @@ -44,131 +33,15 @@ API_BASE = http://192.168.220.204:12345 - `npm run build` 项目打包 (打包后根目录会创建一个名为dist文件夹，用于发布线上Nginx) - -### 2.自动化部署 +### 2.自动部署方式 前端自动部署基于`yum`操作，部署之前请先安装更新`yum` -在项目`escheduler-ui`根目录执行`./install(线上环境).sh` - -``` -[root@localhost escheduler-ui]# ./install.sh -escheduler-ui-install.sh -========================================================================配置信息======================================================================= -前端访问端口：8888 -后端代理接口地址：http://192.168.220.154:12345 -静态文件地址：/data/EasyScheduler/escheduler-ui/dist -当前路径：/data/EasyScheduler/escheduler-ui -本机ip：192.168.60.94 -========================================================================配置信息======================================================================= - -获取http://nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpm -警告：/var/tmp/rpm-tmp.pgG0MW: 头V4 RSA/SHA1 Signature, 密钥 ID 7bd9bf62: NOKEY -准备中... ################################# [100%] -正在升级/安装... - 1:nginx-release-centos-7-0.el7.ngx ################################# [100%] -已加载插件：fastestmirror -Loading mirror speeds from cached hostfile - * base: mirrors.zju.edu.cn - * extras: mirrors.tuna.tsinghua.edu.cn - * updates: mirrors.tuna.tsinghua.edu.cn -正在解决依赖关系 ---> 正在检查事务 ----> 软件包 nginx.x86_64.1.1.14.2-1.el7_4.ngx 将被 安装 ---> 解决依赖关系完成 - -依赖关系解决 - -======================================================================================================================================================================================================== - Package 架构 版本 源 大小 -======================================================================================================================================================================================================== -正在安装: - nginx x86_64 1:1.14.2-1.el7_4.ngx nginx 754 k - -事务概要 -======================================================================================================================================================================================================== -安装 1 软件包 - -总下载量：754 k -安装大小：2.6 M -Downloading packages: -nginx-1.14.2-1.el7_4.ngx.x86_64.rpm | 754 kB 00:00:18 -Running transaction check -Running transaction test -Transaction test succeeded -Running transaction -警告：RPM 数据库已被非 yum 程序修改。 - 正在安装 : 1:nginx-1.14.2-1.el7_4.ngx.x86_64 1/1 ----------------------------------------------------------------------- - -Thanks for using nginx! - -Please find the official documentation for nginx here: -* http://nginx.org/en/docs/ - -Please subscribe to nginx-announce mailing list to get -the most important news about nginx: -* http://nginx.org/en/support.html - -Commercial subscriptions for nginx are available on: -* http://nginx.com/products/ - ----------------------------------------------------------------------- - 验证中 : 1:nginx-1.14.2-1.el7_4.ngx.x86_64 1/1 - -已安装: - nginx.x86_64 1:1.14.2-1.el7_4.ngx - -完毕！ -nginx 安装成功 -已加载插件：fastestmirror -Loading mirror speeds from cached hostfile - * base: mirrors.zju.edu.cn - * extras: mirrors.tuna.tsinghua.edu.cn - * updates: mirrors.tuna.tsinghua.edu.cn -正在解决依赖关系 ---> 正在检查事务 ----> 软件包 policycoreutils-python.x86_64.0.2.5-29.el7_6.1 将被 安装 ---> 解决依赖关系完成 - -依赖关系解决 - -======================================================================================================================================================================================================== - Package 架构 版本 源 大小 -======================================================================================================================================================================================================== -正在安装: - policycoreutils-python x86_64 2.5-29.el7_6.1 updates 456 k - -事务概要 -======================================================================================================================================================================================================== -安装 1 软件包 - -总下载量：456 k -安装大小：1.2 M -Downloading packages: -policycoreutils-python-2.5-29.el7_6.1.x86_64.rpm | 456 kB 00:00:11 -Running transaction check -Running transaction test -Transaction test succeeded -Running transaction - 正在安装 : policycoreutils-python-2.5-29.el7_6.1.x86_64 1/1 - 验证中 : policycoreutils-python-2.5-29.el7_6.1.x86_64 1/1 - -已安装: - policycoreutils-python.x86_64 0:2.5-29.el7_6.1 - -完毕！ -ValueError: 已定义端口 tcp/8888 -success -success -请浏览器访问：http://192.168.60.94:8888 -[root@localhost escheduler-ui]# - -``` +在项目`escheduler-ui`根目录下，修改install.sh中的参数，执行`./install(线上环境).sh` -### 3.手动部署 +### 3.手动部署方式 安装epel源 `yum install epel-release -y` @@ -247,23 +120,8 @@ systemctl restart nginx 在项目`escheduler-ui`根目录执行 `pm2 start npm -- run dev` 启动项目 -``` +## FAQ -[root@localhost escheduler-ui]# pm2 start npm -- run dev -[PM2] Applying action restartProcessId on app [npm](ids: 0) -[PM2] [npm](0) ✓ -[PM2] Process successfully started -┌──────────┬────┬─────────┬──────┬──────┬────────┬─────────┬────────┬─────┬──────────┬──────┬──────────┐ -│ App name │ id │ version │ mode │ pid │ status │ restart │ uptime │ cpu │ mem │ user │ watching │ -├──────────┼────┼─────────┼──────┼──────┼────────┼─────────┼────────┼─────┼──────────┼──────┼──────────┤ -│ npm │ 0 │ N/A │ fork │ 6168 │ online │ 31 │ 0s │ 0% │ 5.6 MB │ root │ disabled │ -└──────────┴────┴─────────┴──────┴──────┴────────┴─────────┴────────┴─────┴──────────┴──────┴──────────┘ - Use `pm2 show ` to get more details about an app - -``` - - -## 问题 #### 1. 上传文件大小限制 编辑配置文件 `vi /etc/nginx/nginx.conf` ``` From 6c44ac3afa366e48d1d6dbbd33caf72abb3706a4 Mon Sep 17 00:00:00 2001 From: baoliang Date: Fri, 12 Apr 2019 10:13:04 +0800 Subject: [PATCH 15/18] update install document --- docs/zh_CN/后端部署文档.md | 9 ++++----- monitor_server.py => script/monitor_server.py | 0 2 files changed, 4 insertions(+), 5 deletions(-) rename monitor_server.py => script/monitor_server.py (100%) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index 68412f7eab..bba0b0f172 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -6,7 +6,7 @@ * [Mysql](https://blog.csdn.net/u011886447/article/details/79796802) (5.5+) : 必装 * [JDK](https://www.oracle.com/technetwork/java/javase/downloads/index.html) (1.8+) : 必装 * [ZooKeeper](https://www.jianshu.com/p/de90172ea680)(3.4.6) ：必装 - * [Hadoop](https://blog.csdn.net/Evankaka/article/details/51612437)(2.7.3) ：选装， 如果需要使用到资源上传功能，MapReduce任务提交则需要配置Hadoop(上传的资源文件目前保存在Hdfs上) + * [Hadoop](https://blog.csdn.net/Evankaka/article/details/51612437)(2.6+) ：选装， 如果需要使用到资源上传功能，MapReduce任务提交则需要配置Hadoop(上传的资源文件目前保存在Hdfs上) * [Hive](https://staroon.pro/2017/12/09/HiveInstall/)(1.2.1) : 选装，hive任务提交需要安装 * Spark(1.x,2.x) : 选装，Spark任务提交需要安装 * PostgreSQL(8.2.15+) : 选装，PostgreSQL PostgreSQL存储过程需要安装 @@ -65,7 +65,7 @@ mysql -h {host} -u {user} -p{password} -D {db} < quartz.sql ## 创建部署用户 -- 在所有需要部署调度的机器上创建部署用户，因为worker是以 sudo -u {linux-user} 方式来执行作业，所以部署用户需要有 sudo 权限，而且是免密的。 +- 在所有需要部署调度的机器上创建部署用户，因为worker服务是以 sudo -u {linux-user} 方式来执行作业，所以部署用户需要有 sudo 权限，而且是免密的。 ```部署账号 vi /etc/sudoers @@ -78,7 +78,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ``` ## ssh免密配置 - 在部署机和其他安装机器上配置ssh免密登录，如果要在部署机上安装调度，需要配置本机免密登录自己 + 在部署机器和其他安装机器上配置ssh免密登录，如果要在部署机上安装调度，需要配置本机免密登录自己 - [将 **主机器** 和各个其它机器SSH打通](http://geek.analysys.cn/topic/113) @@ -97,7 +97,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL sql ``` -- 修改权限 +- 修改权限(deployUser修改为对应部署用户) `sudo chown -R deployUser:deployUser *` @@ -107,7 +107,6 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL ### 3. 修改部署参数 - - 修改 **install.sh**中的参数，替换成自身业务所需的值 - 如果使用hdfs相关功能，需要拷贝**hdfs-site.xml**和**core-site.xml**到conf目录下 diff --git a/monitor_server.py b/script/monitor_server.py similarity index 100% rename from monitor_server.py rename to script/monitor_server.py From 636205d6ad0727016ecf4615ac990969c8e2ccb2 Mon Sep 17 00:00:00 2001 From: gongzijian Date: Fri, 12 Apr 2019 11:01:02 +0800 Subject: [PATCH 16/18] update queue --- .../home/pages/security/pages/queue/_source/createQueue.vue | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/escheduler-ui/src/js/conf/home/pages/security/pages/queue/_source/createQueue.vue b/escheduler-ui/src/js/conf/home/pages/security/pages/queue/_source/createQueue.vue index d3fb95e02c..4e66b1432e 100644 --- a/escheduler-ui/src/js/conf/home/pages/security/pages/queue/_source/createQueue.vue +++ b/escheduler-ui/src/js/conf/home/pages/security/pages/queue/_source/createQueue.vue @@ -65,6 +65,10 @@ // edit if (this.item) { param.id = this.item.id + if (this.item.queueName === this.queueName && this.item.queue === this.queue) { + this.$message.warning(`名称和队列值未做更改`) + return + } } this._verifyName(param).then(() => { this.$refs['popup'].spinnerLoading = true From 2dff02b86aaec0646c180580a438ccf943413cfe Mon Sep 17 00:00:00 2001 From: gongzijian Date: Fri, 12 Apr 2019 11:09:20 +0800 Subject: [PATCH 17/18] update queue --- docs/zh_CN/_book/前端部署文档.html | 33 +- docs/zh_CN/_book/后端部署文档.html | 405 +++--------------- .../pages/queue/_source/createQueue.vue | 44 +- 3 files changed, 102 insertions(+), 380 deletions(-) diff --git a/docs/zh_CN/_book/前端部署文档.html b/docs/zh_CN/_book/前端部署文档.html index 2101b6012b..26b990d9e9 100644 --- a/docs/zh_CN/_book/前端部署文档.html +++ b/docs/zh_CN/_book/前端部署文档.html @@ -430,22 +430,12 @@

前端部署文档

-
    -
  • 1. 开发环境搭建
    -
    • -
  • 2. 自动化部署
    -
    • -
  • 3. 手动部署
    -
    • -
  • 4. Liunx下使用node启动并且守护进程
    -
    • -

1.开发环境搭建

    -

  • node安装

    +

  • 安装node

    Node包下载 (注意版本 8.9.4) https://nodejs.org/download/release/v8.9.4/

    • -

  • 前端项目构建

    +

  • 构建项目

    用命令行模式 cd 进入 escheduler-ui项目目录并执行 npm install 拉取项目依赖包

@@ -469,6 +459,7 @@ API_BASE = http://192.168.220.204:12345

  • npm run build 项目打包 (打包后根目录会创建一个名为dist文件夹，用于发布线上Nginx)

    • +

    2.自动部署方式

    2.自动化部署`

    在项目escheduler-ui根目录编辑安装文件vi install(线上环境).sh

    更改前端访问端口和后端代理接口地址

    @@ -478,6 +469,8 @@ esc_proxy="8888" # 配置代理后端接口 esc_proxy_port="http://192.168.220.154:12345"

    前端自动部署基于yum操作，部署之前请先安装更新`yum

    +

    在项目escheduler-ui根目录下，修改install.sh中的参数，执行./install(线上环境).sh

    +

    3.手动部署方式

    在项目escheduler-ui根目录执行./install(线上环境).sh

    3.手动部署

    安装epel源 yum install epel-release -y

    @@ -560,12 +553,14 @@ esc_proxy_port="http://192.168.220.154:12345" │ npm │ 0 │ N/A │ fork │ 6168 │ online │ 31 │ 0s │ 0% │ 5.6 MB │ root │ disabled │ └──────────┴────┴─────────┴──────┴──────┴────────┴─────────┴────────┴─────┴──────────┴──────┴──────────┘ Use `pm2 show <id|name>` to get more details about an app -

    问题

    -

    1. 上传文件大小限制

    -

    编辑配置文件 vi /etc/nginx/nginx.conf

    -
    # 更改上传大小
-client_max_body_size 1024m
-
    +## FAQ + +#### 1. 上传文件大小限制 +编辑配置文件 `vi /etc/nginx/nginx.conf` +

    更改上传大小

    +

    client_max_body_size 1024m +```

    +
    @@ -604,7 +599,7 @@ client_max_body_size 1024m diff --git a/docs/zh_CN/_book/后端部署文档.html b/docs/zh_CN/_book/后端部署文档.html index ab7bf3765e..68cdcf9ca2 100644 --- a/docs/zh_CN/_book/后端部署文档.html +++ b/docs/zh_CN/_book/后端部署文档.html @@ -435,7 +435,7 @@
  • Mysql (5.5+) : 必装
  • JDK (1.8+) : 必装
  • ZooKeeper(3.4.6) ：必装
    • -
  • Hadoop(2.7.3) ：选装， 如果需要使用到资源上传功能，MapReduce任务提交则需要配置Hadoop(上传的资源文件目前保存在Hdfs上)
    • +
  • Hadoop(2.6+) ：选装， 如果需要使用到资源上传功能，MapReduce任务提交则需要配置Hadoop(上传的资源文件目前保存在Hdfs上)
  • Hive(1.2.1) : 选装，hive任务提交需要安装
  • Spark(1.x,2.x) : 选装，Spark任务提交需要安装
  • PostgreSQL(8.2.15+) : 选装，PostgreSQL PostgreSQL存储过程需要安装
    • @@ -450,13 +450,7 @@
  • 查看目录

    • 正常编译完后，会在当前目录生成 target/escheduler-{version}/

    -
        bin
-    conf
-    lib
-    script
-    sql
-    install.sh
-
      +
      • 说明
      bin : 基础服务启动脚本
@@ -483,7 +477,9 @@ mysql -h {host} -u {user} -p{password} -D {db} < escheduler.sql
 
 mysql -h {host} -u {user} -p{password} -D {db} < quartz.sql

      创建部署用户

      -

      因为escheduler worker都是以 sudo -u {linux-user} 方式来执行作业，所以部署用户需要有 sudo 权限，而且是免密的。

      +
        +
      • 在所有需要部署调度的机器上创建部署用户，因为worker服务是以 sudo -u {linux-user} 方式来执行作业，所以部署用户需要有 sudo 权限，而且是免密的。
        • +
      vi /etc/sudoers
 
 # 部署用户是 escheduler 账号
@@ -492,301 +488,65 @@ escheduler  ALL=(ALL)       NOPASSWD: NOPASSWD: ALL
 # 并且需要注释掉 Default requiretty 一行
 #Default requiretty
      -

      配置文件说明

      -
      说明：配置文件位于 target/escheduler-{version}/conf 下面
-

      escheduler-alert

      -

      配置邮件告警信息

      +

      ssh免密配置

      +

      在部署机器和其他安装机器上配置ssh免密登录，如果要在部署机上安装调度，需要配置本机免密登录自己

      -
      #以qq邮箱为例，如果是别的邮箱，请更改对应配置
-#alert type is EMAIL/SMS
-alert.type=EMAIL
-
-# mail server configuration
-mail.protocol=SMTP
-mail.server.host=smtp.exmail.qq.com
-mail.server.port=25
-mail.sender=xxxxxxx@qq.com
-mail.passwd=xxxxxxx
-
-# xls file path, need manually create it before use if not exist
-xls.file.path=/opt/xls
-

      escheduler-common

      -

      通用配置文件配置，队列选择及地址配置，通用文件目录配置

      +

      部署

      +

      1. 修改安装目录权限

        -
      • common/common.properties
        • +
      • 安装目录如下：
      -
      #task queue implementation, default "zookeeper"
-escheduler.queue.impl=zookeeper
-
-# user data directory path, self configuration, please make sure the directory exists and have read write permissions
-data.basedir.path=/tmp/escheduler
-
-# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
-data.download.basedir.path=/tmp/escheduler/download
-
-# process execute directory. self configuration, please make sure the directory exists and have read write permissions
-process.exec.basepath=/tmp/escheduler/exec
-
-# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/escheduler" is recommended
-data.store2hdfs.basepath=/escheduler
-
-# whether hdfs starts
-hdfs.startup.state=true
-
-# system env path. self configuration, please make sure the directory and file exists and have read write execute permissions
-escheduler.env.path=/opt/.escheduler_env.sh
-escheduler.env.py=/opt/escheduler_env.py
-
-#resource.view.suffixs
-resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml
-
-# is development state? default "false"
-development.state=false
-

      SHELL任务 环境变量配置

      -
      说明：配置文件位于 target/escheduler-{version}/conf/env 下面，这个会是Worker执行任务时加载的环境
-

      .escheduler_env.sh

      -
      export HADOOP_HOME=/opt/soft/hadoop
-export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
-export SPARK_HOME1=/opt/soft/spark1
-export SPARK_HOME2=/opt/soft/spark2
-export PYTHON_HOME=/opt/soft/python
-export JAVA_HOME=/opt/soft/java
-export HIVE_HOME=/opt/soft/hive
-
-export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH
-

      ​

      -

      Python任务 环境变量配置

      -
      说明：配置文件位于 target/escheduler-{version}/conf/env 下面
-

      escheduler_env.py

      -
      import os
-
-HADOOP_HOME="/opt/soft/hadoop"
-SPARK_HOME1="/opt/soft/spark1"
-SPARK_HOME2="/opt/soft/spark2"
-PYTHON_HOME="/opt/soft/python"
-JAVA_HOME="/opt/soft/java"
-HIVE_HOME="/opt/soft/hive"
-PATH=os.environ['PATH']
-PATH="%s/bin:%s/bin:%s/bin:%s/bin:%s/bin:%s/bin:%s"%(HIVE_HOME,HADOOP_HOME,SPARK_HOME1,SPARK_HOME2,JAVA_HOME,PYTHON_HOME,PATH)
-
-os.putenv('PATH','%s'%PATH)
-

      hadoop 配置文件

      -
        -
      • common/hadoop/hadoop.properties
        • -
      -
      # ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory
-fs.defaultFS=hdfs://mycluster:8020
-
-#resourcemanager ha note this need ips , this empty if single
-yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
-
-# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine
-yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s
-

      定时器配置文件

      -
        -
      • quartz.properties
        • -
      -
      #============================================================================
-# Configure Main Scheduler Properties
-#============================================================================
-org.quartz.scheduler.instanceName = EasyScheduler
-org.quartz.scheduler.instanceId = AUTO
-org.quartz.scheduler.makeSchedulerThreadDaemon = true
-org.quartz.jobStore.useProperties = false
-
-#============================================================================
-# Configure ThreadPool
-#============================================================================
-
-org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool
-org.quartz.threadPool.makeThreadsDaemons = true
-org.quartz.threadPool.threadCount = 25
-org.quartz.threadPool.threadPriority = 5
-
-#============================================================================
-# Configure JobStore
-#============================================================================
-
-org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX
-org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate
-org.quartz.jobStore.tablePrefix = QRTZ_
-org.quartz.jobStore.isClustered = true
-org.quartz.jobStore.misfireThreshold = 60000
-org.quartz.jobStore.clusterCheckinInterval = 5000
-org.quartz.jobStore.dataSource = myDs
-
-#============================================================================
-# Configure Datasources  
-#============================================================================
-
-org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver
-org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/escheduler?characterEncoding=utf8&useSSL=false
-org.quartz.dataSource.myDs.user = xx
-org.quartz.dataSource.myDs.password = xx
-org.quartz.dataSource.myDs.maxConnections = 10
-org.quartz.dataSource.myDs.validationQuery = select 1
-

      zookeeper 配置文件

      -
        -
      • zookeeper.properties
        • -
      -
      #zookeeper cluster
-zookeeper.quorum=192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
-
-#escheduler root directory
-zookeeper.escheduler.root=/escheduler
-
-#zookeeper server dirctory
-zookeeper.escheduler.dead.servers=/escheduler/dead-servers
-zookeeper.escheduler.masters=/escheduler/masters
-zookeeper.escheduler.workers=/escheduler/workers
-
-#zookeeper lock dirctory
-zookeeper.escheduler.lock.masters=/escheduler/lock/masters
-zookeeper.escheduler.lock.workers=/escheduler/lock/workers
-
-#escheduler failover directory
-zookeeper.escheduler.lock.masters.failover=/escheduler/lock/failover/masters
-zookeeper.escheduler.lock.workers.failover=/escheduler/lock/failover/workers
-
-#escheduler failover directory
-zookeeper.session.timeout=300
-zookeeper.connection.timeout=300
-zookeeper.retry.sleep=1000
-zookeeper.retry.maxtime=5
-

      escheduler-dao

      -

      dao数据源配置

      -
        -
      • dao/data_source.properties
        • -
      -
      # base spring data source configuration
-spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
-spring.datasource.driver-class-name=com.mysql.jdbc.Driver
-spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/escheduler?characterEncoding=UTF-8
-spring.datasource.username=xx
-spring.datasource.password=xx
-
-# connection configuration
-spring.datasource.initialSize=5
-# min connection number
-spring.datasource.minIdle=5
-# max connection number
-spring.datasource.maxActive=50
-
-# max wait time for get a connection in milliseconds. if configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases.
-# If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true.
-spring.datasource.maxWait=60000
-
-# milliseconds for check to close free connections
-spring.datasource.timeBetweenEvictionRunsMillis=60000
-
-# the Destroy thread detects the connection interval and closes the physical connection in milliseconds if the connection idle time is greater than or equal to minEvictableIdleTimeMillis.
-spring.datasource.timeBetweenConnectErrorMillis=60000
-
-# the longest time a connection remains idle without being evicted, in milliseconds
-spring.datasource.minEvictableIdleTimeMillis=300000
-
-#the SQL used to check whether the connection is valid requires a query statement. If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work.
-spring.datasource.validationQuery=SELECT 1
-#check whether the connection is valid for timeout, in seconds
-spring.datasource.validationQueryTimeout=3
-
-# when applying for a connection, if it is detected that the connection is idle longer than time Between Eviction Runs Millis,
-# validation Query is performed to check whether the connection is valid
-spring.datasource.testWhileIdle=true
-
-#execute validation to check if the connection is valid when applying for a connection
-spring.datasource.testOnBorrow=true
-#execute validation to check if the connection is valid when the connection is returned
-spring.datasource.testOnReturn=false
-spring.datasource.defaultAutoCommit=true
-spring.datasource.keepAlive=true
-
-# open PSCache, specify count PSCache for every connection
-spring.datasource.poolPreparedStatements=true
-spring.datasource.maxPoolPreparedStatementPerConnectionSize=20
-

      escheduler-server

      -

      master配置文件

      -
        -
      • master.properties
        • -
      -
      # master execute thread num
-master.exec.threads=100
-
-# master execute task number in parallel
-master.exec.task.number=20
-
-# master heartbeat interval
-master.heartbeat.interval=10
-
-# master commit task retry times
-master.task.commit.retryTimes=5
-
-# master commit task interval
-master.task.commit.interval=100
-
-
-# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2
-master.max.cpuload.avg=10
-
-# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.
-master.reserved.memory=1
-

      worker配置文件

      -
        -
      • worker.properties
        • -
      -
      # worker execute thread num
-worker.exec.threads=100
-
-# worker heartbeat interval
-worker.heartbeat.interval=10
-
-# submit the number of tasks at a time
-worker.fetch.task.num = 10
-
-
-# only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2
-worker.max.cpuload.avg=10
-
-# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G.
-worker.reserved.memory=1
-

      escheduler-api

      -

      web配置文件

      -
        -
      • application.properties
        • -
      -
      # server port
-server.port=12345
-
-# session config
-server.session.timeout=7200
-
-server.context-path=/escheduler/
-
-# file size limit for upload
-spring.http.multipart.max-file-size=1024MB
-spring.http.multipart.max-request-size=1024MB
-
-# post content
-server.max-http-post-size=5000000
-

      伪分布式部署

      -

      1，创建部署用户

      -

      ​ 如上 创建部署用户

      -

      2，根据实际需求来创建HDFS根路径

      -

      ​ 根据 common/common.properties 中 hdfs.startup.state 的配置来判断是否启动HDFS，如果启动，则需要创建HDFS根路径，并将 owner 修改为部署用户，否则忽略此步骤

      -

      3，项目编译

      -

      ​ 如上进行 项目编译

      -

      4，修改配置文件

      -

      ​ 根据 配置文件说明 修改配置文件和 环境变量 文件

      -

      5，创建目录并将环境变量文件复制到指定目录

      -
        -

      • 创建 common/common.properties 下的data.basedir.path、data.download.basedir.path和process.exec.basepath路径

        -
        • -

      • 将.escheduler_env.sh 和 escheduler_env.py 两个环境变量文件复制到 common/common.properties配置的escheduler.env.path 和 escheduler.env.py 的目录下，并将 owner 修改为部署用户

        +
            bin
+    conf
+    install.sh
+    lib
+    script
+    sql
+
          +

        • 修改权限(deployUser修改为对应部署用户)

          +

          sudo chown -R deployUser:deployUser *

        -

        6，启停服务

        +

        2. 修改环境变量文件

        +
          +
        • 根据业务需求，修改conf/env/目录下的escheduler_env.py，.escheduler_env.sh两个文件中的环境变量
          • +
        +

        3. 修改部署参数

        +
          +

        • 修改 install.sh中的参数，替换成自身业务所需的值

          +
          • +

        • 如果使用hdfs相关功能，需要拷贝hdfs-site.xml和core-site.xml到conf目录下

          +
          • +
        +

        4. 一键部署

        +
          +

        • 安装zookeeper工具

          +

          pip install kazoo

          +
          • +

        • 切换到部署用户，一键部署

          +

          sh install.sh

          +
          • +

        • jps查看服务是否启动

          +
          • +
        +
            MasterServer         ----- master服务
+    WorkerServer         ----- worker服务
+    LoggerServer         ----- logger服务
+    ApiApplicationServer ----- api服务
+    AlertServer          ----- alert服务
+
        +

        日志查看

        +

        日志统一存放于指定文件夹内

        +
         logs/
+    ├── escheduler-alert-server.log
+    ├── escheduler-master-server.log
+    |—— escheduler-worker-server.log
+    |—— escheduler-api-server.log
+    |—— escheduler-logger-server.log
+
        +

        启停服务

        • 启停Master
        @@ -813,54 +573,7 @@ sh ./bin/escheduler-daemon.sh stop logger-server 
      sh ./bin/escheduler-daemon.sh start alert-server
 sh ./bin/escheduler-daemon.sh stop alert-server
-

      分布式部署

      -

      1，创建部署用户

      - -

      2，根据实际需求来创建HDFS根路径

      -

      ​ 根据 common/common.properties 中 hdfs.startup.state 的配置来判断是否启动HDFS，如果启动，则需要创建HDFS根路径，并将 owner 修改为部署用户，否则忽略此步骤

      -

      3，项目编译

      -

      ​ 如上进行 项目编译

      -

      4，将环境变量文件复制到指定目录

      -

      ​ 将.escheduler_env.sh 和 escheduler_env.py 两个环境变量文件复制到 common/common.properties配置的escheduler.env.path 和 escheduler.env.py 的目录下，并将 owner 修改为部署用户

      -

      5，修改 install.sh

      -

      ​ 修改 install.sh 中变量的值，替换成自身业务所需的值

      -

      6，一键部署

      -
        -
      • 安装 pip install kazoo
        • -
      • 安装目录如下：
        • -
      -
          bin
-    conf
-    escheduler-1.0.0-SNAPSHOT.tar.gz
-    install.sh
-    lib
-    monitor_server.py
-    script
-    sql
-
        -

      • 使用部署用户 sh install.sh 一键部署

        -
          -
        • 注意：scp_hosts.sh 里 tar -zxvf $workDir/../escheduler-1.0.0.tar.gz -C $installPath 中的版本号(1.0.0)需要执行前手动替换成对应的版本号
          • -
        -
        • -
      -

      服务监控

      -

      monitor_server.py 脚本是监听，master和worker服务挂掉重启的脚本

      -

      注意：在全部服务都启动之后启动

      -

      nohup python -u monitor_server.py > nohup.out 2>&1 &

      -

      日志查看

      -

      日志统一存放于指定文件夹内

      -
       logs/
-    ├── escheduler-alert-server.log
-    ├── escheduler-master-server.log
-    |—— escheduler-worker-server.log
-    |—— escheduler-api-server.log
-    |—— escheduler-logger-server.log
      - @@ -899,7 +612,7 @@ sh ./bin/escheduler-daemon.sh stop alert-server diff --git a/escheduler-ui/src/js/conf/home/pages/security/pages/queue/_source/createQueue.vue b/escheduler-ui/src/js/conf/home/pages/security/pages/queue/_source/createQueue.vue index 4e66b1432e..7061d02bf0 100644 --- a/escheduler-ui/src/js/conf/home/pages/security/pages/queue/_source/createQueue.vue +++ b/escheduler-ui/src/js/conf/home/pages/security/pages/queue/_source/createQueue.vue @@ -65,26 +65,40 @@ // edit if (this.item) { param.id = this.item.id - if (this.item.queueName === this.queueName && this.item.queue === this.queue) { - this.$message.warning(`名称和队列值未做更改`) - return - } } - this._verifyName(param).then(() => { + + let $then = (res) => { + this.$emit('onUpdate') + this.$message.success(res.msg) + setTimeout(() => { + this.$refs['popup'].spinnerLoading = false + }, 800) + } + + let $catch = (e) => { + this.$message.error(e.msg || '') + this.$refs['popup'].spinnerLoading = false + } + + if (this.item) { this.$refs['popup'].spinnerLoading = true - this.store.dispatch(`security/${this.item ? 'updateQueueQ' : 'createQueueQ'}`, param).then(res => { - this.$emit('onUpdate') - this.$message.success(res.msg) - setTimeout(() => { - this.$refs['popup'].spinnerLoading = false - }, 800) + this.store.dispatch(`security/updateQueueQ`, param).then(res => { + $then(res) + }).catch(e => { + $catch(e) + }) + }else{ + this._verifyName(param).then(() => { + this.$refs['popup'].spinnerLoading = true + this.store.dispatch(`security/createQueueQ`, param).then(res => { + $then(res) + }).catch(e => { + $catch(e) + }) }).catch(e => { this.$message.error(e.msg || '') - this.$refs['popup'].spinnerLoading = false }) - }).catch(e => { - this.$message.error(e.msg || '') - }) + } }, _verification(){ From 57227cd602fc2a2553006d2a777598ee1bb0ab12 Mon Sep 17 00:00:00 2001 From: gongzijian Date: Fri, 12 Apr 2019 11:17:42 +0800 Subject: [PATCH 18/18] update doc --- docs/zh_CN/_book/前端部署文档.html | 35 ++++++++++++++---------- docs/zh_CN/前端部署文档.md | 28 ++++++++++++------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/docs/zh_CN/_book/前端部署文档.html b/docs/zh_CN/_book/前端部署文档.html index 26b990d9e9..4d624c39db 100644 --- a/docs/zh_CN/_book/前端部署文档.html +++ b/docs/zh_CN/_book/前端部署文档.html @@ -430,12 +430,22 @@

      前端部署文档

      +
        +
      • 1. 开发环境搭建
        +
        • +
      • 2. 自动化部署
        +
        • +
      • 3. 手动部署
        +
        • +
      • 4. Liunx下使用node启动并且守护进程
        +
        • +

      1.开发环境搭建

        -

      • 安装node

        +

      • node安装

        Node包下载 (注意版本 8.9.4) https://nodejs.org/download/release/v8.9.4/

        • -

      • 构建项目

        +

      • 前端项目构建

        用命令行模式 cd 进入 escheduler-ui项目目录并执行 npm install 拉取项目依赖包

      @@ -459,8 +469,7 @@ API_BASE = http://192.168.220.204:12345

    • npm run build 项目打包 (打包后根目录会创建一个名为dist文件夹，用于发布线上Nginx)

    -

    2.自动部署方式

    -

    2.自动化部署`

    +

    2.自动化部署

    在项目escheduler-ui根目录编辑安装文件vi install(线上环境).sh

    更改前端访问端口和后端代理接口地址

    # 配置前端访问端口
@@ -469,8 +478,6 @@ esc_proxy="8888"
 # 配置代理后端接口
 esc_proxy_port="http://192.168.220.154:12345"

    前端自动部署基于yum操作，部署之前请先安装更新`yum

    -

    在项目escheduler-ui根目录下，修改install.sh中的参数，执行./install(线上环境).sh

    -

    3.手动部署方式

    在项目escheduler-ui根目录执行./install(线上环境).sh

    3.手动部署

    安装epel源 yum install epel-release -y

    @@ -553,14 +560,12 @@ esc_proxy_port="http://192.168.220.154:12345" │ npm │ 0 │ N/A │ fork │ 6168 │ online │ 31 │ 0s │ 0% │ 5.6 MB │ root │ disabled │ └──────────┴────┴─────────┴──────┴──────┴────────┴─────────┴────────┴─────┴──────────┴──────┴──────────┘ Use `pm2 show <id|name>` to get more details about an app -## FAQ - -#### 1. 上传文件大小限制 -编辑配置文件 `vi /etc/nginx/nginx.conf` -

    更改上传大小

    -

    client_max_body_size 1024m -```

    - +

    问题

    +

    1. 上传文件大小限制

    +

    编辑配置文件 vi /etc/nginx/nginx.conf

    +
    # 更改上传大小
+client_max_body_size 1024m
+
    @@ -599,7 +604,7 @@ esc_proxy_port="http://192.168.220.154:12345" diff --git a/docs/zh_CN/前端部署文档.md b/docs/zh_CN/前端部署文档.md index 77ade9680d..7f22a5ddaa 100644 --- a/docs/zh_CN/前端部署文档.md +++ b/docs/zh_CN/前端部署文档.md @@ -1,11 +1,20 @@ # 前端部署文档 +- ##### 1. 开发环境搭建 + +- ##### 2. 自动化部署 + +- ##### 3. 手动部署 + +- ##### 4. Liunx下使用node启动并且守护进程 + + ### 1.开发环境搭建 -- #### 安装node +- #### node安装 Node包下载 (注意版本 8.9.4) `https://nodejs.org/download/release/v8.9.4/` -- #### 构建项目 +- #### 前端项目构建 用命令行模式 `cd` 进入 `escheduler-ui`项目目录并执行 `npm install` 拉取项目依赖包 > 如果 `npm install` 速度非常慢 @@ -14,6 +23,8 @@ Node包下载 (注意版本 8.9.4) `https://nodejs.org/download/release/v8.9.4/` > 运行 `cnpm install` + + > ##### ！！！这里特别注意 项目如果在拉取依赖包的过程中报 " node-sass error " 错误，请在执行完后再次执行以下命令 ``` npm install node-sass --unsafe-perm //单独安装node-sass依赖 @@ -33,9 +44,8 @@ API_BASE = http://192.168.220.204:12345 - `npm run build` 项目打包 (打包后根目录会创建一个名为dist文件夹，用于发布线上Nginx) -### 2.自动部署方式 -### 2.自动化部署` +### 2.自动化部署 在项目`escheduler-ui`根目录编辑安装文件`vi install(线上环境).sh` @@ -51,11 +61,6 @@ esc_proxy_port="http://192.168.220.154:12345" 前端自动部署基于`yum`操作，部署之前请先安装更新`yum -在项目`escheduler-ui`根目录下，修改install.sh中的参数，执行`./install(线上环境).sh` - - - -### 3.手动部署方式 在项目`escheduler-ui`根目录执行`./install(线上环境).sh` @@ -162,8 +167,11 @@ systemctl restart nginx │ npm │ 0 │ N/A │ fork │ 6168 │ online │ 31 │ 0s │ 0% │ 5.6 MB │ root │ disabled │ └──────────┴────┴─────────┴──────┴──────┴────────┴─────────┴────────┴─────┴──────────┴──────┴──────────┘ Use `pm2 show ` to get more details about an app -## FAQ +``` + + +## 问题 #### 1. 上传文件大小限制 编辑配置文件 `vi /etc/nginx/nginx.conf` ```