test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-12-01 11:47:51 +08:00
Code Issues Actions 10 Packages Projects Releases Wiki Activity

update version 2.0.0 backup to 1.3.0 (#2531)

Browse Source 
* Ambari plugin development for the dolphin scheduler in version 2.0.0 is complete

* Update the Ambari plugin of DS  usage documentation

* delete readme whic is not completed

* update version 2.0.0 backup to 1.3.0

* update ambari plugin version to 1.3.0

Co-authored-by: zhangchunyang <zhangchunyang@analysys.com.cn>
Co-authored-by: zhangchunyang <18910529250@163.com>
This commit is contained in:
zhangchunyang 2020-04-26 18:50:38 +08:00 committed by gaojun2048
parent f1f6b242af
commit dbe504cded
33 changed files with 3385 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json Normal file
View File

@ -0,0 +1,164 @@
{
"DOLPHIN": {
"service": [],
"DOLPHIN_API": [
{
"name": "dolphin_api_port_check",
"label": "dolphin_api_port_check",
"description": "dolphin_api_port_check.",
"interval": 10,
"scope": "ANY",
"source": {
"type": "PORT",
"uri": "{{dolphin-application-api/server.port}}",
"default_port": 12345,
"reporting": {
"ok": {
"text": "TCP OK - {0:.3f}s response on port {1}"
},
"warning": {
"text": "TCP OK - {0:.3f}s response on port {1}",
"value": 1.5
},
"critical": {
"text": "Connection failed: {0} to {1}:{2}",
"value": 5.0
}
}
}
}
],
"DOLPHIN_MASTER": [
{
"name": "DOLPHIN_MASTER_CHECK",
"label": "check dolphin scheduler master status",
"description": "",
"interval":10,
"scope": "HOST",
"enabled": true,
"source": {
"type": "SCRIPT",
"path": "DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py",
"parameters": [
{
"name": "connection.timeout",
"display_name": "Connection Timeout",
"value": 5.0,
"type": "NUMERIC",
"description": "The maximum time before this alert is considered to be CRITICAL",
"units": "seconds",
"threshold": "CRITICAL"
},
{
"name": "alertName",
"display_name": "alertName",
"value": "DOLPHIN_MASTER",
"type": "STRING",
"description": "alert name"
}
]
}
}
],
"DOLPHIN_WORKER": [
{
"name": "DOLPHIN_WORKER_CHECK",
"label": "check dolphin scheduler worker status",
"description": "",
"interval":10,
"scope": "HOST",
"enabled": true,
"source": {
"type": "SCRIPT",
"path": "DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py",
"parameters": [
{
"name": "connection.timeout",
"display_name": "Connection Timeout",
"value": 5.0,
"type": "NUMERIC",
"description": "The maximum time before this alert is considered to be CRITICAL",
"units": "seconds",
"threshold": "CRITICAL"
},
{
"name": "alertName",
"display_name": "alertName",
"value": "DOLPHIN_WORKER",
"type": "STRING",
"description": "alert name"
}
]
}
}
],
"DOLPHIN_ALERT": [
{
"name": "DOLPHIN_DOLPHIN_ALERT_CHECK",
"label": "check dolphin scheduler alert status",
"description": "",
"interval":10,
"scope": "HOST",
"enabled": true,
"source": {
"type": "SCRIPT",
"path": "DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py",
"parameters": [
{
"name": "connection.timeout",
"display_name": "Connection Timeout",
"value": 5.0,
"type": "NUMERIC",
"description": "The maximum time before this alert is considered to be CRITICAL",
"units": "seconds",
"threshold": "CRITICAL"
},
{
"name": "alertName",
"display_name": "alertName",
"value": "DOLPHIN_ALERT",
"type": "STRING",
"description": "alert name"
}
]
}
}
],
"DOLPHIN_ALERT": [
{
"name": "DOLPHIN_DOLPHIN_LOGGER_CHECK",
"label": "check dolphin scheduler alert status",
"description": "",
"interval":10,
"scope": "HOST",
"enabled": true,
"source": {
"type": "SCRIPT",
"path": "DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py",
"parameters": [
{
"name": "connection.timeout",
"display_name": "Connection Timeout",
"value": 5.0,
"type": "NUMERIC",
"description": "The maximum time before this alert is considered to be CRITICAL",
"units": "seconds",
"threshold": "CRITICAL"
},
{
"name": "alertName",
"display_name": "alertName",
"value": "DOLPHIN_LOGGER",
"type": "STRING",
"description": "alert name"
}
]
}
}
]
}
}

143
ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-alert.xml Normal file
View File

@ -0,0 +1,143 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>alert.type</name>
<value>EMAIL</value>
<description>alert type is EMAIL/SMS</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>alert.template</name>
<value>html</value>
<description>alter msg template, default is html template</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mail.protocol</name>
<value>SMTP</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mail.server.host</name>
<value>xxx.xxx.com</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mail.server.port</name>
<value>25</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mail.sender</name>
<value>admin</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mail.user</name>
<value>admin</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mail.passwd</name>
<value>000000</value>
<description></description>
<property-type>PASSWORD</property-type>
<value-attributes>
<type>password</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mail.smtp.starttls.enable</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mail.smtp.ssl.enable</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mail.smtp.ssl.trust</name>
<value>xxx.xxx.com</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>enterprise.wechat.enable</name>
<value>false</value>
<description></description>
<value-attributes>
<type>value-list</type>
<entries>
<entry>
<value>true</value>
<label>Enabled</label>
</entry>
<entry>
<value>false</value>
<label>Disabled</label>
</entry>
</entries>
<selection-cardinality>1</selection-cardinality>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>enterprise.wechat.corp.id</name>
<value>wechatId</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>enterprise.wechat.secret</name>
<value>secret</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>enterprise.wechat.agent.id</name>
<value>agentId</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>enterprise.wechat.users</name>
<value>wechatUsers</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
</configuration>

87
ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-application-api.xml Normal file
View File

@ -0,0 +1,87 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>server.port</name>
<value>12345</value>
<description>
server port
</description>
<value-attributes>
<type>int</type>
</value-attributes>
</property>
<property>
<name>server.servlet.session.timeout</name>
<value>7200</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
</description>
</property>
<property>
<name>server.servlet.context-path</name>
<value>/dolphinscheduler/</value>
<description>
</description>
</property>
<property>
<name>spring.servlet.multipart.max-file-size</name>
<value>1024</value>
<value-attributes>
<unit>MB</unit>
<type>int</type>
</value-attributes>
<description>
</description>
</property>
<property>
<name>spring.servlet.multipart.max-request-size</name>
<value>1024</value>
<value-attributes>
<unit>MB</unit>
<type>int</type>
</value-attributes>
<description>
</description>
</property>
<property>
<name>server.jetty.max-http-post-size</name>
<value>5000000</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
</description>
</property>
<property>
<name>spring.messages.encoding</name>
<value>UTF-8</value>
<description></description>
</property>
<property>
<name>spring.messages.basename</name>
<value>i18n/messages</value>
<description></description>
</property>
<property>
<name>security.authentication.type</name>
<value>PASSWORD</value>
<description></description>
</property>
</configuration>

158
ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-common.xml Normal file
View File

@ -0,0 +1,158 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>resource.storage.type</name>
<display-name>Choose Resource Upload Startup Type</display-name>
<description>
Resource upload startup type : HDFS,S3,NONE
</description>
<value>NONE</value>
<value-attributes>
<type>value-list</type>
<entries>
<entry>
<value>HDFS</value>
<label>HDFS</label>
</entry>
<entry>
<value>S3</value>
<label>S3</label>
</entry>
<entry>
<value>NONE</value>
<label>NONE</label>
</entry>
</entries>
<selection-cardinality>1</selection-cardinality>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>resource.upload.path</name>
<value>/dolphinscheduler</value>
<description>
resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>data.basedir.path</name>
<value>/tmp/dolphinscheduler</value>
<description>
user data local directory path, please make sure the directory exists and have read write permissions
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>hadoop.security.authentication.startup.state</name>
<value>false</value>
<value-attributes>
<type>value-list</type>
<entries>
<entry>
<value>true</value>
<label>Enabled</label>
</entry>
<entry>
<value>false</value>
<label>Disabled</label>
</entry>
</entries>
<selection-cardinality>1</selection-cardinality>
</value-attributes>
<description>whether kerberos starts</description>
</property>
<property>
<name>java.security.krb5.conf.path</name>
<value>/opt/krb5.conf</value>
<description>
java.security.krb5.conf path
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>login.user.keytab.username</name>
<value>hdfs-mycluster@ESZ.COM</value>
<description>
LoginUserFromKeytab user
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>login.user.keytab.path</name>
<value>/opt/hdfs.headless.keytab</value>
<description>
LoginUserFromKeytab path
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>resource.view.suffixs</name>
<value>txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties</value>
<description></description>
</property>
<property>
<name>hdfs.root.user</name>
<value>hdfs</value>
<description>
Users who have permission to create directories under the HDFS root path
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://mycluster:8020</value>
<description>
HA or single namenode,
If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory，
support s3，for example : s3a://dolphinscheduler
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>fs.s3a.endpoint</name>
<value>http://host:9010</value>
<description>
s3 need，s3 endpoint
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>fs.s3a.access.key</name>
<value>A3DXS30FO22544RE</value>
<description>
s3 need，s3 access key
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>fs.s3a.secret.key</name>
<value>OloCLq3n+8+sdPHUhJ21XrSxTC+JK</value>
<description>
s3 need，s3 secret key
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>kerberos.expire.time</name>
<value>7</value>
<description></description>
</property>
</configuration>

467
ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-datasource.xml Normal file
View File

@ -0,0 +1,467 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>spring.datasource.initialSize</name>
<value>5</value>
<description>
Init connection number
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.minIdle</name>
<value>5</value>
<description>
Min connection number
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.maxActive</name>
<value>50</value>
<description>
Max connection number
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.maxWait</name>
<value>60000</value>
<description>
Max wait time for get a connection in milliseconds.
If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases.
If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true.
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.timeBetweenEvictionRunsMillis</name>
<value>60000</value>
<description>
Milliseconds for check to close free connections
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.timeBetweenConnectErrorMillis</name>
<value>60000</value>
<description>
The Destroy thread detects the connection interval and closes the physical connection in milliseconds
if the connection idle time is greater than or equal to minEvictableIdleTimeMillis.
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.minEvictableIdleTimeMillis</name>
<value>300000</value>
<description>
The longest time a connection remains idle without being evicted, in milliseconds
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.validationQuery</name>
<value>SELECT 1</value>
<description>
The SQL used to check whether the connection is valid requires a query statement.
If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work.
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.validationQueryTimeout</name>
<value>3</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
Check whether the connection is valid for timeout, in seconds
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.testWhileIdle</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
When applying for a connection,
if it is detected that the connection is idle longer than time Between Eviction Runs Millis,
validation Query is performed to check whether the connection is valid
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.testOnBorrow</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
Execute validation to check if the connection is valid when applying for a connection
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.testOnReturn</name>
<value>false</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
Execute validation to check if the connection is valid when the connection is returned
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.defaultAutoCommit</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.keepAlive</name>
<value>false</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.poolPreparedStatements</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
Open PSCache, specify count PSCache for every connection
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.maxPoolPreparedStatementPerConnectionSize</name>
<value>20</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.spring.datasource.filters</name>
<value>stat,wall,log4j</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.connectionProperties</name>
<value>druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.mapper-locations</name>
<value>classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.typeEnumsPackage</name>
<value>org.apache.dolphinscheduler.*.enums</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.typeAliasesPackage</name>
<value>org.apache.dolphinscheduler.dao.entity</value>
<description>
Entity scan, where multiple packages are separated by a comma or semicolon
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.global-config.db-config.id-type</name>
<value>AUTO</value>
<value-attributes>
<type>value-list</type>
<entries>
<entry>
<value>AUTO</value>
<label>AUTO</label>
</entry>
<entry>
<value>INPUT</value>
<label>INPUT</label>
</entry>
<entry>
<value>ID_WORKER</value>
<label>ID_WORKER</label>
</entry>
<entry>
<value>UUID</value>
<label>UUID</label>
</entry>
</entries>
<selection-cardinality>1</selection-cardinality>
</value-attributes>
<description>
Primary key type AUTO:" database ID AUTO ",
INPUT:" user INPUT ID",
ID_WORKER:" global unique ID (numeric type unique ID)",
UUID:" global unique ID UUID";
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.global-config.db-config.field-strategy</name>
<value>NOT_NULL</value>
<value-attributes>
<type>value-list</type>
<entries>
<entry>
<value>IGNORED</value>
<label>IGNORED</label>
</entry>
<entry>
<value>NOT_NULL</value>
<label>NOT_NULL</label>
</entry>
<entry>
<value>NOT_EMPTY</value>
<label>NOT_EMPTY</label>
</entry>
</entries>
<selection-cardinality>1</selection-cardinality>
</value-attributes>
<description>
Field policy IGNORED:" ignore judgment ",
NOT_NULL:" not NULL judgment "),
NOT_EMPTY:" not NULL judgment"
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.global-config.db-config.column-underline</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.global-config.db-config.logic-delete-value</name>
<value>1</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.global-config.db-config.logic-not-delete-value</name>
<value>0</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.global-config.db-config.banner</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.configuration.map-underscore-to-camel-case</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.configuration.cache-enabled</name>
<value>false</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.configuration.call-setters-on-nulls</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.configuration.jdbc-type-for-null</name>
<value>null</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.exec.threads</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.exec.task.num</name>
<value>20</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.heartbeat.interval</name>
<value>10</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.task.commit.retryTimes</name>
<value>5</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.task.commit.interval</name>
<value>1000</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.max.cpuload.avg</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.reserved.memory</name>
<value>0.1</value>
<value-attributes>
<type>float</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.exec.threads</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.heartbeat.interval</name>
<value>10</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.fetch.task.num</name>
<value>3</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.max.cpuload.avg</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.reserved.memory</name>
<value>0.1</value>
<value-attributes>
<type>float</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
</configuration>

123
ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml Normal file
View File

@ -0,0 +1,123 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>dolphin.database.type</name>
<value>mysql</value>
<description>Dolphin Scheduler DataBase Type Which Is Select</description>
<display-name>Dolphin Database Type</display-name>
<value-attributes>
<type>value-list</type>
<entries>
<entry>
<value>mysql</value>
<label>Mysql</label>
</entry>
<entry>
<value>postgresql</value>
<label>Postgresql</label>
</entry>
</entries>
<selection-cardinality>1</selection-cardinality>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>dolphin.database.host</name>
<value></value>
<display-name>Dolphin Database Host</display-name>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>dolphin.database.port</name>
<value></value>
<display-name>Dolphin Database Port</display-name>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>dolphin.database.username</name>
<value></value>
<display-name>Dolphin Database Username</display-name>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>dolphin.database.password</name>
<value></value>
<display-name>Dolphin Database Password</display-name>
<property-type>PASSWORD</property-type>
<value-attributes>
<type>password</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>dolphin.user</name>
<value></value>
<description>Which user to install and admin dolphin scheduler</description>
<display-name>Deploy User</display-name>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>dolphin.group</name>
<value></value>
<description>Which user to install and admin dolphin scheduler</description>
<display-name>Deploy Group</display-name>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>dolphinscheduler-env-content</name>
<display-name>Dolphinscheduler Env template</display-name>
<description>This is the jinja template for dolphinscheduler.env.sh file</description>
<value>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
export HADOOP_HOME=/opt/soft/hadoop
export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
export SPARK_HOME1=/opt/soft/spark1
export SPARK_HOME2=/opt/soft/spark2
export PYTHON_HOME=/opt/soft/python
export JAVA_HOME=/opt/soft/java
export HIVE_HOME=/opt/soft/hive
export FLINK_HOME=/opt/soft/flink</value>
<value-attributes>
<type>content</type>
<empty-value-valid>false</empty-value-valid>
<show-property-name>false</show-property-name>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
</configuration>

88
ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-master.xml Normal file
View File

@ -0,0 +1,88 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>master.exec.threads</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>master execute thread num</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.exec.task.num</name>
<value>20</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>master execute task number in parallel</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.heartbeat.interval</name>
<value>10</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>master heartbeat interval</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.task.commit.retryTimes</name>
<value>5</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>master commit task retry times</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.task.commit.interval</name>
<value>1000</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>master commit task interval</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.max.cpuload.avg</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>only less than cpu avg load, master server can work. default value : the number of cpu cores * 2</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.reserved.memory</name>
<value>0.3</value>
<description>only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.listen.port</name>
<value>5678</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>master listen port</description>
<on-ambari-upgrade add="true"/>
</property>
</configuration>

126
ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-quartz.xml Normal file
View File

@ -0,0 +1,126 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>org.quartz.scheduler.instanceName</name>
<value>DolphinScheduler</value>
<description></description>
</property>
<property>
<!-- 列举枚举值 -->
<name>org.quartz.scheduler.instanceId</name>
<value>AUTO</value>
<description></description>
</property>
<property>
<name>org.quartz.scheduler.makeSchedulerThreadDaemon</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
</property>
<property>
<name>org.quartz.jobStore.useProperties</name>
<value>false</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
</property>
<property>
<name>org.quartz.threadPool.class</name>
<value>org.quartz.simpl.SimpleThreadPool</value>
<description></description>
</property>
<property>
<name>org.quartz.threadPool.makeThreadsDaemons</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
</property>
<property>
<name>org.quartz.threadPool.threadCount</name>
<value>25</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
</property>
<property>
<name>org.quartz.threadPool.threadPriority</name>
<value>5</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
</property>
<property>
<name>org.quartz.jobStore.class</name>
<value>org.quartz.impl.jdbcjobstore.JobStoreTX</value>
<description></description>
</property>
<property>
<name>org.quartz.jobStore.tablePrefix</name>
<value>QRTZ_</value>
<description></description>
</property>
<property>
<name>org.quartz.jobStore.isClustered</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
</property>
<property>
<name>org.quartz.jobStore.misfireThreshold</name>
<value>60000</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
</property>
<property>
<name>org.quartz.jobStore.clusterCheckinInterval</name>
<value>5000</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
</property>
<property>
<name>org.quartz.jobStore.acquireTriggersWithinLock</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
</property>
<property>
<name>org.quartz.jobStore.dataSource</name>
<value>myDs</value>
<description></description>
</property>
<property>
<name>org.quartz.dataSource.myDs.connectionProvider.class</name>
<value>org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider</value>
<description></description>
</property>
</configuration>

76
ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-worker.xml Normal file
View File

@ -0,0 +1,76 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>worker.exec.threads</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker execute thread num</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.heartbeat.interval</name>
<value>10</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker heartbeat interval</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.fetch.task.num</name>
<value>3</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>submit the number of tasks at a time</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.max.cpuload.avg</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.reserved.memory</name>
<value>0.3</value>
<description>only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G.</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.listen.port</name>
<value>1234</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker listen port</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.group</name>
<value>default</value>
<description>default worker group</description>
<on-ambari-upgrade add="true"/>
</property>
</configuration>

84
ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-zookeeper.xml Normal file
View File

@ -0,0 +1,84 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>dolphinscheduler.queue.impl</name>
<value>zookeeper</value>
<description>
Task queue implementation, default "zookeeper"
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>zookeeper.dolphinscheduler.root</name>
<value>/dolphinscheduler</value>
<description>
dolphinscheduler root directory
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>zookeeper.session.timeout</name>
<value>300</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>zookeeper.connection.timeout</name>
<value>300</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>zookeeper.retry.base.sleep</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>zookeeper.retry.max.sleep</name>
<value>30000</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>zookeeper.retry.maxtime</name>
<value>5</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
</configuration>

137
ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml Normal file
View File

@ -0,0 +1,137 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<metainfo>
<schemaVersion>2.0</schemaVersion>
<services>
<service>
<name>DOLPHIN</name>
<displayName>Dolphin Scheduler</displayName>
<comment>分布式易扩展的可视化DAG工作流任务调度系统</comment>
<version>1.3.0</version>
<components>
<component>
<name>DOLPHIN_MASTER</name>
<displayName>DS Master</displayName>
<category>MASTER</category>
<cardinality>1+</cardinality>
<commandScript>
<script>scripts/dolphin_master_service.py</script>
<scriptType>PYTHON</scriptType>
<timeout>600</timeout>
</commandScript>
</component>
<component>
<name>DOLPHIN_LOGGER</name>
<displayName>DS Logger</displayName>
<category>SLAVE</category>
<cardinality>1+</cardinality>
<commandScript>
<script>scripts/dolphin_logger_service.py</script>
<scriptType>PYTHON</scriptType>
<timeout>600</timeout>
</commandScript>
</component>
<component>
<name>DOLPHIN_WORKER</name>
<displayName>DS Worker</displayName>
<category>SLAVE</category>
<cardinality>1+</cardinality>
<dependencies>
<dependency>
<name>DOLPHIN/DOLPHIN_LOGGER</name>
<scope>host</scope>
<auto-deploy>
<enabled>true</enabled>
</auto-deploy>
</dependency>
</dependencies>
<commandScript>
<script>scripts/dolphin_worker_service.py</script>
<scriptType>PYTHON</scriptType>
<timeout>600</timeout>
</commandScript>
</component>
<component>
<name>DOLPHIN_ALERT</name>
<displayName>DS Alert</displayName>
<category>SLAVE</category>
<cardinality>1</cardinality>
<commandScript>
<script>scripts/dolphin_alert_service.py</script>
<scriptType>PYTHON</scriptType>
<timeout>600</timeout>
</commandScript>
</component>
<component>
<name>DOLPHIN_API</name>
<displayName>DS_Api</displayName>
<category>SLAVE</category>
<cardinality>1</cardinality>
<commandScript>
<script>scripts/dolphin_api_service.py</script>
<scriptType>PYTHON</scriptType>
<timeout>600</timeout>
</commandScript>
</component>
</components>
<requiredServices>
<service>ZOOKEEPER</service>
</requiredServices>
<osSpecifics>
<osSpecific>
<osFamily>any</osFamily>
<packages>
<package>
<name>apache-dolphinscheduler-incubating-1.3.0*</name>
</package>
</packages>
</osSpecific>
</osSpecifics>
<configuration-dependencies>
<config-type>dolphin-alert</config-type>
<config-type>dolphin-app-api</config-type>
<config-type>dolphin-app-dao</config-type>
<config-type>dolphin-common</config-type>
<config-type>dolphin-env</config-type>
<config-type>dolphin-quartz</config-type>
</configuration-dependencies>
<themes>
<theme>
<fileName>theme.json</fileName>
<default>true</default>
</theme>
</themes>
<quickLinksConfigurations-dir>quicklinks</quickLinksConfigurations-dir>
<quickLinksConfigurations>
<quickLinksConfiguration>
<fileName>quicklinks.json</fileName>
<default>true</default>
</quickLinksConfiguration>
</quickLinksConfigurations>
</service>
</services>
</metainfo>

124
ambari_plugin/common-services/DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py Normal file
View File

@ -0,0 +1,124 @@
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import socket
import urllib2
import os
import logging
import ambari_simplejson as json
from resource_management.libraries.script.script import Script
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
logger = logging.getLogger('ambari_alerts')
config = Script.get_config()
def get_tokens():
"""
Returns a tuple of tokens in the format {{site/property}} that will be used
to build the dictionary passed into execute
:rtype tuple
"""
def get_info(url, connection_timeout):
response = None
try:
response = urllib2.urlopen(url, timeout=connection_timeout)
json_data = response.read()
return json_data
finally:
if response is not None:
try:
response.close()
except:
pass
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations : a mapping of configuration key to value
parameters : a mapping of script parameter key to value
host_name : the name of this host where the alert is running
:type configurations dict
:type parameters dict
:type host_name str
"""
alert_name = parameters['alertName']
dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler"
pid = "0"
from resource_management.core import sudo
is_running = True
pid_file_path = ""
if alert_name == 'DOLPHIN_MASTER':
pid_file_path = dolphin_pidfile_dir + "/master-server.pid"
elif alert_name == 'DOLPHIN_WORKER':
pid_file_path = dolphin_pidfile_dir + "/worker-server.pid"
elif alert_name == 'DOLPHIN_ALERT':
pid_file_path = dolphin_pidfile_dir + "/alert-server.pid"
elif alert_name == 'DOLPHIN_LOGGER':
pid_file_path = dolphin_pidfile_dir + "/logger-server.pid"
elif alert_name == 'DOLPHIN_API':
pid_file_path = dolphin_pidfile_dir + "/api-server.pid"
if not pid_file_path or not os.path.isfile(pid_file_path):
is_running = False
try:
pid = int(sudo.read_file(pid_file_path))
except:
is_running = False
try:
# Kill will not actually kill the process
# From the doc:
# If sig is 0, then no signal is sent, but error checking is still
# performed; this can be used to check for the existence of a
# process ID or process group ID.
sudo.kill(pid, 0)
except OSError:
is_running = False
if host_name is None:
host_name = socket.getfqdn()
if not is_running:
result_code = "CRITICAL"
else:
result_code = "OK"
label = "The comment {0} of DOLPHIN_SCHEDULER on {1} is {2}".format(alert_name, host_name, result_code)
return ((result_code, [label]))
if __name__ == "__main__":
pass

61
ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py Normal file
View File

@ -0,0 +1,61 @@
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import time
from resource_management import *
from dolphin_env import dolphin_env
class DolphinAlertService(Script):
def install(self, env):
import params
env.set_params(params)
self.install_packages(env)
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
def configure(self, env):
import params
params.pika_slave = True
env.set_params(params)
dolphin_env()
def start(self, env):
import params
env.set_params(params)
self.configure(env)
no_op_test = format("ls {dolphin_pidfile_dir}/alert-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/alert-server.pid` | grep `cat {dolphin_pidfile_dir}/alert-server.pid` >/dev/null 2>&1")
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start alert-server")
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test)
def stop(self, env):
import params
env.set_params(params)
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop alert-server")
Execute(stop_cmd, user=params.dolphin_user)
time.sleep(5)
def status(self, env):
import status_params
env.set_params(status_params)
check_process_status(status_params.dolphin_run_dir + "alert-server.pid")
if __name__ == "__main__":
DolphinAlertService().execute()

70
ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py Normal file
View File

@ -0,0 +1,70 @@
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import time
from resource_management import *
from dolphin_env import dolphin_env
class DolphinApiService(Script):
def install(self, env):
import params
env.set_params(params)
self.install_packages(env)
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
def configure(self, env):
import params
params.pika_slave = True
env.set_params(params)
dolphin_env()
def start(self, env):
import params
env.set_params(params)
self.configure(env)
#init
init_cmd=format("sh " + params.dolphin_home + "/script/create-dolphinscheduler.sh")
Execute(init_cmd, user=params.dolphin_user)
#upgrade
upgrade_cmd=format("sh " + params.dolphin_home + "/script/upgrade-dolphinscheduler.sh")
Execute(upgrade_cmd, user=params.dolphin_user)
no_op_test = format("ls {dolphin_pidfile_dir}/api-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/api-server.pid` | grep `cat {dolphin_pidfile_dir}/api-server.pid` >/dev/null 2>&1")
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start api-server")
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test)
def stop(self, env):
import params
env.set_params(params)
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop api-server")
Execute(stop_cmd, user=params.dolphin_user)
time.sleep(5)
def status(self, env):
import status_params
env.set_params(status_params)
check_process_status(status_params.dolphin_run_dir + "api-server.pid")
if __name__ == "__main__":
DolphinApiService().execute()

123
ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_env.py Normal file
View File

@ -0,0 +1,123 @@
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from resource_management import *
def dolphin_env():
import params
Directory(params.dolphin_pidfile_dir,
mode=0777,
owner=params.dolphin_user,
group=params.dolphin_group,
create_parents=True
)
Directory(params.dolphin_log_dir,
mode=0777,
owner=params.dolphin_user,
group=params.dolphin_group,
create_parents=True
)
Directory(params.dolphin_conf_dir,
mode=0777,
owner=params.dolphin_user,
group=params.dolphin_group,
create_parents=True
)
Directory(params.dolphin_common_map['data.basedir.path'],
mode=0777,
owner=params.dolphin_user,
group=params.dolphin_group,
create_parents=True
)
File(format(params.dolphin_env_path),
mode=0777,
content=InlineTemplate(params.dolphin_env_content),
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh"),
mode=0755,
content=Template("dolphin-daemon.sh.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/master.properties"),
mode=0755,
content=Template("master.properties.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/worker.properties"),
mode=0755,
content=Template("worker.properties.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/alert.properties"),
mode=0755,
content=Template("alert.properties.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/datasource.properties"),
mode=0755,
content=Template("datasource.properties.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/application-api.properties"),
mode=0755,
content=Template("application-api.properties.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/common.properties"),
mode=0755,
content=Template("common.properties.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/quartz.properties"),
mode=0755,
content=Template("quartz.properties.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/zookeeper.properties"),
mode=0755,
content=Template("zookeeper.properties.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)

61
ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py Normal file
View File

@ -0,0 +1,61 @@
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import time
from resource_management import *
from dolphin_env import dolphin_env
class DolphinLoggerService(Script):
def install(self, env):
import params
env.set_params(params)
self.install_packages(env)
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
def configure(self, env):
import params
params.pika_slave = True
env.set_params(params)
dolphin_env()
def start(self, env):
import params
env.set_params(params)
self.configure(env)
no_op_test = format("ls {dolphin_pidfile_dir}/logger-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/logger-server.pid` | grep `cat {dolphin_pidfile_dir}/logger-server.pid` >/dev/null 2>&1")
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start logger-server")
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test)
def stop(self, env):
import params
env.set_params(params)
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop logger-server")
Execute(stop_cmd, user=params.dolphin_user)
time.sleep(5)
def status(self, env):
import status_params
env.set_params(status_params)
check_process_status(status_params.dolphin_run_dir + "logger-server.pid")
if __name__ == "__main__":
DolphinLoggerService().execute()

61
ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py Normal file
View File

@ -0,0 +1,61 @@
# -*- coding: utf-8 -*-
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import time
from resource_management import *
from dolphin_env import dolphin_env
class DolphinMasterService(Script):
def install(self, env):
import params
env.set_params(params)
self.install_packages(env)
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
def configure(self, env):
import params
params.pika_slave = True
env.set_params(params)
dolphin_env()
def start(self, env):
import params
env.set_params(params)
self.configure(env)
no_op_test = format("ls {dolphin_pidfile_dir}/master-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/master-server.pid` | grep `cat {dolphin_pidfile_dir}/master-server.pid` >/dev/null 2>&1")
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start master-server")
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test)
def stop(self, env):
import params
env.set_params(params)
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop master-server")
Execute(stop_cmd, user=params.dolphin_user)
time.sleep(5)
def status(self, env):
import status_params
env.set_params(status_params)
check_process_status(status_params.dolphin_run_dir + "master-server.pid")
if __name__ == "__main__":
DolphinMasterService().execute()

60
ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py Normal file
View File

@ -0,0 +1,60 @@
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import time
from resource_management import *
from dolphin_env import dolphin_env
class DolphinWorkerService(Script):
def install(self, env):
import params
env.set_params(params)
self.install_packages(env)
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
def configure(self, env):
import params
params.pika_slave = True
env.set_params(params)
dolphin_env()
def start(self, env):
import params
env.set_params(params)
self.configure(env)
no_op_test = format("ls {dolphin_pidfile_dir}/worker-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/worker-server.pid` | grep `cat {dolphin_pidfile_dir}/worker-server.pid` >/dev/null 2>&1")
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start worker-server")
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test)
def stop(self, env):
import params
env.set_params(params)
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop worker-server")
Execute(stop_cmd, user=params.dolphin_user)
time.sleep(5)
def status(self, env):
import status_params
env.set_params(status_params)
check_process_status(status_params.dolphin_run_dir + "worker-server.pid")
if __name__ == "__main__":
DolphinWorkerService().execute()

154
ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py Normal file
View File

@ -0,0 +1,154 @@
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import sys
from resource_management import *
from resource_management.core.logger import Logger
from resource_management.libraries.functions import default
Logger.initialize_logger()
reload(sys)
sys.setdefaultencoding('utf-8')
# server configurations
config = Script.get_config()
# conf_dir = "/etc/"
dolphin_home = "/opt/soft/dolphinscheduler"
dolphin_conf_dir = dolphin_home + "/conf"
dolphin_log_dir = dolphin_home + "/logs"
dolphin_bin_dir = dolphin_home + "/bin"
dolphin_lib_jars = dolphin_home + "/lib/*"
dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler"
rmHosts = default("/clusterHostInfo/rm_host", [])
# dolphin-env
dolphin_env_map = {}
dolphin_env_map.update(config['configurations']['dolphin-env'])
# which user to install and admin dolphin scheduler
dolphin_user = dolphin_env_map['dolphin.user']
dolphin_group = dolphin_env_map['dolphin.group']
# .dolphinscheduler_env.sh
dolphin_env_path = dolphin_conf_dir + '/env/dolphinscheduler_env.sh'
dolphin_env_content = dolphin_env_map['dolphinscheduler-env-content']
# database config
dolphin_database_config = {}
dolphin_database_config['dolphin_database_type'] = dolphin_env_map['dolphin.database.type']
dolphin_database_config['dolphin_database_username'] = dolphin_env_map['dolphin.database.username']
dolphin_database_config['dolphin_database_password'] = dolphin_env_map['dolphin.database.password']
if 'mysql' == dolphin_database_config['dolphin_database_type']:
dolphin_database_config['dolphin_database_driver'] = 'com.mysql.jdbc.Driver'
dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.StdJDBCDelegate'
dolphin_database_config['dolphin_database_url'] = 'jdbc:mysql://' + dolphin_env_map['dolphin.database.host'] \
+ ':' + dolphin_env_map['dolphin.database.port'] \
+ '/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8'
else:
dolphin_database_config['dolphin_database_driver'] = 'org.postgresql.Driver'
dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.PostgreSQLDelegate'
dolphin_database_config['dolphin_database_url'] = 'jdbc:postgresql://' + dolphin_env_map['dolphin.database.host'] \
+ ':' + dolphin_env_map['dolphin.database.port'] \
+ '/dolphinscheduler'
# application-alert.properties
dolphin_alert_map = {}
wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token'
wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret'
wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}'
wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}'
dolphin_alert_config_map = config['configurations']['dolphin-alert']
if dolphin_alert_config_map['enterprise.wechat.enable']:
dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url
dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url
dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg
dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg
dolphin_alert_map.update(dolphin_alert_config_map)
# application-api.properties
dolphin_app_api_map = {}
dolphin_app_api_map.update(config['configurations']['dolphin-application-api'])
# common.properties
dolphin_common_map = {}
if 'yarn-site' in config['configurations'] and \
'yarn.resourcemanager.webapp.address' in config['configurations']['yarn-site']:
yarn_resourcemanager_webapp_address = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address']
yarn_application_status_address = 'http://' + yarn_resourcemanager_webapp_address + '/ws/v1/cluster/apps/%s'
dolphin_common_map['yarn.application.status.address'] = yarn_application_status_address
rmHosts = default("/clusterHostInfo/rm_host", [])
if len(rmHosts) > 1:
dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = ','.join(rmHosts)
else:
dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = ''
dolphin_common_map_tmp = config['configurations']['dolphin-common']
data_basedir_path = dolphin_common_map_tmp['data.basedir.path']
process_exec_basepath = data_basedir_path + '/exec'
data_download_basedir_path = data_basedir_path + '/download'
dolphin_common_map['process.exec.basepath'] = process_exec_basepath
dolphin_common_map['data.download.basedir.path'] = data_download_basedir_path
dolphin_common_map['dolphinscheduler.env.path'] = dolphin_env_path
dolphin_common_map.update(config['configurations']['dolphin-common'])
# datasource.properties
dolphin_datasource_map = {}
dolphin_datasource_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource'
dolphin_datasource_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver']
dolphin_datasource_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url']
dolphin_datasource_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username']
dolphin_datasource_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password']
dolphin_datasource_map.update(config['configurations']['dolphin-datasource'])
# master.properties
dolphin_master_map = config['configurations']['dolphin-master']
# quartz.properties
dolphin_quartz_map = {}
dolphin_quartz_map['org.quartz.jobStore.driverDelegateClass'] = dolphin_database_config['driverDelegateClass']
dolphin_quartz_map.update(config['configurations']['dolphin-quartz'])
# worker.properties
dolphin_worker_map = config['configurations']['dolphin-worker']
# zookeeper.properties
dolphin_zookeeper_map={}
zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", [])
if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']:
clientPort = config['configurations']['zoo.cfg']['clientPort']
zookeeperPort = ":" + clientPort + ","
dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort
dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper'])

31
ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/service_check.py Normal file
View File

@ -0,0 +1,31 @@
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from resource_management import *
from resource_management.libraries.functions import get_unique_id_and_date
class ServiceCheck(Script):
def service_check(self, env):
import params
#env.set_params(params)
# Execute(format("which pika_server"))
if __name__ == "__main__":
ServiceCheck().execute()

23
ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/status_params.py Normal file
View File

@ -0,0 +1,23 @@
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from resource_management import *
config = Script.get_config()
dolphin_run_dir = "/opt/soft/run/dolphinscheduler/"

20
ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/alert.properties.j2 Normal file
View File

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_alert_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}

20
ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/application-api.properties.j2 Normal file
View File

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_app_api_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}

20
ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/common.properties.j2 Normal file
View File

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_common_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}

20
ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/datasource.properties.j2 Normal file
View File

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_datasource_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}

116
ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/dolphin-daemon.sh.j2 Normal file
View File

@ -0,0 +1,116 @@
#!/bin/sh
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
usage="Usage: dolphinscheduler-daemon.sh (start|stop) <command> "
# if no args specified, show usage
if [ $# -le 1 ]; then
echo $usage
exit 1
fi
startStop=$1
shift
command=$1
shift
echo "Begin $startStop $command......"
BIN_DIR=`dirname $0`
BIN_DIR=`cd "$BIN_DIR"; pwd`
DOLPHINSCHEDULER_HOME=$BIN_DIR/..
export HOSTNAME=`hostname`
DOLPHINSCHEDULER_LIB_JARS={{dolphin_lib_jars}}
DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms4g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
STOP_TIMEOUT=5
log={{dolphin_log_dir}}/dolphinscheduler-$command-$HOSTNAME.out
pid={{dolphin_pidfile_dir}}/$command.pid
cd $DOLPHINSCHEDULER_HOME
if [ "$command" = "api-server" ]; then
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-api.xml -Dspring.profiles.active=api"
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
elif [ "$command" = "master-server" ]; then
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-master.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.master.MasterServer
elif [ "$command" = "worker-server" ]; then
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-worker.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
elif [ "$command" = "alert-server" ]; then
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-alert.xml"
CLASS=org.apache.dolphinscheduler.alert.AlertServer
elif [ "$command" = "logger-server" ]; then
CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
else
echo "Error: No command named \`$command' was found."
exit 1
fi
case $startStop in
(start)
if [ -f $pid ]; then
if kill -0 `cat $pid` > /dev/null 2>&1; then
echo $command running as process `cat $pid`. Stop it first.
exit 1
fi
fi
echo starting $command, logging to $log
exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath {{dolphin_conf_dir}}:{{dolphin_lib_jars}} $CLASS"
echo "nohup java $exec_command > $log 2>&1 < /dev/null &"
nohup java $exec_command > $log 2>&1 < /dev/null &
echo $! > $pid
;;
(stop)
if [ -f $pid ]; then
TARGET_PID=`cat $pid`
if kill -0 $TARGET_PID > /dev/null 2>&1; then
echo stopping $command
kill $TARGET_PID
sleep $STOP_TIMEOUT
if kill -0 $TARGET_PID > /dev/null 2>&1; then
echo "$command did not stop gracefully after $STOP_TIMEOUT seconds: killing with kill -9"
kill -9 $TARGET_PID
fi
else
echo no $command to stop
fi
rm -f $pid
else
echo no $command to stop
fi
;;
(*)
echo $usage
exit 1
;;
esac
echo "End $startStop $command."

20
ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/master.properties.j2 Normal file
View File

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_master_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}

20
ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/quartz.properties.j2 Normal file
View File

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_quartz_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}

20
ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/worker.properties.j2 Normal file
View File

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_worker_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}

20
ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/zookeeper.properties.j2 Normal file
View File

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_zookeeper_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}

26
ambari_plugin/common-services/DOLPHIN/1.3.0/quicklinks/quicklinks.json Executable file
View File

@ -0,0 +1,26 @@
{
"name": "default",
"description": "default quick links configuration",
"configuration": {
"protocol":
{
"type":"http"
},
"links": [
{
"name": "dolphin-application-ui",
"label": "DolphinApplication UI",
"requires_user_name": "false",
"component_name": "DOLPHIN_API",
"url": "%@://%@:%@/dolphinscheduler/ui/view/login/index.html",
"port":{
"http_property": "server.port",
"http_default_port": "12345",
"regex": "^(\\d+)$",
"site": "dolphin-application-api"
}
}
]
}
}

661
ambari_plugin/common-services/DOLPHIN/1.3.0/themes/theme.json Normal file
View File

@ -0,0 +1,661 @@
{
"name": "default",
"description": "Default theme for Dolphin Scheduler service",
"configuration": {
"layouts": [
{
"name": "default",
"tabs": [
{
"name": "settings",
"display-name": "Settings",
"layout": {
"tab-rows": "3",
"tab-columns": "3",
"sections": [
{
"name": "dolphin-env-config",
"display-name": "Dolphin Env Config",
"row-index": "0",
"column-index": "0",
"row-span": "1",
"column-span": "2",
"section-rows": "1",
"section-columns": "2",
"subsections": [
{
"name": "env-row1-col1",
"display-name": "Deploy User Info",
"row-index": "0",
"column-index": "0",
"row-span": "1",
"column-span": "1"
},
{
"name": "env-row1-col2",
"display-name": "System Env Optimization",
"row-index": "0",
"column-index": "1",
"row-span": "1",
"column-span": "1"
}
]
},
{
"name": "dolphin-database-config",
"display-name": "Database Config",
"row-index": "1",
"column-index": "0",
"row-span": "1",
"column-span": "2",
"section-rows": "1",
"section-columns": "3",
"subsections": [
{
"name": "database-row1-col1",
"row-index": "0",
"column-index": "0",
"row-span": "1",
"column-span": "1"
},
{
"name": "database-row1-col2",
"row-index": "0",
"column-index": "1",
"row-span": "1",
"column-span": "1"
},
{
"name": "database-row1-col3",
"row-index": "0",
"column-index": "2",
"row-span": "1",
"column-span": "1"
}
]
},
{
"name": "dynamic-config",
"row-index": "2",
"column-index": "0",
"row-span": "1",
"column-span": "2",
"section-rows": "1",
"section-columns": "3",
"subsections": [
{
"name": "dynamic-row1-col1",
"display-name": "Resource FS Config",
"row-index": "0",
"column-index": "0",
"row-span": "1",
"column-span": "1"
},
{
"name": "dynamic-row1-col2",
"display-name": "Kerberos Info",
"row-index": "0",
"column-index": "1",
"row-span": "1",
"column-span": "1"
},
{
"name": "dynamic-row1-col3",
"display-name": "Wechat Info",
"row-index": "0",
"column-index": "1",
"row-span": "1",
"column-span": "1"
}
]
}
]
}
}
]
}
],
"placement": {
"configuration-layout": "default",
"configs": [
{
"config": "dolphin-env/dolphin.database.type",
"subsection-name": "database-row1-col1"
},
{
"config": "dolphin-env/dolphin.database.host",
"subsection-name": "database-row1-col2"
},
{
"config": "dolphin-env/dolphin.database.port",
"subsection-name": "database-row1-col2"
},
{
"config": "dolphin-env/dolphin.database.username",
"subsection-name": "database-row1-col3"
},
{
"config": "dolphin-env/dolphin.database.password",
"subsection-name": "database-row1-col3"
},
{
"config": "dolphin-env/dolphin.user",
"subsection-name": "env-row1-col1"
},
{
"config": "dolphin-env/dolphin.group",
"subsection-name": "env-row1-col1"
},
{
"config": "dolphin-env/dolphinscheduler-env-content",
"subsection-name": "env-row1-col2"
},
{
"config": "dolphin-common/resource.storage.type",
"subsection-name": "dynamic-row1-col1"
},
{
"config": "dolphin-common/resource.upload.path",
"subsection-name": "dynamic-row1-col1",
"depends-on": [
{
"configs":[
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/resource.storage.type} === HDFS || ${dolphin-common/resource.storage.type} === S3",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-common/hdfs.root.user",
"subsection-name": "dynamic-row1-col1",
"depends-on": [
{
"configs":[
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/resource.storage.type} === HDFS",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-common/data.store2hdfs.basepath",
"subsection-name": "dynamic-row1-col1",
"depends-on": [
{
"configs":[
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/resource.storage.type} === HDFS",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-common/fs.defaultFS",
"subsection-name": "dynamic-row1-col1",
"depends-on": [
{
"configs":[
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/resource.storage.type} === HDFS",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-common/fs.s3a.endpoint",
"subsection-name": "dynamic-row1-col1",
"depends-on": [
{
"configs":[
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/resource.storage.type} === S3",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-common/fs.s3a.access.key",
"subsection-name": "dynamic-row1-col1",
"depends-on": [
{
"configs":[
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/resource.storage.type} === S3",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-common/fs.s3a.secret.key",
"subsection-name": "dynamic-row1-col1",
"depends-on": [
{
"configs":[
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/resource.storage.type} === S3",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-common/hadoop.security.authentication.startup.state",
"subsection-name": "dynamic-row1-col2"
},
{
"config": "dolphin-common/java.security.krb5.conf.path",
"subsection-name": "dynamic-row1-col2",
"depends-on": [
{
"configs":[
"dolphin-common/hadoop.security.authentication.startup.state"
],
"if": "${dolphin-common/hadoop.security.authentication.startup.state}",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-common/login.user.keytab.username",
"subsection-name": "dynamic-row1-col2",
"depends-on": [
{
"configs":[
"dolphin-common/hadoop.security.authentication.startup.state"
],
"if": "${dolphin-common/hadoop.security.authentication.startup.state}",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-common/login.user.keytab.path",
"subsection-name": "dynamic-row1-col2",
"depends-on": [
{
"configs":[
"dolphin-common/hadoop.security.authentication.startup.state"
],
"if": "${dolphin-common/hadoop.security.authentication.startup.state}",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-common/kerberos.expire.time",
"subsection-name": "dynamic-row1-col2",
"depends-on": [
{
"configs":[
"dolphin-common/hadoop.security.authentication.startup.state"
],
"if": "${dolphin-common/hadoop.security.authentication.startup.state}",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-alert/enterprise.wechat.enable",
"subsection-name": "dynamic-row1-col3"
},
{
"config": "dolphin-alert/enterprise.wechat.corp.id",
"subsection-name": "dynamic-row1-col3",
"depends-on": [
{
"configs":[
"dolphin-alert/enterprise.wechat.enable"
],
"if": "${dolphin-alert/enterprise.wechat.enable}",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-alert/enterprise.wechat.secret",
"subsection-name": "dynamic-row1-col3",
"depends-on": [
{
"configs":[
"dolphin-alert/enterprise.wechat.enable"
],
"if": "${dolphin-alert/enterprise.wechat.enable}",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-alert/enterprise.wechat.agent.id",
"subsection-name": "dynamic-row1-col3",
"depends-on": [
{
"configs":[
"dolphin-alert/enterprise.wechat.enable"
],
"if": "${dolphin-alert/enterprise.wechat.enable}",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-alert/enterprise.wechat.users",
"subsection-name": "dynamic-row1-col3",
"depends-on": [
{
"configs":[
"dolphin-alert/enterprise.wechat.enable"
],
"if": "${dolphin-alert/enterprise.wechat.enable}",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
}
]
},
"widgets": [
{
"config": "dolphin-env/dolphin.database.type",
"widget": {
"type": "combo"
}
},
{
"config": "dolphin-env/dolphin.database.host",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-env/dolphin.database.port",
"widget": {
"type": "text-field",
"units": [
{
"unit-name": "int"
}
]
}
},
{
"config": "dolphin-env/dolphin.database.username",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-env/dolphin.database.password",
"widget": {
"type": "password"
}
},
{
"config": "dolphin-env/dolphin.user",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-env/dolphin.group",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-env/dolphinscheduler-env-content",
"widget": {
"type": "text-area"
}
},
{
"config": "dolphin-common/resource.storage.type",
"widget": {
"type": "combo"
}
},
{
"config": "dolphin-common/resource.upload.path",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-common/hdfs.root.user",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-common/data.store2hdfs.basepath",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-common/fs.defaultFS",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-common/fs.s3a.endpoint",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-common/fs.s3a.access.key",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-common/fs.s3a.secret.key",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-common/hadoop.security.authentication.startup.state",
"widget": {
"type": "toggle"
}
},
{
"config": "dolphin-common/java.security.krb5.conf.path",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-common/login.user.keytab.username",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-common/login.user.keytab.path",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-common/kerberos.expire.time",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-alert/enterprise.wechat.enable",
"widget": {
"type": "toggle"
}
},
{
"config": "dolphin-alert/enterprise.wechat.corp.id",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-alert/enterprise.wechat.secret",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-alert/enterprise.wechat.agent.id",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-alert/enterprise.wechat.users",
"widget": {
"type": "text-field"
}
}
]
}
}

2
ambari_plugin/statcks/DOLPHIN/metainfo.xml
View File

@ -20,7 +20,7 @@
<services>
<service>
<name>DOLPHIN</name>
<extends>common-services/DOLPHIN/2.0.0</extends>
<extends>common-services/DOLPHIN/1.3.0</extends>
</service>
</services>
</metainfo>