add postgre performance monitor (#1137)

* update english documents

* refactor zk client

* update documents

* update zkclient

* update zkclient

* update documents

* add architecture-design

* change i18n

* update i18n

* update english documents

* add architecture-design

* update english documents

* update en-US documents

* add architecture-design

* update demo site

* add mybatis plus model

* modify mybatisplus

* modify mybatisplus

* change interface by mybatisplus

* add unit test

* refactor dao interface.

* add unit test for dao...

* add unit test for dao...

* add unit test for dao...

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml

* update some dao bugs

* update for some bugs

* update some bugs

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml

* update

* update

* add multiply settings for application.yml

* add multiply settings for application.yml

* revert

* update configuration settings in task record dao...

* change application_master to application-master

* change application_master to application-master

* update application.yml to application.properties

* revert

* revert

* add properties

* add properties

* revert

* revert

* add api start up..
add alert send try catch

* update dao info level

* fix bug: task cannot submit when recovery failover

* fix bug: task cannot submit when recovery failover

* merge from dev-db

* revert

* revert

* fix bug: get process definition list failed.

* fix bug: process instance interval is error

* revert

* revert

* update

* support stop submit success tasks

* update kill process

* update for stop process

* update for stop process

* add some logs for stop process

* update for small bug.

* add check strategy before submit task

* revert

* update

* update

* revert

* wait task instance exists if null.

* revert

* update

* change desc to description.

* add check user and definitions function when delete tenant

* update

* change desc to description.

* change desc to description.

* change desc to description.

* remove check resources when delete tenant

* change desc to description.

* change mybatisplus version to 3.2.0

* update

* change the notice to apache.

* update

* update postgre sql

* fix bug: phone can be empty.

* fix bug: postgre test error.

* update create table for postgre quartz

* fix some bugs about postgre.

* update create table for postgre quartz

* add postgre db performance monitor

* add postgre performance monitor

* update performance monitor

* revert

* revert
This commit is contained in:
bao liang 2019-10-31 18:52:20 +08:00 committed by qiaozhanwei
parent aa575d84d6
commit 9545cb2008
6 changed files with 254 additions and 88 deletions

View File

@ -16,17 +16,21 @@
*/ */
package org.apache.dolphinscheduler.dao; package org.apache.dolphinscheduler.dao;
import com.alibaba.druid.pool.DruidDataSource;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.dolphinscheduler.dao.utils.MysqlPerformance;
import org.apache.dolphinscheduler.dao.utils.PostgrePerformance;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.sql.*; import java.sql.*;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date;
import java.util.List; import java.util.List;
@ -52,29 +56,41 @@ public class MonitorDBDao {
} }
} }
public static MonitorRecord getCurrentDbPerformance(){
/** MonitorRecord monitorRecord = null;
* create connection
* @return
*/
private static Connection getConn() {
String url = conf.getString(Constants.SPRING_DATASOURCE_URL);
String username = conf.getString(Constants.SPRING_DATASOURCE_USERNAME);
String password = conf.getString(Constants.SPRING_DATASOURCE_PASSWORD);
Connection conn = null; Connection conn = null;
try { DruidDataSource dataSource = null;
//classloader,load driver try{
Class.forName(Constants.JDBC_MYSQL_CLASS_NAME); dataSource = ConnectionFactory.getDataSource();
conn = DriverManager.getConnection(url, username, password); dataSource.setInitialSize(2);
} catch (ClassNotFoundException e) { dataSource.setMinIdle(2);
logger.error("ClassNotFoundException ", e); dataSource.setMaxActive(2);
} catch (SQLException e) { conn = dataSource.getConnection();
logger.error("SQLException ", e); if(conn == null){
return monitorRecord;
}
if(conf.getString(Constants.SPRING_DATASOURCE_DRIVER_CLASS_NAME).contains(DbType.MYSQL.toString().toLowerCase())) {
return new MysqlPerformance().getMonitorRecord(conn);
} else if(conf.getString(Constants.SPRING_DATASOURCE_DRIVER_CLASS_NAME).contains(DbType.POSTGRESQL.toString().toLowerCase())){
return new PostgrePerformance().getMonitorRecord(conn);
}
}catch (Exception e) {
logger.error("SQLException " + e);
}finally {
try {
if (conn != null) {
conn.close();
}
if(dataSource != null){
dataSource.close();
}
} catch (SQLException e) {
logger.error("SQLException ", e);
}
} }
return conn; return monitorRecord;
} }
/** /**
* query database state * query database state
* @return * @return
@ -82,69 +98,10 @@ public class MonitorDBDao {
public static List<MonitorRecord> queryDatabaseState() { public static List<MonitorRecord> queryDatabaseState() {
List<MonitorRecord> list = new ArrayList<>(1); List<MonitorRecord> list = new ArrayList<>(1);
Connection conn = null; MonitorRecord monitorRecord = getCurrentDbPerformance();
Statement pstmt = null; if(monitorRecord != null){
long maxConnections = 0; list.add(monitorRecord);
long maxUsedConnections = 0;
long threadsConnections = 0;
long threadsRunningConnections = 0;
//mysql running state
int state = 1;
MonitorRecord monitorRecord = new MonitorRecord();
try {
conn = getConn();
if(conn == null){
return list;
}
pstmt = conn.createStatement();
ResultSet rs1 = pstmt.executeQuery("show global variables");
while(rs1.next()){
if(rs1.getString(VARIABLE_NAME).toUpperCase().equals("MAX_CONNECTIONS")){
maxConnections= Long.parseLong(rs1.getString("value"));
}
}
ResultSet rs2 = pstmt.executeQuery("show global status");
while(rs2.next()){
if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("MAX_USED_CONNECTIONS")){
maxUsedConnections = Long.parseLong(rs2.getString("value"));
}else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_CONNECTED")){
threadsConnections = Long.parseLong(rs2.getString("value"));
}else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_RUNNING")){
threadsRunningConnections= Long.parseLong(rs2.getString("value"));
}
}
} catch (SQLException e) {
logger.error("SQLException ", e);
state = 0;
}finally {
try {
if(pstmt != null) {
pstmt.close();
}
if(conn != null){
conn.close();
}
} catch (SQLException e) {
logger.error("SQLException ", e);
}
} }
monitorRecord.setDate(new Date());
monitorRecord.setMaxConnections(maxConnections);
monitorRecord.setMaxUsedConnections(maxUsedConnections);
monitorRecord.setThreadsConnections(threadsConnections);
monitorRecord.setThreadsRunningConnections(threadsRunningConnections);
monitorRecord.setState(state);
list.add(monitorRecord);
return list; return list;
} }
} }

View File

@ -16,6 +16,9 @@
*/ */
package org.apache.dolphinscheduler.dao.entity; package org.apache.dolphinscheduler.dao.entity;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.Flag;
import java.util.Date; import java.util.Date;
/** /**
@ -23,10 +26,12 @@ import java.util.Date;
*/ */
public class MonitorRecord { public class MonitorRecord {
private DbType dbType;
/** /**
* is normal or not , 1: normal * is normal or not , 1: normal
*/ */
private int state; private Flag state;
/** /**
* max connections * max connections
@ -53,11 +58,11 @@ public class MonitorRecord {
*/ */
private Date date; private Date date;
public int getState() { public Flag getState() {
return state; return state;
} }
public void setState(int state) { public void setState(Flag state) {
this.state = state; this.state = state;
} }
@ -105,6 +110,7 @@ public class MonitorRecord {
public String toString() { public String toString() {
return "MonitorRecord{" + return "MonitorRecord{" +
"state=" + state + "state=" + state +
", dbType=" + dbType +
", maxConnections=" + maxConnections + ", maxConnections=" + maxConnections +
", maxUsedConnections=" + maxUsedConnections + ", maxUsedConnections=" + maxUsedConnections +
", threadsConnections=" + threadsConnections + ", threadsConnections=" + threadsConnections +
@ -112,4 +118,12 @@ public class MonitorRecord {
", date=" + date + ", date=" + date +
'}'; '}';
} }
public DbType getDbType() {
return dbType;
}
public void setDbType(DbType dbType) {
this.dbType = dbType;
}
} }

View File

@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.utils;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import java.sql.Connection;
/**
*
*/
public abstract class BaseDBPerformance {
/**
* return the current database performance
* @return
*/
protected abstract MonitorRecord getMonitorRecord(Connection conn);
}

View File

@ -0,0 +1,83 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.utils;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.dao.MonitorDBDao;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Date;
import static org.apache.dolphinscheduler.dao.MonitorDBDao.VARIABLE_NAME;
public class MysqlPerformance extends BaseDBPerformance{
private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class);
@Override
public MonitorRecord getMonitorRecord(Connection conn) {
MonitorRecord monitorRecord = new MonitorRecord();
monitorRecord.setDate(new Date());
monitorRecord.setDbType(DbType.MYSQL);
monitorRecord.setState(Flag.YES);
Statement pstmt= null;
try{
pstmt = conn.createStatement();
ResultSet rs1 = pstmt.executeQuery("show global variables");
while(rs1.next()){
if(rs1.getString(VARIABLE_NAME).toUpperCase().equals("MAX_CONNECTIONS")){
monitorRecord.setMaxConnections( Long.parseLong(rs1.getString("value")));
}
}
ResultSet rs2 = pstmt.executeQuery("show global status");
while(rs2.next()){
if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("MAX_USED_CONNECTIONS")){
monitorRecord.setMaxUsedConnections(Long.parseLong(rs2.getString("value")));
}else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_CONNECTED")){
monitorRecord.setThreadsConnections(Long.parseLong(rs2.getString("value")));
}else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_RUNNING")){
monitorRecord.setThreadsRunningConnections(Long.parseLong(rs2.getString("value")));
}
}
}catch (Exception e) {
monitorRecord.setState(Flag.NO);
logger.error("SQLException " + e);
}finally {
try {
if (pstmt != null) {
pstmt.close();
}
}catch (SQLException e) {
logger.error("SQLException ", e);
}
}
return monitorRecord;
}
}

View File

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.utils;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.dao.MonitorDBDao;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Date;
public class PostgrePerformance extends BaseDBPerformance {
private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class);
@Override
public MonitorRecord getMonitorRecord(Connection conn) {
MonitorRecord monitorRecord = new MonitorRecord();
monitorRecord.setDate(new Date());
monitorRecord.setState(Flag.YES);
monitorRecord.setDbType(DbType.POSTGRESQL);
Statement pstmt= null;
try{
pstmt = conn.createStatement();
ResultSet rs1 = pstmt.executeQuery("select count(*) from pg_stat_activity;");
while(rs1.next()){
monitorRecord.setThreadsConnections(rs1.getInt("count"));
break;
}
ResultSet rs2 = pstmt.executeQuery("show max_connections");
while(rs2.next()){
monitorRecord.setMaxConnections( rs2.getInt("max_connections"));
break;
}
ResultSet rs3 = pstmt.executeQuery("select count(*) from pg_stat_activity pg where pg.state = 'active';");
while(rs3.next()){
monitorRecord.setThreadsRunningConnections(rs3.getInt("count"));
break;
}
}catch (Exception e) {
monitorRecord.setState(Flag.NO);
logger.error("SQLException " + e);
}finally {
try {
if (pstmt != null) {
pstmt.close();
}
}catch (SQLException e) {
logger.error("SQLException ", e);
}
}
return monitorRecord;
}
}

View File

@ -1,8 +1,9 @@
# mysql
# url=jdbc:postgresql://192.168.220.154:5432/dolphinscheduler
# base spring data source configuration # base spring data source configuration
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
# postgre
#spring.datasource.driver-class-name=org.postgresql.Driver #spring.datasource.driver-class-name=org.postgresql.Driver
#spring.datasource.url=jdbc:postgresql://192.168.220.154:5432/dolphinscheduler
# mysql
spring.datasource.driver-class-name=com.mysql.jdbc.Driver spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 spring.datasource.url=jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
spring.datasource.username=root spring.datasource.username=root
@ -83,7 +84,7 @@ mybatis-plus.configuration.call-setters-on-nulls=true
mybatis-plus.configuration.jdbc-type-for-null=null mybatis-plus.configuration.jdbc-type-for-null=null
# data quality analysis is not currently in use. please ignore the following configuration # data quality analysis is not currently in use. please ignore the following configuration
# task record flag # task record
task.record.flag=false task.record.flag=false
task.record.datasource.url=jdbc:mysql://192.168.xx.xx:3306/etl?characterEncoding=UTF-8 task.record.datasource.url=jdbc:mysql://192.168.xx.xx:3306/etl?characterEncoding=UTF-8
task.record.datasource.username=xx task.record.datasource.username=xx