mirror of
https://gitee.com/dolphinscheduler/DolphinScheduler.git
synced 2024-12-04 05:09:48 +08:00
add postgre performance monitor (#1137)
* update english documents * refactor zk client * update documents * update zkclient * update zkclient * update documents * add architecture-design * change i18n * update i18n * update english documents * add architecture-design * update english documents * update en-US documents * add architecture-design * update demo site * add mybatis plus model * modify mybatisplus * modify mybatisplus * change interface by mybatisplus * add unit test * refactor dao interface. * add unit test for dao... * add unit test for dao... * add unit test for dao... * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml * update some dao bugs * update for some bugs * update some bugs * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml * update * update * add multiply settings for application.yml * add multiply settings for application.yml * revert * update configuration settings in task record dao... * change application_master to application-master * change application_master to application-master * update application.yml to application.properties * revert * revert * add properties * add properties * revert * revert * add api start up.. add alert send try catch * update dao info level * fix bug: task cannot submit when recovery failover * fix bug: task cannot submit when recovery failover * merge from dev-db * revert * revert * fix bug: get process definition list failed. * fix bug: process instance interval is error * revert * revert * update * support stop submit success tasks * update kill process * update for stop process * update for stop process * add some logs for stop process * update for small bug. * add check strategy before submit task * revert * update * update * revert * wait task instance exists if null. * revert * update * change desc to description. * add check user and definitions function when delete tenant * update * change desc to description. * change desc to description. * change desc to description. * remove check resources when delete tenant * change desc to description. * change mybatisplus version to 3.2.0 * update * change the notice to apache. * update * update postgre sql * fix bug: phone can be empty. * fix bug: postgre test error. * update create table for postgre quartz * fix some bugs about postgre. * update create table for postgre quartz * add postgre db performance monitor * add postgre performance monitor * update performance monitor * revert * revert
This commit is contained in:
bao liang
qiaozhanwei
parent
aa575d84d6
commit
9545cb2008
@ -16,17 +16,21 @@
|
||||
*/
|
||||
package org.apache.dolphinscheduler.dao;
|
||||
|
||||
import com.alibaba.druid.pool.DruidDataSource;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.DbType;
|
||||
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
|
||||
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
|
||||
import org.apache.commons.configuration.Configuration;
|
||||
import org.apache.commons.configuration.ConfigurationException;
|
||||
import org.apache.commons.configuration.PropertiesConfiguration;
|
||||
import org.apache.dolphinscheduler.dao.utils.MysqlPerformance;
|
||||
import org.apache.dolphinscheduler.dao.utils.PostgrePerformance;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.sql.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
@ -52,29 +56,41 @@ public class MonitorDBDao {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* create connection
|
||||
* @return
|
||||
*/
|
||||
private static Connection getConn() {
|
||||
String url = conf.getString(Constants.SPRING_DATASOURCE_URL);
|
||||
String username = conf.getString(Constants.SPRING_DATASOURCE_USERNAME);
|
||||
String password = conf.getString(Constants.SPRING_DATASOURCE_PASSWORD);
|
||||
public static MonitorRecord getCurrentDbPerformance(){
|
||||
MonitorRecord monitorRecord = null;
|
||||
Connection conn = null;
|
||||
try {
|
||||
//classloader,load driver
|
||||
Class.forName(Constants.JDBC_MYSQL_CLASS_NAME);
|
||||
conn = DriverManager.getConnection(url, username, password);
|
||||
} catch (ClassNotFoundException e) {
|
||||
logger.error("ClassNotFoundException ", e);
|
||||
} catch (SQLException e) {
|
||||
logger.error("SQLException ", e);
|
||||
DruidDataSource dataSource = null;
|
||||
try{
|
||||
dataSource = ConnectionFactory.getDataSource();
|
||||
dataSource.setInitialSize(2);
|
||||
dataSource.setMinIdle(2);
|
||||
dataSource.setMaxActive(2);
|
||||
conn = dataSource.getConnection();
|
||||
if(conn == null){
|
||||
return monitorRecord;
|
||||
}
|
||||
if(conf.getString(Constants.SPRING_DATASOURCE_DRIVER_CLASS_NAME).contains(DbType.MYSQL.toString().toLowerCase())) {
|
||||
return new MysqlPerformance().getMonitorRecord(conn);
|
||||
} else if(conf.getString(Constants.SPRING_DATASOURCE_DRIVER_CLASS_NAME).contains(DbType.POSTGRESQL.toString().toLowerCase())){
|
||||
return new PostgrePerformance().getMonitorRecord(conn);
|
||||
}
|
||||
}catch (Exception e) {
|
||||
logger.error("SQLException " + e);
|
||||
}finally {
|
||||
try {
|
||||
if (conn != null) {
|
||||
conn.close();
|
||||
}
|
||||
if(dataSource != null){
|
||||
dataSource.close();
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
logger.error("SQLException ", e);
|
||||
}
|
||||
}
|
||||
return conn;
|
||||
return monitorRecord;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* query database state
|
||||
* @return
|
||||
@ -82,69 +98,10 @@ public class MonitorDBDao {
|
||||
public static List<MonitorRecord> queryDatabaseState() {
|
||||
List<MonitorRecord> list = new ArrayList<>(1);
|
||||
|
||||
Connection conn = null;
|
||||
Statement pstmt = null;
|
||||
long maxConnections = 0;
|
||||
long maxUsedConnections = 0;
|
||||
long threadsConnections = 0;
|
||||
long threadsRunningConnections = 0;
|
||||
//mysql running state
|
||||
int state = 1;
|
||||
|
||||
|
||||
MonitorRecord monitorRecord = new MonitorRecord();
|
||||
try {
|
||||
conn = getConn();
|
||||
if(conn == null){
|
||||
return list;
|
||||
}
|
||||
|
||||
pstmt = conn.createStatement();
|
||||
|
||||
ResultSet rs1 = pstmt.executeQuery("show global variables");
|
||||
while(rs1.next()){
|
||||
if(rs1.getString(VARIABLE_NAME).toUpperCase().equals("MAX_CONNECTIONS")){
|
||||
maxConnections= Long.parseLong(rs1.getString("value"));
|
||||
}
|
||||
}
|
||||
|
||||
ResultSet rs2 = pstmt.executeQuery("show global status");
|
||||
while(rs2.next()){
|
||||
if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("MAX_USED_CONNECTIONS")){
|
||||
maxUsedConnections = Long.parseLong(rs2.getString("value"));
|
||||
}else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_CONNECTED")){
|
||||
threadsConnections = Long.parseLong(rs2.getString("value"));
|
||||
}else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_RUNNING")){
|
||||
threadsRunningConnections= Long.parseLong(rs2.getString("value"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (SQLException e) {
|
||||
logger.error("SQLException ", e);
|
||||
state = 0;
|
||||
}finally {
|
||||
try {
|
||||
if(pstmt != null) {
|
||||
pstmt.close();
|
||||
}
|
||||
if(conn != null){
|
||||
conn.close();
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
logger.error("SQLException ", e);
|
||||
}
|
||||
MonitorRecord monitorRecord = getCurrentDbPerformance();
|
||||
if(monitorRecord != null){
|
||||
list.add(monitorRecord);
|
||||
}
|
||||
|
||||
monitorRecord.setDate(new Date());
|
||||
monitorRecord.setMaxConnections(maxConnections);
|
||||
monitorRecord.setMaxUsedConnections(maxUsedConnections);
|
||||
monitorRecord.setThreadsConnections(threadsConnections);
|
||||
monitorRecord.setThreadsRunningConnections(threadsRunningConnections);
|
||||
monitorRecord.setState(state);
|
||||
|
||||
list.add(monitorRecord);
|
||||
|
||||
return list;
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,6 +16,9 @@
|
||||
*/
|
||||
package org.apache.dolphinscheduler.dao.entity;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.DbType;
|
||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
@ -23,10 +26,12 @@ import java.util.Date;
|
||||
*/
|
||||
public class MonitorRecord {
|
||||
|
||||
private DbType dbType;
|
||||
|
||||
/**
|
||||
* is normal or not , 1: normal
|
||||
*/
|
||||
private int state;
|
||||
private Flag state;
|
||||
|
||||
/**
|
||||
* max connections
|
||||
@ -53,11 +58,11 @@ public class MonitorRecord {
|
||||
*/
|
||||
private Date date;
|
||||
|
||||
public int getState() {
|
||||
public Flag getState() {
|
||||
return state;
|
||||
}
|
||||
|
||||
public void setState(int state) {
|
||||
public void setState(Flag state) {
|
||||
this.state = state;
|
||||
}
|
||||
|
||||
@ -105,6 +110,7 @@ public class MonitorRecord {
|
||||
public String toString() {
|
||||
return "MonitorRecord{" +
|
||||
"state=" + state +
|
||||
", dbType=" + dbType +
|
||||
", maxConnections=" + maxConnections +
|
||||
", maxUsedConnections=" + maxUsedConnections +
|
||||
", threadsConnections=" + threadsConnections +
|
||||
@ -112,4 +118,12 @@ public class MonitorRecord {
|
||||
", date=" + date +
|
||||
'}';
|
||||
}
|
||||
|
||||
public DbType getDbType() {
|
||||
return dbType;
|
||||
}
|
||||
|
||||
public void setDbType(DbType dbType) {
|
||||
this.dbType = dbType;
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.dolphinscheduler.dao.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
|
||||
|
||||
import java.sql.Connection;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public abstract class BaseDBPerformance {
|
||||
|
||||
|
||||
/**
|
||||
* return the current database performance
|
||||
* @return
|
||||
*/
|
||||
protected abstract MonitorRecord getMonitorRecord(Connection conn);
|
||||
|
||||
}
|
||||
@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.dolphinscheduler.dao.utils;
|
||||
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.DbType;
|
||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
import org.apache.dolphinscheduler.dao.MonitorDBDao;
|
||||
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.Date;
|
||||
|
||||
import static org.apache.dolphinscheduler.dao.MonitorDBDao.VARIABLE_NAME;
|
||||
|
||||
public class MysqlPerformance extends BaseDBPerformance{
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class);
|
||||
|
||||
|
||||
@Override
|
||||
public MonitorRecord getMonitorRecord(Connection conn) {
|
||||
MonitorRecord monitorRecord = new MonitorRecord();
|
||||
monitorRecord.setDate(new Date());
|
||||
monitorRecord.setDbType(DbType.MYSQL);
|
||||
monitorRecord.setState(Flag.YES);
|
||||
Statement pstmt= null;
|
||||
try{
|
||||
pstmt = conn.createStatement();
|
||||
|
||||
ResultSet rs1 = pstmt.executeQuery("show global variables");
|
||||
while(rs1.next()){
|
||||
if(rs1.getString(VARIABLE_NAME).toUpperCase().equals("MAX_CONNECTIONS")){
|
||||
monitorRecord.setMaxConnections( Long.parseLong(rs1.getString("value")));
|
||||
}
|
||||
}
|
||||
|
||||
ResultSet rs2 = pstmt.executeQuery("show global status");
|
||||
while(rs2.next()){
|
||||
if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("MAX_USED_CONNECTIONS")){
|
||||
monitorRecord.setMaxUsedConnections(Long.parseLong(rs2.getString("value")));
|
||||
}else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_CONNECTED")){
|
||||
monitorRecord.setThreadsConnections(Long.parseLong(rs2.getString("value")));
|
||||
}else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_RUNNING")){
|
||||
monitorRecord.setThreadsRunningConnections(Long.parseLong(rs2.getString("value")));
|
||||
}
|
||||
}
|
||||
}catch (Exception e) {
|
||||
monitorRecord.setState(Flag.NO);
|
||||
logger.error("SQLException " + e);
|
||||
}finally {
|
||||
try {
|
||||
if (pstmt != null) {
|
||||
pstmt.close();
|
||||
}
|
||||
}catch (SQLException e) {
|
||||
logger.error("SQLException ", e);
|
||||
}
|
||||
}
|
||||
return monitorRecord;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.dolphinscheduler.dao.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.DbType;
|
||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
import org.apache.dolphinscheduler.dao.MonitorDBDao;
|
||||
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.Date;
|
||||
|
||||
public class PostgrePerformance extends BaseDBPerformance {
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class);
|
||||
|
||||
@Override
|
||||
public MonitorRecord getMonitorRecord(Connection conn) {
|
||||
MonitorRecord monitorRecord = new MonitorRecord();
|
||||
monitorRecord.setDate(new Date());
|
||||
monitorRecord.setState(Flag.YES);
|
||||
monitorRecord.setDbType(DbType.POSTGRESQL);
|
||||
Statement pstmt= null;
|
||||
try{
|
||||
pstmt = conn.createStatement();
|
||||
ResultSet rs1 = pstmt.executeQuery("select count(*) from pg_stat_activity;");
|
||||
while(rs1.next()){
|
||||
monitorRecord.setThreadsConnections(rs1.getInt("count"));
|
||||
break;
|
||||
}
|
||||
|
||||
ResultSet rs2 = pstmt.executeQuery("show max_connections");
|
||||
while(rs2.next()){
|
||||
monitorRecord.setMaxConnections( rs2.getInt("max_connections"));
|
||||
break;
|
||||
}
|
||||
|
||||
ResultSet rs3 = pstmt.executeQuery("select count(*) from pg_stat_activity pg where pg.state = 'active';");
|
||||
while(rs3.next()){
|
||||
monitorRecord.setThreadsRunningConnections(rs3.getInt("count"));
|
||||
break;
|
||||
}
|
||||
}catch (Exception e) {
|
||||
monitorRecord.setState(Flag.NO);
|
||||
logger.error("SQLException " + e);
|
||||
}finally {
|
||||
try {
|
||||
if (pstmt != null) {
|
||||
pstmt.close();
|
||||
}
|
||||
}catch (SQLException e) {
|
||||
logger.error("SQLException ", e);
|
||||
}
|
||||
}
|
||||
return monitorRecord;
|
||||
}
|
||||
}
|
||||
@ -1,8 +1,9 @@
|
||||
# mysql
|
||||
# url=jdbc:postgresql://192.168.220.154:5432/dolphinscheduler
|
||||
# base spring data source configuration
|
||||
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
|
||||
# postgre
|
||||
#spring.datasource.driver-class-name=org.postgresql.Driver
|
||||
#spring.datasource.url=jdbc:postgresql://192.168.220.154:5432/dolphinscheduler
|
||||
# mysql
|
||||
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
|
||||
spring.datasource.url=jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
|
||||
spring.datasource.username=root
|
||||
@ -83,7 +84,7 @@ mybatis-plus.configuration.call-setters-on-nulls=true
|
||||
mybatis-plus.configuration.jdbc-type-for-null=null
|
||||
|
||||
# data quality analysis is not currently in use. please ignore the following configuration
|
||||
# task record flag
|
||||
# task record
|
||||
task.record.flag=false
|
||||
task.record.datasource.url=jdbc:mysql://192.168.xx.xx:3306/etl?characterEncoding=UTF-8
|
||||
task.record.datasource.username=xx
|
||||
|
||||
Loading…
Reference in New Issue
Block a user