test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-12-05 05:38:30 +08:00
Code Issues Actions 10 Packages Projects Releases Wiki Activity

Merge branch 'dev' of https://github.com/apache/incubator-dolphinscheduler into dev

Browse Source
This commit is contained in:
break60 2020-03-24 09:57:27 +08:00
commit 1964d47bdb
7 changed files with 44 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.github/workflows/ci_e2e.yml vendored
View File

@ -49,7 +49,8 @@ jobs:
- name: Docker Run
run: |
VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`
docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -p 8888:8888 dolphinscheduler:$VERSION all
mkdir -p /tmp/logs
docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -v /tmp/logs:/opt/dolphinscheduler/logs -p 8888:8888 dolphinscheduler:$VERSION all
- name: Check Server Status
run: sh ./dockerfile/hooks/check
- name: Prepare e2e env
@ -65,7 +66,10 @@ jobs:
- name: Run e2e Test
run: cd ./e2e && mvn -B clean test
- name: Collect logs
run: |
mkdir -p ${LOG_DIR}
docker logs dolphinscheduler > ${LOG_DIR}/dolphinscheduler.txt
continue-on-error: true
if: failure()
uses: actions/upload-artifact@v1
with:
name: dslogs
path: /tmp/logs

7
.github/workflows/ci_ut.yml vendored
View File

@ -15,7 +15,11 @@
# limitations under the License.
#
on: ["pull_request", "push"]
on:
pull_request:
push:
branches:
- dev
env:
DOCKER_DIR: ./docker
LOG_DIR: /tmp/dolphinscheduler
@ -58,7 +62,6 @@ jobs:
export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g'
mvn test -B -Dmaven.test.skip=false
- name: Upload coverage report to codecov
if: github.event_name == 'pull_request'
run: |
CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash)
- name: Run SonarCloud Analysis

10
dockerfile/hooks/check
View File

@ -16,7 +16,7 @@
# limitations under the License.
#
echo "------ dolphinscheduler check - server - status -------"
sleep 20
sleep 60
server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
if [ $server_num -eq 5 ]
then
@ -25,3 +25,11 @@ else
echo "Server start failed "$server_num
exit 1
fi
ready=`curl http://127.0.0.1:8888/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l`
if [ $ready -eq 1 ]
then
echo "Servers is ready"
else
echo "Servers is not ready"
exit 1
fi

6
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
View File

@ -425,13 +425,13 @@ public class OSUtils {
* @return check memory and cpu usage
*/
public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory){
// judging usage
// system load average
double loadAverage = OSUtils.loadAverage();
//
// system available physical memory
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
if(loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory){
logger.warn("load or availablePhysicalMemorySize(G) is too high, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage);
logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage);
return false;
}else{
return true;

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java
View File

@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.sun.jna.platform.win32.WinBase.INVALID_HANDLE_VALUE;
import static com.sun.jna.platform.win32.WinBase.STILL_ACTIVE;
import static java.util.Objects.requireNonNull;
@ -742,7 +743,9 @@ public class ProcessImplForWin32 extends Process {
}
private static void closeHandle(WinNT.HANDLE handle) {
Kernel32Util.closeHandle(handle);
if (!handle.equals(INVALID_HANDLE_VALUE)) {
Kernel32Util.closeHandle(handle);
}
}
/**

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
View File

@ -34,7 +34,7 @@ public class WorkerConfig {
@Value("${worker.max.cpuload.avg}")
private int workerMaxCpuloadAvg;
@Value("${master.reserved.memory}")
@Value("${worker.reserved.memory}")
private double workerReservedMemory;
public int getWorkerExecThreads() {

26
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
View File

@ -363,20 +363,20 @@ public class SqlTask extends AbstractTask {
// is the timeout set
boolean timeoutFlag = taskProps.getTaskTimeoutStrategy() == TaskTimeoutStrategy.FAILED ||
taskProps.getTaskTimeoutStrategy() == TaskTimeoutStrategy.WARNFAILED;
try (PreparedStatement stmt = connection.prepareStatement(sqlBinds.getSql())) {
if(timeoutFlag){
stmt.setQueryTimeout(taskProps.getTaskTimeout());
}
Map<Integer, Property> params = sqlBinds.getParamsMap();
if(params != null) {
for (Map.Entry<Integer, Property> entry : params.entrySet()) {
Property prop = entry.getValue();
ParameterUtils.setInParameter(entry.getKey(), stmt, prop.getType(), prop.getValue());
}
}
logger.info("prepare statement replace sql : {} ", stmt);
return stmt;
// prepare statement
PreparedStatement stmt = connection.prepareStatement(sqlBinds.getSql());
if(timeoutFlag){
stmt.setQueryTimeout(taskProps.getTaskTimeout());
}
Map<Integer, Property> params = sqlBinds.getParamsMap();
if(params != null) {
for (Map.Entry<Integer, Property> entry : params.entrySet()) {
Property prop = entry.getValue();
ParameterUtils.setInParameter(entry.getKey(), stmt, prop.getType(), prop.getValue());
}
}
logger.info("prepare statement replace sql : {} ", stmt);
return stmt;
}
/**