From 83e88c4bc6cc3081dd6c99663d06fe32bb4a8e29 Mon Sep 17 00:00:00 2001 From: ronyang1985 Date: Mon, 21 Feb 2022 11:14:39 +0800 Subject: [PATCH] [Fix-8389][Common] Fix oshi compatibility issues cause it fail to boot on m1 mac (#8390) --- .../common/utils/OSUtils.java | 16 ++++++++-- .../dolphinscheduler/common/os/OshiTest.java | 21 +++++++++----- dolphinscheduler-dist/release-docs/LICENSE | 6 ++-- .../licenses/LICENSE-jna-platform.txt | 16 ++++++---- .../release-docs/licenses/LICENSE-jna.txt | 16 ++++++---- .../licenses/LICENSE-oshi-core.txt | 29 +++++++++++-------- pom.xml | 16 +++++++++- tools/dependencies/known-dependencies.txt | 6 ++-- 8 files changed, 85 insertions(+), 41 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index e0b32ad54b..a4e79236ca 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -63,6 +63,9 @@ public class OSUtils { public static final double NEGATIVE_ONE = -1; private static final HardwareAbstractionLayer hal = SI.getHardware(); + private static long[] prevTicks = new long[CentralProcessor.TickType.values().length]; + private static long prevTickTime = 0L; + private static double cpuUsage = 0.0D; private OSUtils() { throw new UnsupportedOperationException("Construct OSUtils"); @@ -117,7 +120,7 @@ public class OSUtils { loadAverage = osBean.getSystemLoadAverage(); } catch (Exception e) { logger.error("get operation system load average exception, try another method ", e); - loadAverage = hal.getProcessor().getSystemLoadAverage(); + loadAverage = hal.getProcessor().getSystemLoadAverage(1)[0]; if (Double.isNaN(loadAverage)) { return NEGATIVE_ONE; } @@ -134,7 +137,16 @@ public class OSUtils { */ public static double cpuUsage() { CentralProcessor processor = hal.getProcessor(); - double cpuUsage = processor.getSystemCpuLoad(); + + // Check if > ~ 0.95 seconds since last tick count. + long now = System.currentTimeMillis(); + if (now - prevTickTime > 950) { + // Enough time has elapsed. + cpuUsage = processor.getSystemCpuLoadBetweenTicks(prevTicks); + prevTickTime = System.currentTimeMillis(); + prevTicks = processor.getSystemCpuLoadTicks(); + } + if (Double.isNaN(cpuUsage)) { return NEGATIVE_ONE; } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OshiTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OshiTest.java index cfc069f9bb..1d17541b81 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OshiTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OshiTest.java @@ -51,7 +51,7 @@ public class OshiTest { logger.info("Checking CPU..."); - printCpu(hal.getProcessor()); + printCpu(si); } @@ -64,18 +64,21 @@ public class OshiTest { } - private static void printCpu(CentralProcessor processor) { - logger.info(String.format("CPU load: %.1f%% (OS MXBean)%n", processor.getSystemCpuLoad() * 100));//CPU load: 24.9% (OS MXBean) - logger.info("CPU load averages : {}", processor.getSystemLoadAverage());//CPU load averages : 1.5234375 + private static void printCpu(SystemInfo si) { + CentralProcessor processor = si.getHardware().getProcessor(); + long[] systemCpuLoadTicks = processor.getSystemCpuLoadTicks(); + Util.sleep(1000); + logger.info(String.format("CPU load: %.1f%% (OS MXBean)%n", processor.getSystemCpuLoadBetweenTicks(systemCpuLoadTicks) * 100));//CPU load: 24.9% (OS MXBean) + logger.info("CPU load averages : {}", processor.getSystemLoadAverage(1)[0]);//CPU load averages : 1.5234375 - logger.info("Uptime: " + FormatUtil.formatElapsedSecs(processor.getSystemUptime())); + logger.info("Uptime: " + FormatUtil.formatElapsedSecs(si.getOperatingSystem().getSystemUptime())); logger.info("Context Switches/Interrupts: " + processor.getContextSwitches() + " / " + processor.getInterrupts()); long[] prevTicks = processor.getSystemCpuLoadTicks(); logger.info("CPU, IOWait, and IRQ ticks @ 0 sec:" + Arrays.toString(prevTicks)); - //Wait a second... + //Wait a second... Util.sleep(1000); long[] ticks = processor.getSystemCpuLoadTicks(); logger.info("CPU, IOWait, and IRQ ticks @ 1 sec:" + Arrays.toString(ticks)); @@ -93,7 +96,7 @@ public class OshiTest { "User: %.1f%% Nice: %.1f%% System: %.1f%% Idle: %.1f%% IOwait: %.1f%% IRQ: %.1f%% SoftIRQ: %.1f%% Steal: %.1f%%%n", 100d * user / totalCpu, 100d * nice / totalCpu, 100d * sys / totalCpu, 100d * idle / totalCpu, 100d * iowait / totalCpu, 100d * irq / totalCpu, 100d * softirq / totalCpu, 100d * steal / totalCpu)); - logger.info(String.format("CPU load: %.1f%% (counting ticks)%n", processor.getSystemCpuLoadBetweenTicks() * 100)); + logger.info(String.format("CPU load: %.1f%% (counting ticks)%n", processor.getSystemCpuLoadBetweenTicks(prevTicks) * 100)); @@ -103,7 +106,9 @@ public class OshiTest { + (loadAverage[2] < 0 ? " N/A" : String.format(" %.2f", loadAverage[2]))); // per core CPU StringBuilder procCpu = new StringBuilder("CPU load per processor:"); - double[] load = processor.getProcessorCpuLoadBetweenTicks(); + long[][] processorCpuLoadTicks = processor.getProcessorCpuLoadTicks(); + Util.sleep(1000); + double[] load = processor.getProcessorCpuLoadBetweenTicks(processorCpuLoadTicks); for (double avg : load) { procCpu.append(String.format(" %.1f%%", avg * 100)); } diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 642bee488b..c989a964ee 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -319,8 +319,8 @@ The text of each license is also included at licenses/LICENSE-[project].txt. jetty-util-ajax 9.4.44.v20210927: https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-util-ajax/9.4.44.v20210927, Apache 2.0 and EPL 1.0 jetty-webapp 9.4.44.v20210927: https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-webapp/9.4.44.v20210927, Apache 2.0 and EPL 1.0 jetty-xml 9.4.44.v20210927: https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-xml/9.4.44.v20210927, Apache 2.0 and EPL 1.0 - jna 4.5.2: https://mvnrepository.com/artifact/net.java.dev.jna/jna/4.5.2, Apache 2.0 and LGPL 2.1 - jna-platform 4.5.2: https://mvnrepository.com/artifact/net.java.dev.jna/jna-platform/4.5.2, Apache 2.0 and LGPL 2.1 + jna 5.10.0: https://mvnrepository.com/artifact/net.java.dev.jna/jna/5.10.0, Apache 2.0 and LGPL 2.1 + jna-platform 5.10.0: https://mvnrepository.com/artifact/net.java.dev.jna/jna-platform/5.10.0, Apache 2.0 and LGPL 2.1 joda-time 2.10.13: https://github.com/JodaOrg/joda-time, Apache 2.0 jpam 1.1: https://mvnrepository.com/artifact/net.sf.jpam/jpam/1.1, Apache 2.0 jsqlparser 2.1: https://github.com/JSQLParser/JSqlParser, Apache 2.0 or LGPL 2.1 @@ -489,7 +489,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt. aspectjweaver 1.9.7:https://mvnrepository.com/artifact/org.aspectj/aspectjweaver/1.9.7, EPL 1.0 logback-classic 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-classic/1.2.3, EPL 1.0 and LGPL 2.1 logback-core 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-core/1.2.3, EPL 1.0 and LGPL 2.1 - oshi-core 3.9.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.9.1, EPL 1.0 h2-1.4.200 https://github.com/h2database/h2database/blob/master/LICENSE.txt, MPL 2.0 or EPL 1.0 ======================================================================== @@ -506,6 +505,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. checker-compat-qual 2.0.0 https://mvnrepository.com/artifact/org.checkerframework/checker-compat-qual/2.0.0, MIT + GPLv2 checker-qual 3.10.0 https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.10.0, MIT + GPLv2 Java-WebSocket 1.5.1: https://github.com/TooTallNate/Java-WebSocket MIT + oshi-core 6.1.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/6.1.1, MIT ======================================================================== MPL 1.1 licenses diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-jna-platform.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-jna-platform.txt index 19f51707d5..b456098518 100644 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-jna-platform.txt +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-jna-platform.txt @@ -1,9 +1,10 @@ -Java Native Access project (JNA) is dual-licensed under 2 -alternative Open Source/Free licenses: LGPL 2.1 or later and -Apache License 2.0. (starting with JNA version 4.0.0). +SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1 -You can freely decide which license you want to apply to -the project. +Java Native Access (JNA) is licensed under the LGPL, version 2.1 +or later, or (from version 4.0 onward) the Apache License, +version 2.0. + +You can freely decide which license you want to apply to the project. You may obtain a copy of the LGPL License at: @@ -19,4 +20,7 @@ http://www.apache.org/licenses/ A copy is also included in the downloadable source code package containing JNA, in file "AL2.0", under the same directory -as this file. \ No newline at end of file +as this file. + +Commercial support may be available, please e-mail +twall[at]users[dot]sf[dot]net. \ No newline at end of file diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-jna.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-jna.txt index 19f51707d5..b456098518 100644 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-jna.txt +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-jna.txt @@ -1,9 +1,10 @@ -Java Native Access project (JNA) is dual-licensed under 2 -alternative Open Source/Free licenses: LGPL 2.1 or later and -Apache License 2.0. (starting with JNA version 4.0.0). +SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1 -You can freely decide which license you want to apply to -the project. +Java Native Access (JNA) is licensed under the LGPL, version 2.1 +or later, or (from version 4.0 onward) the Apache License, +version 2.0. + +You can freely decide which license you want to apply to the project. You may obtain a copy of the LGPL License at: @@ -19,4 +20,7 @@ http://www.apache.org/licenses/ A copy is also included in the downloadable source code package containing JNA, in file "AL2.0", under the same directory -as this file. \ No newline at end of file +as this file. + +Commercial support may be available, please e-mail +twall[at]users[dot]sf[dot]net. \ No newline at end of file diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-oshi-core.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-oshi-core.txt index d0594b7db2..c938d017bb 100644 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-oshi-core.txt +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-oshi-core.txt @@ -1,16 +1,21 @@ -Oshi (https://github.com/oshi/oshi) +MIT License -Copyright (c) 2010 - 2018 The Oshi Project Team +Copyright (c) 2010 - 2021 The OSHI Project Contributors: https://github.com/oshi/oshi/graphs/contributors -All rights reserved. This program and the accompanying materials -are made available under the terms of the Eclipse Public License v1.0 -which accompanies this distribution, and is available at -http://www.eclipse.org/legal/epl-v10.html +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -Maintainers: -dblock[at]dblock[dot]org -widdis[at]gmail[dot]com -enrico.bianchi[at]gmail[dot]com +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -Contributors: -https://github.com/oshi/oshi/graphs/contributors \ No newline at end of file +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/pom.xml b/pom.xml index 8d3fb24923..9c0fc717b6 100644 --- a/pom.xml +++ b/pom.xml @@ -86,7 +86,7 @@ 42.2.5 2.1.0 2.4 - 3.9.1 + 6.1.1 0.1.52 6.1.0.jre8 0.238.1 @@ -792,6 +792,20 @@ com.github.oshi oshi-core ${oshi.core.version} + + + org.slf4j + slf4j-simple + + + org.junit.jupiter + junit-jupiter-api + + + org.hamcrest + hamcrest + + diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 6e9ecfbbf1..8cf740278d 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -134,8 +134,8 @@ jetty-util-ajax-9.4.44.v20210927.jar jetty-webapp-9.4.44.v20210927.jar jetty-xml-9.4.44.v20210927.jar jline-0.9.94.jar -jna-4.5.2.jar -jna-platform-4.5.2.jar +jna-5.10.0.jar +jna-platform-5.10.0.jar joda-time-2.10.13.jar jpam-1.1.jar jsch-0.1.42.jar @@ -166,7 +166,7 @@ mybatis-spring-2.0.2.jar netty-3.6.2.Final.jar netty-all-4.1.53.Final.jar opencsv-2.3.jar -oshi-core-3.9.1.jar +oshi-core-6.1.1.jar paranamer-2.3.jar parquet-hadoop-bundle-1.8.1.jar poi-4.1.2.jar