test/DolphinScheduler
1
0
Fork 0
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git synced 2024-11-29 18:58:05 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[DSIP-25][Remote Logging] Split remote logging configuration (#15826)

Browse Source 
Co-authored-by: Rick Cheng <rickchengx@gmail.com>
This commit is contained in:
JohnHuang 2024-05-06 11:33:16 +08:00 committed by GitHub
parent fa6ea8ba7b
commit d218b021e7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 249 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml
View File

@ -52,6 +52,7 @@
<directory>${basedir}/../../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.yaml</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>

1
dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml
View File

@ -53,6 +53,7 @@
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.yaml</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>

33
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java
View File

@ -31,12 +31,18 @@ import lombok.extern.slf4j.Slf4j;
* This class will get the property by the priority of the following: env > jvm > properties.
*/
@Slf4j
public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate {
public class ImmutablePriorityPropertyDelegate implements IPropertyDelegate {
private static final Map<String, Optional<ConfigValue<String>>> configValueMap = new ConcurrentHashMap<>();
public ImmutablePriorityPropertyDelegate(String propertyAbsolutePath) {
super(propertyAbsolutePath);
private ImmutablePropertyDelegate immutablePropertyDelegate;
private ImmutableYamlDelegate immutableYamlDelegate;
public ImmutablePriorityPropertyDelegate(ImmutablePropertyDelegate immutablePropertyDelegate,
ImmutableYamlDelegate immutableYamlDelegate) {
this.immutablePropertyDelegate = immutablePropertyDelegate;
this.immutableYamlDelegate = immutableYamlDelegate;
}
@Override
@ -56,8 +62,14 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate
return value;
}
value = getConfigValueFromProperties(key);
if (value.isPresent()) {
log.debug("Get config value from properties, key: {} actualKey: {}, value: {}",
k, value.get().getActualKey(), value.get().getValue());
return value;
}
value = getConfigValueFromYaml(key);
value.ifPresent(
stringConfigValue -> log.debug("Get config value from properties, key: {} actualKey: {}, value: {}",
stringConfigValue -> log.debug("Get config value from yaml, key: {} actualKey: {}, value: {}",
k, stringConfigValue.getActualKey(), stringConfigValue.getValue()));
return value;
});
@ -76,7 +88,8 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate
@Override
public Set<String> getPropertyKeys() {
Set<String> propertyKeys = new HashSet<>();
propertyKeys.addAll(super.getPropertyKeys());
propertyKeys.addAll(this.immutablePropertyDelegate.getPropertyKeys());
propertyKeys.addAll(this.immutableYamlDelegate.getPropertyKeys());
propertyKeys.addAll(System.getProperties().stringPropertyNames());
propertyKeys.addAll(System.getenv().keySet());
return propertyKeys;
@ -104,7 +117,15 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate
}
private Optional<ConfigValue<String>> getConfigValueFromProperties(String key) {
String value = super.get(key);
String value = this.immutablePropertyDelegate.get(key);
if (value != null) {
return Optional.of(ConfigValue.fromProperties(key, value));
}
return Optional.empty();
}
private Optional<ConfigValue<String>> getConfigValueFromYaml(String key) {
String value = this.immutableYamlDelegate.get(key);
if (value != null) {
return Optional.of(ConfigValue.fromProperties(key, value));
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java
View File

@ -49,7 +49,7 @@ public class ImmutablePropertyDelegate implements IPropertyDelegate {
} catch (IOException e) {
log.error("Load property: {} error, please check if the file exist under classpath",
propertyAbsolutePath, e);
System.exit(1);
throw new RuntimeException(e);
}
}
printProperties();

82
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.config;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
import org.springframework.core.io.InputStreamResource;
@Slf4j
public class ImmutableYamlDelegate implements IPropertyDelegate {
private static final String REMOTE_LOGGING_YAML_NAME = "/remote-logging.yaml";
private final Properties properties;
public ImmutableYamlDelegate() {
this(REMOTE_LOGGING_YAML_NAME);
}
public ImmutableYamlDelegate(String... yamlAbsolutePath) {
properties = new Properties();
// read from classpath
for (String fileName : yamlAbsolutePath) {
try (InputStream fis = ImmutableYamlDelegate.class.getResourceAsStream(fileName)) {
YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean();
factory.setResources(new InputStreamResource(fis));
factory.afterPropertiesSet();
Properties subProperties = factory.getObject();
properties.putAll(subProperties);
} catch (IOException e) {
log.error("Load property: {} error, please check if the file exist under classpath",
yamlAbsolutePath, e);
throw new RuntimeException(e);
}
}
printProperties();
}
public ImmutableYamlDelegate(Properties properties) {
this.properties = properties;
}
@Override
public String get(String key) {
return properties.getProperty(key);
}
@Override
public String get(String key, String defaultValue) {
return properties.getProperty(key, defaultValue);
}
@Override
public Set<String> getPropertyKeys() {
return properties.stringPropertyNames();
}
private void printProperties() {
properties.forEach((k, v) -> log.debug("Get property {} -> {}", k, v));
}
}

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
View File

@ -35,6 +35,8 @@ public final class Constants {
*/
public static final String COMMON_PROPERTIES_PATH = "/common.properties";
public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml";
public static final String FORMAT_SS = "%s%s";
public static final String FORMAT_S_S = "%s/%s";
public static final String FORMAT_S_S_COLON = "%s:%s";
@ -683,9 +685,6 @@ public final class Constants {
public static final Integer QUERY_ALL_ON_WORKFLOW = 2;
public static final Integer QUERY_ALL_ON_TASK = 3;
/**
* remote logging
*/
public static final String REMOTE_LOGGING_ENABLE = "remote.logging.enable";
public static final String REMOTE_LOGGING_TARGET = "remote.logging.target";

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java
View File

@ -18,9 +18,11 @@
package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH;
import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH;
import org.apache.dolphinscheduler.common.config.IPropertyDelegate;
import org.apache.dolphinscheduler.common.config.ImmutablePriorityPropertyDelegate;
import org.apache.dolphinscheduler.common.config.ImmutablePropertyDelegate;
import org.apache.dolphinscheduler.common.config.ImmutableYamlDelegate;
import java.util.HashMap;
import java.util.Map;
@ -37,8 +39,10 @@ import com.google.common.base.Strings;
public class PropertyUtils {
// todo: add another implementation for zookeeper/etcd/consul/xx
private static final IPropertyDelegate propertyDelegate =
new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH);
private final ImmutablePriorityPropertyDelegate propertyDelegate =
new ImmutablePriorityPropertyDelegate(
new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH),
new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH));
public static String getString(String key) {
return propertyDelegate.get(key.trim());

61
dolphinscheduler-common/src/main/resources/remote-logging.yaml Normal file
View File

@ -0,0 +1,61 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
remote-logging:
# Whether to enable remote logging
enable: false
# if remote-logging.enable = true, set the target of remote logging
target: OSS
# if remote-logging.enable = true, set the log base directory
base.dir: logs
# if remote-logging.enable = true, set the number of threads to send logs to remote storage
thread.pool.size: 10
# required if you set remote-logging.target=OSS
oss:
# oss access key id, required if you set remote-logging.target=OSS
access.key.id: <access.key.id>
# oss access key secret, required if you set remote-logging.target=OSS
access.key.secret: <access.key.secret>
# oss bucket name, required if you set remote-logging.target=OSS
bucket.name: <bucket.name>
# oss endpoint, required if you set remote-logging.target=OSS
endpoint: <endpoint>
# required if you set remote-logging.target=S3
s3:
# s3 access key id, required if you set remote-logging.target=S3
access.key.id: <access.key.id>
# s3 access key secret, required if you set remote-logging.target=S3
access.key.secret: <access.key.secret>
# s3 bucket name, required if you set remote-logging.target=S3
bucket.name: <bucket.name>
# s3 endpoint, required if you set remote-logging.target=S3
endpoint: <endpoint>
# s3 region, required if you set remote-logging.target=S3
region: <region>
google.cloud.storage:
# the location of the google cloud credential, required if you set remote-logging.target=GCS
credential: /path/to/credential
# gcs bucket name, required if you set remote-logging.target=GCS
bucket.name: <your-bucket>
abs:
# abs account name, required if you set resource.storage.type=ABS
account.name: <your-account-name>
# abs account key, required if you set resource.storage.type=ABS
account.key: <your-account-key>
# abs container name, required if you set resource.storage.type=ABS
container.name: <your-container-name>

5
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java
View File

@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.config;
import static com.github.stefanbirkner.systemlambda.SystemLambda.withEnvironmentVariable;
import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH;
import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@ -26,7 +27,9 @@ import org.junit.jupiter.api.Test;
class ImmutablePriorityPropertyDelegateTest {
private final ImmutablePriorityPropertyDelegate immutablePriorityPropertyDelegate =
new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH);
new ImmutablePriorityPropertyDelegate(
new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH),
new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH));
@Test
void getOverrideFromEnv() throws Exception {

61
dolphinscheduler-common/src/test/resources/remote-logging.yaml Normal file
View File

@ -0,0 +1,61 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
remote-logging:
# Whether to enable remote logging
enable: false
# if remote-logging.enable = true, set the target of remote logging
target: OSS
# if remote-logging.enable = true, set the log base directory
base.dir: logs
# if remote-logging.enable = true, set the number of threads to send logs to remote storage
thread.pool.size: 10
# required if you set remote-logging.target=OSS
oss:
# oss access key id, required if you set remote-logging.target=OSS
access.key.id: <access.key.id>
# oss access key secret, required if you set remote-logging.target=OSS
access.key.secret: <access.key.secret>
# oss bucket name, required if you set remote-logging.target=OSS
bucket.name: <bucket.name>
# oss endpoint, required if you set remote-logging.target=OSS
endpoint: <endpoint>
# required if you set remote-logging.target=S3
s3:
# s3 access key id, required if you set remote-logging.target=S3
access.key.id: <access.key.id>
# s3 access key secret, required if you set remote-logging.target=S3
access.key.secret: <access.key.secret>
# s3 bucket name, required if you set remote-logging.target=S3
bucket.name: <bucket.name>
# s3 endpoint, required if you set remote-logging.target=S3
endpoint: <endpoint>
# s3 region, required if you set remote-logging.target=S3
region: <region>
google.cloud.storage:
# the location of the google cloud credential, required if you set remote-logging.target=GCS
credential: /path/to/credential
# gcs bucket name, required if you set remote-logging.target=GCS
bucket.name: <your-bucket>
abs:
# abs account name, required if you set resource.storage.type=ABS
account.name: <your-account-name>
# abs account key, required if you set resource.storage.type=ABS
account.key: <your-account-key>
# abs container name, required if you set resource.storage.type=ABS
container.name: <your-container-name>

1
dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml
View File

@ -52,6 +52,7 @@
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.yaml</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>

1
dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml
View File

@ -53,6 +53,7 @@
<directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.yaml</include>
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>