mirror of
https://gitee.com/dolphinscheduler/DolphinScheduler.git
synced 2024-11-30 03:08:01 +08:00
[Fix-13381][plugin] fix error while parsing hive load sql (#13378)
* [fix][task plugin][task sql] fix parsing error while using variables sometimes parsing exception happened if task sql contains a variable followed by \" or \' . for example: input sql: load inpath '/tmp/test_table/dt=${dt}' into table test_table partition(dt=${dt}) after replace: preparing : load inpath '/tmp/test_table/dt=? into table test_table partition(dt=?)
This commit is contained in:
parent
05df0269f6
commit
606b56403e
@ -4,6 +4,7 @@ This document records the incompatible updates between each version. You need to
|
||||
|
||||
## dev
|
||||
|
||||
* Change regex matching sql params in SQL task plugin ([#13378](https://github.com/apache/dolphinscheduler/pull/13378))
|
||||
* Remove the spark version of spark task ([#11860](https://github.com/apache/dolphinscheduler/pull/11860)).
|
||||
* Change the default unix shell executor from sh to bash ([#12180](https://github.com/apache/dolphinscheduler/pull/12180)).
|
||||
|
||||
|
@ -4,6 +4,7 @@
|
||||
|
||||
## dev
|
||||
|
||||
* 更新了SQL任务中用于匹配变量的正则表达式 ([#13378](https://github.com/apache/dolphinscheduler/pull/13378))
|
||||
* Remove the spark version of spark task ([#11860](https://github.com/apache/dolphinscheduler/pull/11860)).
|
||||
* Change the default unix shell executor from sh to bash ([#12180](https://github.com/apache/dolphinscheduler/pull/12180)).
|
||||
|
||||
|
@ -38,7 +38,7 @@ public abstract class AbstractTask {
|
||||
|
||||
protected final Logger log = LoggerFactory.getLogger(AbstractTask.class);
|
||||
|
||||
public String rgex = "['\"]*\\$\\{(.*?)\\}['\"]*";
|
||||
public String rgex = "['\"]\\$\\{(.*?)}['\"]|\\$\\{(.*?)}";
|
||||
|
||||
/**
|
||||
* varPool string
|
||||
|
@ -0,0 +1,96 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.sql;
|
||||
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
|
||||
import org.apache.dolphinscheduler.spi.enums.DbType;
|
||||
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class SqlTaskTest {
|
||||
|
||||
private SqlTask sqlTask;
|
||||
|
||||
@BeforeEach
|
||||
void setup() {
|
||||
DataSourceParameters parameters = new DataSourceParameters();
|
||||
parameters.setType(DbType.HIVE);
|
||||
parameters.setResourceType(ResourceType.DATASOURCE.name());
|
||||
|
||||
ResourceParametersHelper resourceParametersHelper = new ResourceParametersHelper();
|
||||
resourceParametersHelper.put(ResourceType.DATASOURCE, 1, parameters);
|
||||
|
||||
TaskExecutionContext ctx = new TaskExecutionContext();
|
||||
ctx.setResourceParametersHelper(resourceParametersHelper);
|
||||
ctx.setTaskParams("{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\"}");
|
||||
|
||||
sqlTask = new SqlTask(ctx);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReplacingSqlWithoutParams() {
|
||||
String querySql = "select 1";
|
||||
String expected = "select 1";
|
||||
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReplacingSqlWithDollarSymbol() {
|
||||
String querySql = "select concat(amount, '$') as price from product";
|
||||
String expected = "select concat(amount, '$') as price from product";
|
||||
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReplacingHiveLoadSql() {
|
||||
String hiveLoadSql = "load inpath '/tmp/test_table/dt=${dt}' into table test_table partition(dt=${dt})";
|
||||
String expected = "load inpath '/tmp/test_table/dt=?' into table test_table partition(dt=?)";
|
||||
Assertions.assertEquals(expected, hiveLoadSql.replaceAll(sqlTask.rgex, "?"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReplacingSelectSql() {
|
||||
String querySql = "select id from student where dt='${dt}'";
|
||||
String expected = "select id from student where dt=?";
|
||||
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
|
||||
|
||||
querySql = "select id from student where dt=\"${dt}\"";
|
||||
expected = "select id from student where dt=?";
|
||||
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
|
||||
|
||||
querySql = "select id from student where dt=${dt}";
|
||||
expected = "select id from student where dt=?";
|
||||
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
|
||||
|
||||
querySql = "select id from student where dt=${dt} and gender=1";
|
||||
expected = "select id from student where dt=? and gender=1";
|
||||
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReplacingSqlNonGreedy() {
|
||||
String querySql = "select id from student where year=${year} and month=${month} and gender=1";
|
||||
String expected = "select id from student where year=? and month=? and gender=1";
|
||||
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user