Timezone variable support one digital time (#2513)

Support time zone variable like "-8:00","+8:00","8:00" Time zone variable like "-8:00" is illegal in time-zone ID ,so we mush transfer it to standard format
apache · Dec 19, 2019 · 11b7800 · 11b7800
1 parent 6815979
commit 11b7800
Show file tree

Hide file tree

Showing 8 changed files with 55 additions and 8 deletions.
diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java b/fe/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
@@ -328,8 +328,7 @@ private void checkJobProperties() throws UserException {
         if (ConnectContext.get() != null) {
             timezone = ConnectContext.get().getSessionVariable().getTimeZone();
         }
-        timezone = jobProperties.getOrDefault(LoadStmt.TIMEZONE, timezone);
-        TimeUtils.checkTimeZoneValid(timezone);
+        timezone = TimeUtils.checkTimeZoneValidAndStandardize(jobProperties.getOrDefault(LoadStmt.TIMEZONE, timezone));
     }
 
     private void checkDataSourceProperties() throws AnalysisException {

diff --git a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java
@@ -201,7 +201,8 @@ public static void checkProperties(Map<String, String> properties) throws DdlExc
         // time zone
         final String timezone = properties.get(TIMEZONE);
         if (timezone != null) {
-            TimeUtils.checkTimeZoneValid(timezone);
+            properties.put(TIMEZONE, TimeUtils.checkTimeZoneValidAndStandardize(
+                    properties.getOrDefault(LoadStmt.TIMEZONE, TimeUtils.DEFAULT_TIME_ZONE)));
         }
     }
 

diff --git a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java
@@ -70,7 +70,7 @@ public class TimeUtils {
                     + "((0?[1-9])|([1-2][0-9])|(30)))|(0?2[\\-\\/\\s]?((0?[1-9])|(1[0-9])|(2[0-8]))))))"
                     + "(\\s(((0?[0-9])|([1][0-9])|([2][0-3]))\\:([0-5]?[0-9])((\\s)|(\\:([0-5]?[0-9])))))?$");
 
-    private static final Pattern TIMEZONE_OFFSET_FORMAT_REG = Pattern.compile("^[+-]{1}\\d{2}\\:\\d{2}$");
+    private static final Pattern TIMEZONE_OFFSET_FORMAT_REG = Pattern.compile("^[+-]{0,1}\\d{1,2}\\:\\d{2}$");
 
     public static Date MIN_DATE = null;
     public static Date MAX_DATE = null;
@@ -235,8 +235,11 @@ public static long timeStringToLong(String timeStr) {
     }
 
     // Check if the time zone_value is valid
-    public static void checkTimeZoneValid(String value) throws DdlException {
+    public static String checkTimeZoneValidAndStandardize(String value) throws DdlException {
         try {
+            if (value == null) {
+                ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TIME_ZONE, "null");
+            }
             // match offset type, such as +08:00, -07:00
             Matcher matcher = TIMEZONE_OFFSET_FORMAT_REG.matcher(value);
             // it supports offset and region timezone type, "CST" use here is compatibility purposes.
@@ -245,6 +248,11 @@ public static void checkTimeZoneValid(String value) throws DdlException {
                 ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TIME_ZONE, value);
             }
             if (match) {
+                boolean postive = value.charAt(0) != '-';
+                value = (postive ? "+" : "-") + String.format("%02d:%02d",
+                        Integer.parseInt(value.replaceAll("[+-]", "").split(":")[0]),
+                        Integer.parseInt(value.replaceAll("[+-]", "").split(":")[1]));
+
                 // timezone offsets around the world extended from -12:00 to +14:00
                 int tz = Integer.parseInt(value.substring(1, 3)) * 100 + Integer.parseInt(value.substring(4, 6));
                 if (value.charAt(0) == '-' && tz > 1200) {
@@ -254,8 +262,10 @@ public static void checkTimeZoneValid(String value) throws DdlException {
                 }
             }
             ZoneId.of(value, timeZoneAliasMap);
+            return value;
         } catch (DateTimeException ex) {
             ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TIME_ZONE, value);
         }
+        throw new DdlException("Parse time zone " + value + " error");
     }
 }
diff --git a/fe/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/src/main/java/org/apache/doris/qe/VariableMgr.java
@@ -19,6 +19,7 @@
 
 import org.apache.doris.analysis.SetType;
 import org.apache.doris.analysis.SetVar;
+import org.apache.doris.analysis.StringLiteral;
 import org.apache.doris.analysis.SysVariableDesc;
 import org.apache.doris.catalog.Catalog;
 import org.apache.doris.catalog.Type;
@@ -215,7 +216,9 @@ public static void setVar(SessionVariable sessionVariable, SetVar setVar) throws
         checkUpdate(setVar, ctx.getFlag());
         // Check variable time_zone value is valid
         if (setVar.getVariable().toLowerCase().equals("time_zone")) {
-            TimeUtils.checkTimeZoneValid(setVar.getValue().getStringValue());
+            setVar = new SetVar(
+                    setVar.getType(), setVar.getVariable(),
+                    new StringLiteral(TimeUtils.checkTimeZoneValidAndStandardize(setVar.getValue().getStringValue())));
         }
 
         // To modify to default value.

diff --git a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java
@@ -160,8 +160,7 @@ private void setOptionalFromTSLPutRequest(TStreamLoadPutRequest request) throws
             strictMode = request.isStrictMode();
         }
         if (request.isSetTimezone()) {
-            timezone = request.getTimezone();
-            TimeUtils.checkTimeZoneValid(timezone);
+            timezone = TimeUtils.checkTimeZoneValidAndStandardize(request.getTimezone());
         }
         if (request.isSetExecMemLimit()) {
             execMemLimit = request.getExecMemLimit();

diff --git a/fe/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java b/fe/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java
@@ -96,6 +96,7 @@ public void testAnalyze(@Injectable Analyzer analyzer) throws UserException {
         String topicName = "topic1";
         String serverAddress = "127.0.0.1:8080";
         String kafkaPartitionString = "1,2,3";
+        String timeZone = "8:00";
         List<String> partitionNameString = Lists.newArrayList();
         partitionNameString.add("p1");
         PartitionNames partitionNames = new PartitionNames(partitionNameString);
@@ -108,6 +109,7 @@ public void testAnalyze(@Injectable Analyzer analyzer) throws UserException {
         loadPropertyList.add(partitionNames);
         Map<String, String> properties = Maps.newHashMap();
         properties.put(CreateRoutineLoadStmt.DESIRED_CONCURRENT_NUMBER_PROPERTY, "2");
+        properties.put(LoadStmt.TIMEZONE, timeZone);
         String typeName = LoadDataSourceType.KAFKA.name();
         Map<String, String> customProperties = Maps.newHashMap();
 
@@ -134,6 +136,7 @@ public void analyze(Analyzer analyzer1) {
         Assert.assertEquals(0, createRoutineLoadStmt.getMaxErrorNum());
         Assert.assertEquals(serverAddress, createRoutineLoadStmt.getKafkaBrokerList());
         Assert.assertEquals(topicName, createRoutineLoadStmt.getKafkaTopic());
+        Assert.assertEquals("+08:00", createRoutineLoadStmt.getTimezone());
     }
 
 }
diff --git a/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java b/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java
@@ -22,6 +22,7 @@
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.common.AnalysisException;
 
+import org.apache.doris.common.DdlException;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -134,4 +135,23 @@ public void testDateTrans() throws AnalysisException {
         Assert.assertEquals(20150301120000L, datetime.getRealValue());
     }
 
+    @Test
+    public void testTimezone() throws AnalysisException {
+        try {
+            Assert.assertEquals("CST", TimeUtils.checkTimeZoneValidAndStandardize("CST"));
+            Assert.assertEquals("+08:00", TimeUtils.checkTimeZoneValidAndStandardize("+08:00"));
+            Assert.assertEquals("+08:00", TimeUtils.checkTimeZoneValidAndStandardize("+8:00"));
+            Assert.assertEquals("-08:00", TimeUtils.checkTimeZoneValidAndStandardize("-8:00"));
+            Assert.assertEquals("+08:00", TimeUtils.checkTimeZoneValidAndStandardize("8:00"));
+        } catch (DdlException ex) {
+            Assert.fail();
+        }
+        try {
+            TimeUtils.checkTimeZoneValidAndStandardize("FOO");
+            Assert.fail();
+        } catch (DdlException ex) {
+            Assert.assertTrue(ex.getMessage().contains("Unknown or incorrect time zone: 'FOO'"));
+        }
+    }
+
 }
diff --git a/fe/src/test/java/org/apache/doris/qe/VariableMgrTest.java b/fe/src/test/java/org/apache/doris/qe/VariableMgrTest.java
@@ -124,6 +124,18 @@ public void testNormal() throws IllegalAccessException, DdlException, NoSuchFiel
         VariableMgr.setVar(var, setVar4);
         Assert.assertEquals(2L, var.getSqlMode());
 
+        // Test checkTimeZoneValidAndStandardize
+        SetVar setVar5 = new SetVar(SetType.GLOBAL, "time_zone", new StringLiteral("+8:00"));
+        VariableMgr.setVar(var, setVar5);
+        Assert.assertEquals("+08:00", VariableMgr.newSessionVariable().getTimeZone());
+
+        SetVar setVar6 = new SetVar(SetType.GLOBAL, "time_zone", new StringLiteral("8:00"));
+        VariableMgr.setVar(var, setVar6);
+        Assert.assertEquals("+08:00", VariableMgr.newSessionVariable().getTimeZone());
+
+        SetVar setVar7 = new SetVar(SetType.GLOBAL, "time_zone", new StringLiteral("-8:00"));
+        VariableMgr.setVar(var, setVar7);
+        Assert.assertEquals("-08:00", VariableMgr.newSessionVariable().getTimeZone());
     }
 
     @Test(expected = DdlException.class)