diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java index 5dd9f407c23316f0a52651446a6038b9b69c6f12..a6e311dd70697dac0c9ab5356ca7bc0b2023b774 100644 --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -536,6 +536,14 @@ public static boolean mkdir(FileSystem fs, Path f, boolean inheritPerms, Configu } } + public static Path makeAbsolute(FileSystem fileSystem, Path path) throws IOException { + if (path.isAbsolute()) { + return path; + } else { + return new Path(fileSystem.getWorkingDirectory(), path); + } + } + /** * Copies files between filesystems. */ diff --git a/common/src/java/org/apache/hadoop/hive/conf/SystemVariables.java b/common/src/java/org/apache/hadoop/hive/conf/SystemVariables.java index 9f59f11ca6459853b15ca80fa9751db934befc71..11d077ff5be873558e4806bdc89a81fd069c36a7 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/SystemVariables.java +++ b/common/src/java/org/apache/hadoop/hive/conf/SystemVariables.java @@ -21,12 +21,17 @@ import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.valcoersion.JavaIOTmpdirVariableCoercion; +import org.apache.hadoop.hive.conf.valcoersion.VariableCoercionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class SystemVariables { private static final Logger l4j = LoggerFactory.getLogger(SystemVariables.class); + private static final VariableCoercionSet COERCIONS = new VariableCoercionSet() + .add(new JavaIOTmpdirVariableCoercion()); + protected static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); public static final String ENV_PREFIX = "env:"; @@ -36,22 +41,34 @@ public static final String METACONF_PREFIX = "metaconf:"; public static final String SET_COLUMN_NAME = "set"; - protected String getSubstitute(Configuration conf, String var) { - String val = null; + protected String getSubstitute(Configuration conf, String variableName) { try { - if (var.startsWith(SYSTEM_PREFIX)) { - val = System.getProperty(var.substring(SYSTEM_PREFIX.length())); + if (variableName.startsWith(SYSTEM_PREFIX)) { + String propertyName = variableName.substring(SYSTEM_PREFIX.length()); + String originalValue = System.getProperty(propertyName); + return getCoerced(conf, variableName, originalValue); } } catch(SecurityException se) { l4j.warn("Unexpected SecurityException in Configuration", se); } - if (val == null && var.startsWith(ENV_PREFIX)) { - val = System.getenv(var.substring(ENV_PREFIX.length())); + + if (variableName.startsWith(ENV_PREFIX)) { + return System.getenv(variableName.substring(ENV_PREFIX.length())); + } + + if (conf != null && variableName.startsWith(HIVECONF_PREFIX)) { + return conf.get(variableName.substring(HIVECONF_PREFIX.length())); } - if (val == null && conf != null && var.startsWith(HIVECONF_PREFIX)) { - val = conf.get(var.substring(HIVECONF_PREFIX.length())); + + return null; + } + + public String getCoerced(Configuration configuration, String variableName, String originalValue) { + if (COERCIONS.contains(variableName)) { + return COERCIONS.get(variableName).getCoerced(configuration, originalValue); + } else { + return originalValue; } - return val; } public static boolean containsVar(String expr) { diff --git a/common/src/java/org/apache/hadoop/hive/conf/valcoersion/JavaIOTmpdirVariableCoercion.java b/common/src/java/org/apache/hadoop/hive/conf/valcoersion/JavaIOTmpdirVariableCoercion.java new file mode 100644 index 0000000000000000000000000000000000000000..1dcfd580f728856a2da4fc3750870f3b6edd8ed3 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/conf/valcoersion/JavaIOTmpdirVariableCoercion.java @@ -0,0 +1,61 @@ +package org.apache.hadoop.hive.conf.valcoersion; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.IOException; + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class JavaIOTmpdirVariableCoercion extends VariableCoercion { + private static final Log log = LogFactory.getLog(JavaIOTmpdirVariableCoercion.class); + private static final String NAME = "system:java.io.tmpdir"; + private static final FileSystem localFileSystem = new LocalFileSystem(); + + public JavaIOTmpdirVariableCoercion() { + super(NAME); + } + + private String coerce(String originalValue) { + if (originalValue == null || originalValue.isEmpty()) return originalValue; + + try { + Path originalPath = new Path(originalValue); + Path absolutePath = FileUtils.makeAbsolute(localFileSystem, originalPath); + return absolutePath.toString(); + } catch (IOException exception) { + log.warn(String.format("Unable to resolve 'java.io.tmpdir' for absolute path '%s'", originalValue)); + return originalValue; + } + } + + @Override + public String getCoerced(Configuration configuration, String originalValue) { + return coerce(originalValue); + } + + @Override + public String setCoerced(Configuration configuration, String originalValue) { + return coerce(originalValue); + } +} diff --git a/common/src/java/org/apache/hadoop/hive/conf/valcoersion/VariableCoercion.java b/common/src/java/org/apache/hadoop/hive/conf/valcoersion/VariableCoercion.java new file mode 100644 index 0000000000000000000000000000000000000000..39a67f24d48be5a154c101818ed743bad6dd6593 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/conf/valcoersion/VariableCoercion.java @@ -0,0 +1,32 @@ +package org.apache.hadoop.hive.conf.valcoersion; + +import org.apache.hadoop.conf.Configuration; + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public abstract class VariableCoercion { + private final String name; + + public VariableCoercion(String name) { + this.name = name; + } + + public String getName() { return this.name; } + public abstract String getCoerced(Configuration configuration, String originalValue); + public abstract String setCoerced(Configuration configuration, String originalValue); +} diff --git a/common/src/java/org/apache/hadoop/hive/conf/valcoersion/VariableCoercionSet.java b/common/src/java/org/apache/hadoop/hive/conf/valcoersion/VariableCoercionSet.java new file mode 100644 index 0000000000000000000000000000000000000000..0784424fa7bd1182f89e40b877d2d88c22fb1ad3 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/conf/valcoersion/VariableCoercionSet.java @@ -0,0 +1,38 @@ +package org.apache.hadoop.hive.conf.valcoersion; + +import java.util.HashMap; +import java.util.Map; + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class VariableCoercionSet { + private final Map coersions = new HashMap<>(); + + public VariableCoercionSet add(VariableCoercion variableCoercion) { + coersions.put(variableCoercion.getName(), variableCoercion); + return this; + } + + public boolean contains(String name) { + return coersions.containsKey(name); + } + + public VariableCoercion get(String name) { + return coersions.get(name); + } +} diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestValueCoersions.java b/common/src/test/org/apache/hadoop/hive/conf/TestValueCoersions.java new file mode 100644 index 0000000000000000000000000000000000000000..adf16178058b5faa282402f1973f28964ad5d8ea --- /dev/null +++ b/common/src/test/org/apache/hadoop/hive/conf/TestValueCoersions.java @@ -0,0 +1,51 @@ +package org.apache.hadoop.hive.conf; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +import static junit.framework.TestCase.assertEquals; + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class TestValueCoersions { + public static final String SYSTEM = "system"; + + private String makeVarName(String prefix, String value) { + return String.format("${%s:%s}", prefix, value); + } + + @Test + public void testRelativeJavaIoTmpDir() { + FileSystem localFileSystem = new LocalFileSystem(); + String systemJavaIoTmpDir = makeVarName(SYSTEM, "java.io.tmpdir"); + + System.setProperty("java.io.tmpdir", "./relativePath"); + Path relativePath = new Path(localFileSystem.getWorkingDirectory(), "./relativePath"); + assertEquals(relativePath.toString(), SystemVariables.substitute(systemJavaIoTmpDir)); + + System.setProperty("java.io.tmpdir", "file:/this/is/an/absolute/path"); + Path absolutePath = new Path(localFileSystem.getWorkingDirectory(), "/this/is/an/absolute/path"); + assertEquals(absolutePath.toString(), SystemVariables.substitute(systemJavaIoTmpDir)); + + System.setProperty("java.io.tmpdir", "this/is/a/relative/path"); + Path thisIsARelativePath= new Path(localFileSystem.getWorkingDirectory(), "this/is/a/relative/path"); + assertEquals(thisIsARelativePath.toString(), SystemVariables.substitute(systemJavaIoTmpDir)); + } +}