diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java index 81804cfb2e..8202fa5196 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java @@ -739,7 +739,7 @@ public static void copyJobPropertiesToJobConf( public static boolean isHadoop23() { String version = org.apache.hadoop.util.VersionInfo.getVersion(); - if (version.matches("\\b0\\.23\\..+\\b")||version.matches("\\b2\\..*")) + if (version.matches("\\b0\\.23\\..+\\b")||version.matches("\\b2\\..*")||version.matches("\\b3\\..*")) return true; return false; } diff --git a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java index 22d2cc6bff..341314b218 100644 --- a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java +++ b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java @@ -238,7 +238,7 @@ public void getHadoopVersion() throws Exception { Map props = JsonBuilder.jsonToMap(p.responseBody); Assert.assertEquals("hadoop", props.get("module")); Assert.assertTrue(p.getAssertMsg(), - ((String)props.get("version")).matches("[1-2].[0-9]+.[0-9]+.*")); + ((String)props.get("version")).matches("[1-3].[0-9]+.[0-9]+.*")); } @Test @@ -356,4 +356,4 @@ private static MethodCallRetVal doHttpCall(String uri, HTTP_METHOD_TYPE type, Ma } return new MethodCallRetVal(-1, "Http " + type + " failed; see log file for details", actualUri, method.getName()); } -} \ No newline at end of file +} diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml index eeb6e58b54..1c71b4fe60 100644 --- a/itests/hive-unit/pom.xml +++ b/itests/hive-unit/pom.xml @@ -125,6 +125,18 @@ commmons-logging commons-logging + + org.apache.hadoop + hadoop-mapreduce-client-common + + + org.apache.hadoop + hadoop-yarn-api + + + org.apache.hadoop + hadoop-yarn-common + @@ -360,6 +372,10 @@ commmons-logging commons-logging + + org.apache.hadoop + hadoop-yarn-server-web-proxy + diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java index aea1dfc6f4..c57e82ffca 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java @@ -1207,7 +1207,7 @@ public void testTableProperties() throws Exception { t.init(stop, looped); t.run(); JobConf job = t.getMrJob(); - Assert.assertEquals("2048", job.get("mapreduce.map.memory.mb")); // 2048 comes from tblproperties + Assert.assertEquals(2048, job.getMemoryForMapTask()); // 2048 comes from tblproperties // Compact ttp1 stop = new AtomicBoolean(true); t = new Worker(); @@ -1217,7 +1217,7 @@ public void testTableProperties() throws Exception { t.init(stop, looped); t.run(); job = t.getMrJob(); - Assert.assertEquals("1024", job.get("mapreduce.map.memory.mb")); // 1024 is the default value + Assert.assertEquals(1024, job.getMemoryForMapTask()); // 1024 is the default value // Clean up runCleaner(conf); rsp = txnHandler.showCompact(new ShowCompactRequest()); @@ -1269,7 +1269,7 @@ public void testTableProperties() throws Exception { t.init(stop, looped); t.run(); job = t.getMrJob(); - Assert.assertEquals("3072", job.get("mapreduce.map.memory.mb")); + Assert.assertEquals(3072, job.getMemoryForMapTask()); Assert.assertTrue(job.get("hive.compactor.table.props").contains("orc.compress.size4:8192")); } diff --git a/llap-server/pom.xml b/llap-server/pom.xml index 47a04cc310..ef8e9bffd4 100644 --- a/llap-server/pom.xml +++ b/llap-server/pom.xml @@ -153,6 +153,14 @@ commmons-logging commons-logging + + org.apache.hadoop + hadoop-yarn-client + + + org.apache.hadoop + hadoop-yarn-api + @@ -169,6 +177,14 @@ commmons-logging commons-logging + + io.netty + netty + + + org.apache.hadoop + hadoop-yarn-api + diff --git a/metastore/pom.xml b/metastore/pom.xml index 5430580749..9e58e63277 100644 --- a/metastore/pom.xml +++ b/metastore/pom.xml @@ -155,7 +155,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client ${hadoop.version} true diff --git a/pom.xml b/pom.xml index 52e53012b9..716560297e 100644 --- a/pom.xml +++ b/pom.xml @@ -142,7 +142,7 @@ 14.0.1 2.4.11 1.3.166 - 2.8.1 + 3.0.0-beta1 ${basedir}/${hive.path.to.root}/testutils/hadoop 1.3 1.1.1 @@ -752,6 +752,10 @@ commmons-logging commons-logging + + com.codahale.metrics + metrics-core + @@ -833,6 +837,12 @@ org.apache.hbase hbase-hadoop2-compat ${hbase.version} + + + com.codahale.metrics + metrics-core + + org.apache.hbase diff --git a/serde/pom.xml b/serde/pom.xml index 7419cfb17a..0247c32452 100644 --- a/serde/pom.xml +++ b/serde/pom.xml @@ -152,6 +152,12 @@ org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + test + + + org.apache.hadoop hadoop-hdfs ${hadoop.version} test diff --git a/shims/0.23/pom.xml b/shims/0.23/pom.xml index 3ff1d38776..61cf459a35 100644 --- a/shims/0.23/pom.xml +++ b/shims/0.23/pom.xml @@ -64,6 +64,12 @@ org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + true + + + org.apache.hadoop hadoop-hdfs ${hadoop.version} true @@ -198,6 +204,12 @@ ${hadoop.version} true test-jar + + + com.codahale.metrics + metrics-core + + org.apache.hadoop diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index e9445eb11d..c0777250f8 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -1128,10 +1128,11 @@ public Boolean run() throws Exception { @Override public boolean runDistCp(List srcPaths, Path dst, Configuration conf) throws IOException { - DistCpOptions options = new DistCpOptions(srcPaths, dst); - options.setSyncFolder(true); - options.setSkipCRC(true); - options.preserve(FileAttribute.BLOCKSIZE); + DistCpOptions options = new DistCpOptions.Builder(srcPaths, dst) + .withSyncFolder(true) + .withCRC(true) + .preserve(FileAttribute.BLOCKSIZE) + .build(); // Creates the command-line parameters for distcp List params = constructDistCpParams(srcPaths, dst, conf); @@ -1207,18 +1208,24 @@ public boolean isPathEncrypted(Path path) throws IOException { if(!"hdfs".equalsIgnoreCase(path.toUri().getScheme())) { return false; } - try { - return (hdfsAdmin.getEncryptionZoneForPath(fullPath) != null); - } catch (FileNotFoundException fnfe) { - LOG.debug("Failed to get EZ for non-existent path: "+ fullPath, fnfe); - return false; - } + + return (getEncryptionZoneForPath(fullPath) != null); + } + + private EncryptionZone getEncryptionZoneForPath(Path path) throws IOException { + if (FileSystem.get(conf).exists(path)) { + return hdfsAdmin.getEncryptionZoneForPath(path); + } else if (!path.getParent().equals(path)) { + return getEncryptionZoneForPath(path.getParent()); + } else { + return null; + } } @Override public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException { - return equivalentEncryptionZones(hdfsAdmin.getEncryptionZoneForPath(path1), - hdfsAdmin.getEncryptionZoneForPath(path2)); + return equivalentEncryptionZones(getEncryptionZoneForPath(path1), + getEncryptionZoneForPath(path2)); } private boolean equivalentEncryptionZones(EncryptionZone zone1, EncryptionZone zone2) { @@ -1256,8 +1263,8 @@ public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, public int comparePathKeyStrength(Path path1, Path path2) throws IOException { EncryptionZone zone1, zone2; - zone1 = hdfsAdmin.getEncryptionZoneForPath(path1); - zone2 = hdfsAdmin.getEncryptionZoneForPath(path2); + zone1 = getEncryptionZoneForPath(path1); + zone2 = getEncryptionZoneForPath(path2); if (zone1 == null && zone2 == null) { return 0; diff --git a/shims/common/src/main/java/org/apache/hadoop/fs/ProxyFileSystem.java b/shims/common/src/main/java/org/apache/hadoop/fs/ProxyFileSystem.java index 2c37a51cf4..ba4959cc91 100644 --- a/shims/common/src/main/java/org/apache/hadoop/fs/ProxyFileSystem.java +++ b/shims/common/src/main/java/org/apache/hadoop/fs/ProxyFileSystem.java @@ -264,6 +264,11 @@ public ContentSummary getContentSummary(Path f) throws IOException { } @Override + public FileStatus getFileLinkStatus(Path f) throws IOException { + return swizzleFileStatus(super.getFileLinkStatus(swizzleParamPath(f)), false); + } + + @Override public FileStatus getFileStatus(Path f) throws IOException { return swizzleFileStatus(super.getFileStatus(swizzleParamPath(f)), false); } diff --git a/shims/scheduler/pom.xml b/shims/scheduler/pom.xml index 0eadb69435..076e2eed42 100644 --- a/shims/scheduler/pom.xml +++ b/shims/scheduler/pom.xml @@ -87,8 +87,14 @@ org.apache.hadoop hadoop-yarn-server-tests ${hadoop.version} - true + true test-jar + + + com.codahale.metrics + metrics-core + + diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index d91b22de50..3c7d5f0ffa 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -105,6 +105,22 @@ + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + true + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hive diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java index c10e36f94a..ccb2086f2f 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java @@ -29,6 +29,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.tools.DistCp; import org.apache.hadoop.tools.DistCpOptions; +import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -147,10 +148,11 @@ public Boolean run() throws Exception { public static boolean runDistCp(List srcPaths, Path dst, Configuration conf) throws IOException { - DistCpOptions options = new DistCpOptions(srcPaths, dst); - options.setSyncFolder(true); - options.setSkipCRC(true); - options.preserve(DistCpOptions.FileAttribute.BLOCKSIZE); + DistCpOptions options = new DistCpOptions.Builder(srcPaths, dst) + .withSyncFolder(true) + .withCRC(true) + .preserve(FileAttribute.BLOCKSIZE) + .build(); // Creates the command-line parameters for distcp List params = constructDistCpParams(srcPaths, dst, conf);