diff --git a/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java b/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java index 3417e1b..a2f6419 100644 --- a/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java +++ b/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java @@ -117,9 +117,11 @@ public void configureOutputJobProperties(TableDesc tableDesc, String outputLocation; - if (Boolean.valueOf((String)tableDesc.getProperties().get("EXTERNAL")) + if ((dynHash == null) + && Boolean.valueOf((String)tableDesc.getProperties().get("EXTERNAL")) && jobInfo.getLocation() != null && jobInfo.getLocation().length() > 0) { - // honor external table that specifies the location + // honor custom location for external table apart from what metadata specifies + // only if we're not using dynamic partitioning - see HIVE-5011 outputLocation = jobInfo.getLocation(); } else if (dynHash == null && jobInfo.getPartitionValues().size() == 0) { // For non-partitioned tables, we send them to the temp dir diff --git a/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java index 3c4b4d6..7b7d340 100644 --- a/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java +++ b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; @@ -83,6 +84,10 @@ protected abstract List getTableColumns(); private static FileSystem fs; + + protected Boolean isTableExternal() { + return false; + } protected String inputFormat() { return RCFileInputFormat.class.getName(); @@ -137,7 +142,11 @@ public void createTable() throws Exception { Table tbl = new Table(); tbl.setDbName(databaseName); tbl.setTableName(tableName); - tbl.setTableType("MANAGED_TABLE"); + if (isTableExternal()){ + tbl.setTableType(TableType.EXTERNAL_TABLE.toString()); + } else { + tbl.setTableType(TableType.MANAGED_TABLE.toString()); + } StorageDescriptor sd = new StorageDescriptor(); sd.setCols(getTableColumns()); @@ -150,6 +159,9 @@ public void createTable() throws Exception { sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap()); sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); + if (isTableExternal()){ + sd.getSerdeInfo().getParameters().put("EXTERNAL", "TRUE"); + } sd.getSerdeInfo().setSerializationLib(serdeClass()); sd.setInputFormat(inputFormat()); sd.setOutputFormat(outputFormat()); diff --git a/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java new file mode 100644 index 0000000..ff61675 --- /dev/null +++ b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.mapreduce; + +public class TestHCatExternalDynamicPartitioned extends TestHCatDynamicPartitioned { + + @Override + protected Boolean isTableExternal() { + return true; + } + +} diff --git a/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalHCatNonPartitioned.java b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalHCatNonPartitioned.java new file mode 100644 index 0000000..5fe4d3e --- /dev/null +++ b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalHCatNonPartitioned.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.mapreduce; + +public class TestHCatExternalHCatNonPartitioned extends TestHCatNonPartitioned { + + @Override + protected Boolean isTableExternal() { + return true; + } + +} diff --git a/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalPartitioned.java b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalPartitioned.java new file mode 100644 index 0000000..2fdbeab --- /dev/null +++ b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalPartitioned.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.mapreduce; + +public class TestHCatExternalPartitioned extends TestHCatPartitioned { + + @Override + protected Boolean isTableExternal() { + return true; + } + +}