diff --git hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/InternalUtil.java hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/InternalUtil.java index bb1cb18..fb650dd 100644 --- hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/InternalUtil.java +++ hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/InternalUtil.java @@ -106,7 +106,7 @@ private static ObjectInspector getObjectInspector(TypeInfo type) throws IOExcept case PRIMITIVE: PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type; return PrimitiveObjectInspectorFactory. - getPrimitiveJavaObjectInspector(primitiveType.getPrimitiveCategory()); + getPrimitiveJavaObjectInspector(primitiveType); case MAP: MapTypeInfo mapType = (MapTypeInfo) type; diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java index 85eb528..fcfc642 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java @@ -211,6 +211,13 @@ private void pigValueRangeTest(String tblName, String hiveType, String pigType, String expectedValue) throws Exception { pigValueRangeTest(tblName, hiveType, pigType, goal, inputValue, expectedValue, null); } + + /** + * this should be overridden in subclass to test with different file formats + */ + String getStorageFormat() { + return "RCFILE"; + } /** * This is used to test how Pig values of various data types which are out of range for Hive target * column are handled. Currently the options are to raise an error or write NULL. @@ -234,7 +241,7 @@ private void pigValueRangeTest(String tblName, String hiveType, String pigType, throws Exception { TestHCatLoader.dropTable(tblName, driver); final String field = "f1"; - TestHCatLoader.createTable(tblName, field + " " + hiveType, null, driver, "RCFILE"); + TestHCatLoader.createTable(tblName, field + " " + hiveType, null, driver, getStorageFormat()); HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, new String[] {inputValue}); LOG.debug("File=" + INPUT_FILE_NAME); dumpFile(INPUT_FILE_NAME); @@ -312,7 +319,7 @@ public void testDateCharTypes() throws Exception { final String tblName = "junit_date_char"; TestHCatLoader.dropTable(tblName, driver); TestHCatLoader.createTable(tblName, - "id int, char5 char(5), varchar10 varchar(10), dec52 decimal(5,2)", null, driver, "RCFILE"); + "id int, char5 char(5), varchar10 varchar(10), dec52 decimal(5,2)", null, driver, getStorageFormat()); int NUM_ROWS = 5; String[] rows = new String[NUM_ROWS]; for(int i = 0; i < NUM_ROWS; i++) { @@ -351,7 +358,7 @@ public void testDateCharTypes() throws Exception { rowFromPig.append(t.get(i)).append("\t"); } rowFromPig.setLength(rowFromPig.length() - 1); - Assert.assertEquals("Comparing Pig to Raw data", rowFromPig.toString(), rows[numRowsRead]); + Assert.assertEquals("Comparing Pig to Raw data", rows[numRowsRead], rowFromPig.toString()); //see comment at "Dumping rows via SQL..." for why this doesn't work (for all types) //Assert.assertEquals("Comparing Pig to Hive", rowFromPig.toString(), l.get(numRowsRead)); numRowsRead++; @@ -372,7 +379,7 @@ static void dumpFile(String fileName) throws Exception { public void testPartColsInData() throws IOException, CommandNeedRetryException { driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int) partitioned by (b string) stored as RCFILE"; + String createTable = "create table junit_unparted(a int) partitioned by (b string) stored as " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -408,7 +415,7 @@ public void testMultiPartColsInData() throws IOException, CommandNeedRetryExcept driver.run("drop table employee"); String createTable = "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " + - " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS RCFILE"; + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { @@ -450,7 +457,7 @@ public void testMultiPartColsInData() throws IOException, CommandNeedRetryExcept public void testStoreInPartiitonedTbl() throws IOException, CommandNeedRetryException { driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int) partitioned by (b string) stored as RCFILE"; + String createTable = "create table junit_unparted(a int) partitioned by (b string) stored as " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -484,7 +491,7 @@ public void testStoreInPartiitonedTbl() throws IOException, CommandNeedRetryExce @Test public void testNoAlias() throws IOException, CommandNeedRetryException { driver.run("drop table junit_parted"); - String createTable = "create table junit_parted(a int, b string) partitioned by (ds string) stored as RCFILE"; + String createTable = "create table junit_parted(a int, b string) partitioned by (ds string) stored as " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -527,7 +534,7 @@ public void testNoAlias() throws IOException, CommandNeedRetryException { public void testStoreMultiTables() throws IOException, CommandNeedRetryException { driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as RCFILE"; + String createTable = "create table junit_unparted(a int, b string) stored as " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -582,7 +589,7 @@ public void testStoreMultiTables() throws IOException, CommandNeedRetryException public void testStoreWithNoSchema() throws IOException, CommandNeedRetryException { driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as RCFILE"; + String createTable = "create table junit_unparted(a int, b string) stored as " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -621,7 +628,7 @@ public void testStoreWithNoSchema() throws IOException, CommandNeedRetryExceptio public void testStoreWithNoCtorArgs() throws IOException, CommandNeedRetryException { driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as RCFILE"; + String createTable = "create table junit_unparted(a int, b string) stored as " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -660,7 +667,7 @@ public void testStoreWithNoCtorArgs() throws IOException, CommandNeedRetryExcept public void testEmptyStore() throws IOException, CommandNeedRetryException { driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as RCFILE"; + String createTable = "create table junit_unparted(a int, b string) stored as " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -696,7 +703,7 @@ public void testEmptyStore() throws IOException, CommandNeedRetryException { public void testBagNStruct() throws IOException, CommandNeedRetryException { driver.run("drop table junit_unparted"); String createTable = "create table junit_unparted(b string,a struct, arr_of_struct array, " + - "arr_of_struct2 array>, arr_of_struct3 array>) stored as RCFILE"; + "arr_of_struct2 array>, arr_of_struct3 array>) stored as " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -729,7 +736,7 @@ public void testBagNStruct() throws IOException, CommandNeedRetryException { public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryException { driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, h boolean, f binary, g binary) stored as RCFILE"; + String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, h boolean, f binary, g binary) stored as " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -784,7 +791,7 @@ public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryEx public void testStoreFuncSimple() throws IOException, CommandNeedRetryException { driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as RCFILE"; + String createTable = "create table junit_unparted(a int, b string) stored as " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -826,7 +833,7 @@ public void testDynamicPartitioningMultiPartColsInDataPartialSpec() throws IOExc driver.run("drop table if exists employee"); String createTable = "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " + - " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS RCFILE"; + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { @@ -863,7 +870,7 @@ public void testDynamicPartitioningMultiPartColsInDataNoSpec() throws IOExceptio driver.run("drop table if exists employee"); String createTable = "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " + - " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS RCFILE"; + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { @@ -900,7 +907,7 @@ public void testDynamicPartitioningMultiPartColsNoDataInDataNoSpec() throws IOEx driver.run("drop table if exists employee"); String createTable = "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " + - " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS RCFILE"; + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { @@ -928,7 +935,7 @@ public void testPartitionPublish() throws IOException, CommandNeedRetryException { driver.run("drop table ptn_fail"); - String createTable = "create table ptn_fail(a int, c string) partitioned by (b string) stored as RCFILE"; + String createTable = "create table ptn_fail(a int, c string) partitioned by (b string) stored as " + getStorageFormat(); int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatPigStorer.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatPigStorer.java new file mode 100644 index 0000000..4e2c723 --- /dev/null +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatPigStorer.java @@ -0,0 +1,36 @@ +package org.apache.hive.hcatalog.pig; + +import org.apache.hadoop.hive.ql.CommandNeedRetryException; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +public class TestOrcHCatPigStorer extends TestHCatStorer { + @Override String getStorageFormat() { + return "ORC"; + } + @Ignore("BUG-15650") + @Test + @Override public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryException { + super.testStoreFuncAllSimpleTypes(); + } +}