diff --git a/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java b/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java index 696e081d03eb0ccc92a01a145fa2b68cb523f852..dd957becddff762f59fafd85d1c49c40d89ce109 100644 --- a/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java +++ b/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java @@ -260,6 +260,10 @@ public void setPartitionFilter(Expression partitionFilter) throws IOException { @Override public ResourceStatistics getStatistics(String location, Job job) throws IOException { try { + if (dbName == null || tableName == null) { + throw new IOException("DB or table name unset. setLocation() must be invoked on this " + + "loader to set them"); + } ResourceStatistics stats = new ResourceStatistics(); long inputSize = -1; @@ -267,14 +271,15 @@ public ResourceStatistics getStatistics(String location, Job job) throws IOExcep job.getConfiguration()); for (InputJobInfo inputJobInfo : inputJobInfos) { - if (location.equals(inputJobInfo.getTableName())) { + if (dbName.equals(inputJobInfo.getDatabaseName()) && tableName.equals(inputJobInfo.getTableName())){ inputSize = getSizeInBytes(inputJobInfo); break; } } if (inputSize == -1) { - throw new IOException("Could not calculate input size for location (table) " + location); + throw new IOException("Could not calculate input size for database: " + dbName + ", " + + "table: " + tableName + ". Requested location:" + location); } stats.setSizeInBytes(inputSize); return stats; diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java index 8d1760882dbc690c4210abdf03ee5b27c76d0628..4c4551c13c674dce6732465acb2147fcaa16160f 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java @@ -78,6 +78,7 @@ private static final String COMPLEX_TABLE = "junit_unparted_complex"; private static final String PARTITIONED_TABLE = "junit_parted_basic"; private static final String SPECIFIC_SIZE_TABLE = "junit_specific_size"; + private static final String SPECIFIC_DATABASE = "junit_specific_db"; private static final String SPECIFIC_SIZE_TABLE_2 = "junit_specific_size2"; private static final String PARTITIONED_DATE_TABLE = "junit_parted_date"; @@ -99,11 +100,23 @@ static void dropTable(String tablename, IDriver driver) throws Exception { driver.run("drop table if exists " + tablename); } - private void createTable(String tablename, String schema, String partitionedBy) throws Exception { - createTable(tablename, schema, partitionedBy, driver, storageFormat); + private void createTable(String db, String tablename, String schema, String partitionedBy) throws + Exception { + createTable(db, tablename, schema, partitionedBy, driver, storageFormat); } - static void createTable(String tablename, String schema, String partitionedBy, IDriver driver, String storageFormat) + private void createTableDefaultDB(String tablename, String schema, String partitionedBy) throws + Exception { + createTable(null, tablename, schema, partitionedBy, driver, storageFormat); + } + + static void createTableDefaultDB(String tablename, String schema, String partitionedBy, IDriver + driver, String storageFormat) throws Exception { + createTable(null, tablename, schema, partitionedBy, driver, storageFormat); + } + + static void createTable(String db, String tablename, String schema, String partitionedBy, IDriver + driver, String storageFormat) throws Exception { String createTable; createTable = "create table " + tablename + "(" + schema + ") "; @@ -113,11 +126,21 @@ static void createTable(String tablename, String schema, String partitionedBy, I createTable = createTable + "stored as " +storageFormat; //HCat doesn't support transactional tables createTable += " TBLPROPERTIES ('transactional'='false')"; + if (db != null) { + executeStatementOnDriver("create database if not exists " + db, driver); + executeStatementOnDriver("use " + db + "", driver); + } else { + executeStatementOnDriver("use default", driver); + } executeStatementOnDriver(createTable, driver); } - private void createTable(String tablename, String schema) throws Exception { - createTable(tablename, schema, null); + private void createTable(String db, String tablename, String schema) throws Exception { + createTable(db, tablename, schema, null); + } + + private void createTableDefaultDB(String tablename, String schema) throws Exception { + createTable(null, tablename, schema, null); } /** @@ -140,18 +163,18 @@ private static void checkProjection(FieldSchema fs, String expectedName, byte ex @Before public void setUpTest() throws Exception { - createTable(BASIC_TABLE, "a int, b string"); - createTable(COMPLEX_TABLE, + createTableDefaultDB(BASIC_TABLE, "a int, b string"); + createTableDefaultDB(COMPLEX_TABLE, "name string, studentid int, " + "contact struct, " + "currently_registered_courses array, " + "current_grades map, " + "phnos array>"); - createTable(PARTITIONED_TABLE, "a int, b string", "bkt string"); - createTable(SPECIFIC_SIZE_TABLE, "a int, b string"); - createTable(SPECIFIC_SIZE_TABLE_2, "a int, b string"); - createTable(PARTITIONED_DATE_TABLE, "b string", "dt date"); + createTableDefaultDB(PARTITIONED_TABLE, "a int, b string", "bkt string"); + createTableDefaultDB(SPECIFIC_SIZE_TABLE, "a int, b string"); + createTable(SPECIFIC_DATABASE, SPECIFIC_SIZE_TABLE_2, "a int, b string"); + createTableDefaultDB(PARTITIONED_DATE_TABLE, "b string", "dt date"); AllTypesTable.setupAllTypesTable(driver); int LOOP_SIZE = 3; @@ -187,6 +210,8 @@ public void setUpTest() throws Exception { server.registerQuery("store A into '" + BASIC_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i); server.registerQuery("store A into '" + SPECIFIC_SIZE_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i); + server.registerQuery("store A into '" + SPECIFIC_DATABASE + "." +SPECIFIC_SIZE_TABLE_2 + "' " + + "using org.apache.hive" +".hcatalog.pig.HCatStorer();", ++i); server.registerQuery("B = foreach A generate a,b;", ++i); server.registerQuery("B2 = filter B by a < 2;", ++i); server.registerQuery("store B2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=0');", ++i); @@ -213,9 +238,10 @@ public void tearDown() throws Exception { dropTable(COMPLEX_TABLE); dropTable(PARTITIONED_TABLE); dropTable(SPECIFIC_SIZE_TABLE); - dropTable(SPECIFIC_SIZE_TABLE_2); dropTable(PARTITIONED_DATE_TABLE); dropTable(AllTypesTable.ALL_PRIMITIVE_TYPES_TABLE); + executeStatementOnDriver("drop database if exists " + SPECIFIC_DATABASE + " cascade", + driver); } } finally { FileUtils.deleteDirectory(new File(TEST_DATA_DIR)); @@ -578,7 +604,8 @@ public void testGetInputBytesMultipleTables() throws Exception { RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw"); randomAccessFile.setLength(987654321L); randomAccessFile.close(); - file = new File(TEST_WAREHOUSE_DIR + "/" + SPECIFIC_SIZE_TABLE_2 + "/part-m-00000"); + file = new File(TEST_WAREHOUSE_DIR + "/" + SPECIFIC_DATABASE + ".db/" + + SPECIFIC_SIZE_TABLE_2 + "/part-m-00000"); file.deleteOnExit(); randomAccessFile = new RandomAccessFile(file, "rw"); randomAccessFile.setLength(12345678L); @@ -590,11 +617,13 @@ public void testGetInputBytesMultipleTables() throws Exception { hCatLoader.setUDFContextSignature("testGetInputBytesMultipleTables" + SPECIFIC_SIZE_TABLE); hCatLoader.setLocation(SPECIFIC_SIZE_TABLE, job); - hCatLoader.setUDFContextSignature("testGetInputBytesMultipleTables" + SPECIFIC_SIZE_TABLE_2); - hCatLoader.setLocation(SPECIFIC_SIZE_TABLE_2, job); + HCatLoader hCatLoader2 = new HCatLoader(); + hCatLoader2.setUDFContextSignature("testGetInputBytesMultipleTables" + SPECIFIC_SIZE_TABLE_2); + hCatLoader2.setLocation(SPECIFIC_DATABASE + "." + SPECIFIC_SIZE_TABLE_2, job); - hCatLoader.setUDFContextSignature("testGetInputBytesMultipleTables" + PARTITIONED_TABLE); - hCatLoader.setLocation(PARTITIONED_TABLE, job); + HCatLoader hCatLoader3 = new HCatLoader(); + hCatLoader3.setUDFContextSignature("testGetInputBytesMultipleTables" + PARTITIONED_TABLE); + hCatLoader3.setLocation(PARTITIONED_TABLE, job); long specificTableSize = -1; long specificTableSize2 = -1; @@ -604,11 +633,11 @@ public void testGetInputBytesMultipleTables() throws Exception { specificTableSize=statistics.getSizeInBytes(); assertEquals(987654321, specificTableSize); - statistics = hCatLoader.getStatistics(SPECIFIC_SIZE_TABLE_2, job); + statistics = hCatLoader2.getStatistics(SPECIFIC_SIZE_TABLE_2, job); specificTableSize2=statistics.getSizeInBytes(); assertEquals(12345678, specificTableSize2); - statistics = hCatLoader.getStatistics(PARTITIONED_TABLE, job); + statistics = hCatLoader3.getStatistics(PARTITIONED_TABLE, job); partitionedTableSize=statistics.getSizeInBytes(); //Partitioned table size here is dependent on underlying storage format, it's ~ 20, arr_of_struct array, " + "arr_of_struct2 array>, arr_of_struct3 array>", null, driver, storageFormat); @@ -775,7 +779,7 @@ public void testBagNStruct() throws Exception { @Test public void testStoreFuncAllSimpleTypes() throws Exception { AbstractHCatLoaderTest.dropTable("junit_unparted", driver); - AbstractHCatLoaderTest.createTable("junit_unparted", + AbstractHCatLoaderTest.createTableDefaultDB("junit_unparted", "a int, b float, c double, d bigint, e string, h boolean, f binary, g binary", null, driver, storageFormat); @@ -834,7 +838,7 @@ public void testStoreFuncAllSimpleTypes() throws Exception { @Test public void testStoreFuncSimple() throws Exception { AbstractHCatLoaderTest.dropTable("junit_unparted", driver); - AbstractHCatLoaderTest.createTable("junit_unparted","a int, b string", null, + AbstractHCatLoaderTest.createTableDefaultDB("junit_unparted", "a int, b string", null, driver, storageFormat); int LOOP_SIZE = 3; @@ -872,7 +876,7 @@ public void testStoreFuncSimple() throws Exception { @Test public void testDynamicPartitioningMultiPartColsInDataPartialSpec() throws Exception { AbstractHCatLoaderTest.dropTable("employee", driver); - AbstractHCatLoaderTest.createTable("employee", + AbstractHCatLoaderTest.createTableDefaultDB("employee", "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING", "emp_country STRING , emp_state STRING", driver, storageFormat); @@ -905,7 +909,7 @@ public void testDynamicPartitioningMultiPartColsInDataPartialSpec() throws Excep @Test public void testDynamicPartitioningMultiPartColsInDataNoSpec() throws Exception { AbstractHCatLoaderTest.dropTable("employee", driver); - AbstractHCatLoaderTest.createTable("employee", + AbstractHCatLoaderTest.createTableDefaultDB("employee", "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING", "emp_country STRING , emp_state STRING", driver, storageFormat); @@ -937,7 +941,7 @@ public void testDynamicPartitioningMultiPartColsInDataNoSpec() throws Exception @Test public void testDynamicPartitioningMultiPartColsNoDataInDataNoSpec() throws Exception { AbstractHCatLoaderTest.dropTable("employee", driver); - AbstractHCatLoaderTest.createTable("employee", + AbstractHCatLoaderTest.createTableDefaultDB("employee", "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING", "emp_country STRING , emp_state STRING", driver, storageFormat); @@ -963,7 +967,7 @@ public void testDynamicPartitioningMultiPartColsNoDataInDataNoSpec() throws Exce @Test public void testPartitionPublish() throws Exception { AbstractHCatLoaderTest.dropTable("ptn_fail", driver); - AbstractHCatLoaderTest.createTable("ptn_fail","a int, c string", "b string", + AbstractHCatLoaderTest.createTableDefaultDB("ptn_fail", "a int, c string", "b string", driver, storageFormat); int LOOP_SIZE = 11; diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java index 19246e656043d2e8d15dc8903f92f653f1050368..e373f195fc59d82b60a7e6d6a0b08246859b0512 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java @@ -113,7 +113,8 @@ private void dropTable(String tablename) throws Exception { private void createTable(String tablename, String schema, String partitionedBy, String storageFormat) throws Exception { - AbstractHCatLoaderTest.createTable(tablename, schema, partitionedBy, driver, storageFormat); + AbstractHCatLoaderTest.createTableDefaultDB(tablename, schema, partitionedBy, driver, + storageFormat); } private void driverRun(String cmd) throws Exception { diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java index 37e670c4a80979fe9ceccf8ad72a9463148b687e..b96479b826a0b441f8e90eae1fbaf9fc4ba6a326 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java @@ -96,7 +96,7 @@ private void dropTable(String tablename) throws Exception { } private void createTable(String tablename, String schema, String partitionedBy) throws Exception { - AbstractHCatLoaderTest.createTable(tablename, schema, partitionedBy, driver, storageFormat); + AbstractHCatLoaderTest.createTableDefaultDB(tablename, schema, partitionedBy, driver, storageFormat); } private void createTable(String tablename, String schema) throws Exception { diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java index c7f260613281c61361e039482af53f7b571e9ff5..38e1e7e1ff576a30152c48a6f41e5511d2df1531 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java @@ -65,8 +65,8 @@ public void testReadWrite() throws Exception { " row format delimited fields terminated by '\t' stored as textfile location '" + dataDir.toURI().getPath() + "'", driver); AbstractHCatLoaderTest.dropTable(tblName2, driver); - AbstractHCatLoaderTest.createTable(tblName2, "my_small_int smallint, my_tiny_int tinyint", null, driver, - "textfile"); + AbstractHCatLoaderTest.createTableDefaultDB(tblName2, "my_small_int smallint, " + + "my_tiny_int " + "tinyint", null, driver, "textfile"); LOG.debug("File=" + INPUT_FILE_NAME); TestHCatStorer.dumpFile(INPUT_FILE_NAME); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java index 65ad6ecbb23ac906865688aa27dc590b8d9de0d3..a0c5ce93ff2765d4e07381a290b2d8a6c6545f16 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java @@ -82,7 +82,7 @@ private void dropTable(String tablename) throws Exception { } private void createTable(String tablename, String schema, String partitionedBy) throws Exception { - AbstractHCatLoaderTest.createTable(tablename, schema, partitionedBy, driver, storageFormat); + AbstractHCatLoaderTest.createTableDefaultDB(tablename, schema, partitionedBy, driver, storageFormat); } private void createTable(String tablename, String schema) throws Exception {