diff --git hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java index 4a76010904..75aebe076b 100644 --- hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java +++ hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java @@ -361,9 +361,9 @@ private Partition constructPartition( && Boolean.parseBoolean((String)table.getProperty("EXTERNAL")) && jobInfo.getLocation() != null && jobInfo.getLocation().length() > 0) { // Now, we need to de-scratchify this location - i.e., get rid of any - // _SCRATCH[\d].?[\d]+ from the location. + // _SCRATCH[\d]+.?[\d]+[eE]?[-]?[\d]* from the location. String jobLocation = jobInfo.getLocation(); - String finalLocn = jobLocation.replaceAll(Path.SEPARATOR + SCRATCH_DIR_NAME + "\\d\\.?\\d+",""); + String finalLocn = jobLocation.replaceAll(Path.SEPARATOR + SCRATCH_DIR_NAME + "\\d+\\.?\\d+[eE]?[-]?\\d*", ""); partPath = new Path(finalLocn); } else { partPath = new Path(partLocnRoot); diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java index f7d668bb29..676523afaa 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java @@ -967,6 +967,65 @@ public void testDynamicPartitioningMultiPartColsNoDataInDataNoSpec() throws Exce driver.run("drop table employee"); } + @Test + public void testStaticPartitioningMultiPartCols() throws Exception { + AbstractHCatLoaderTest.dropTable("employee", driver); + AbstractHCatLoaderTest.createTableDefaultDB("employee", + "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING", + "emp_country STRING , emp_state STRING", driver, storageFormat); + + String[] inputData = + {"111237\tKrishna\t01/01/1990\tM\tIN\tKA", "111238\tKalpana\t01/01/2000\tF\tIN\tKA", + "111239\tSatya\t01/01/2001\tM\tIN\tKA", "111240\tKavya\t01/01/2002\tF\tIN\tKA"}; + + HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); + PigServer pig = createPigServer(false); + pig.setBatchOn(); + pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," + + "emp_gender:chararray,emp_country:chararray,emp_state:chararray);"); + pig.registerQuery("IN = FILTER A BY emp_country == 'IN' AND emp_state== 'KA';"); + pig.registerQuery("STORE IN INTO 'employee' USING " + HCatStorer.class.getName() + + "('emp_country=IN, emp_state=KA');"); + pig.executeBatch(); + driver.run("select * from employee"); + ArrayList results = new ArrayList(); + driver.getResults(results); + assertEquals(4, results.size()); + Collections.sort(results); + assertEquals(inputData[0], results.get(0)); + assertEquals(inputData[1], results.get(1)); + assertEquals(inputData[2], results.get(2)); + assertEquals(inputData[3], results.get(3)); + driver.run("drop table employee"); + } + + @Test + public void testStaticPartitioningMultiPartColsNoData() throws Exception { + AbstractHCatLoaderTest.dropTable("employee", driver); + AbstractHCatLoaderTest.createTableDefaultDB("employee", + "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING", + "emp_country STRING , emp_state STRING", driver, storageFormat); + + String[] inputData = {}; + + HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); + PigServer pig = createPigServer(false); + pig.setBatchOn(); + pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," + + "emp_gender:chararray,emp_country:chararray,emp_state:chararray);"); + pig.registerQuery("IN = FILTER A BY emp_country == 'IN' AND emp_state== 'KA';"); + pig.registerQuery("STORE IN INTO 'employee' USING " + HCatStorer.class.getName() + + "('emp_country=IN, emp_state=KA');"); + pig.executeBatch(); + driver.run("select * from employee"); + ArrayList results = new ArrayList(); + driver.getResults(results); + assertEquals(0, results.size()); + driver.run("drop table employee"); + } + @Test public void testPartitionPublish() throws Exception { AbstractHCatLoaderTest.dropTable("ptn_fail", driver); diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java index cb0213943d..090d3998f7 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java @@ -44,6 +44,8 @@ add("testDynamicPartitioningMultiPartColsInDataNoSpec"); add("testDynamicPartitioningMultiPartColsInDataPartialSpec"); add("testDynamicPartitioningMultiPartColsNoDataInDataNoSpec"); + add("testStaticPartitioningMultiPartCols"); + add("testStaticPartitioningMultiPartColsNoData"); add("testEmptyStore"); add("testMultiPartColsInData"); add("testNoAlias"); @@ -282,6 +284,20 @@ public void testDynamicPartitioningMultiPartColsNoDataInDataNoSpec() throws Exce super.testDynamicPartitioningMultiPartColsNoDataInDataNoSpec(); } + @Test + @Override + public void testStaticPartitioningMultiPartCols() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); + super.testStaticPartitioningMultiPartCols(); + } + + @Test + @Override + public void testStaticPartitioningMultiPartColsNoData() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); + super.testStaticPartitioningMultiPartColsNoData(); + } + @Test @Override public void testPartitionPublish() throws Exception {