Index: src/test/org/apache/hcatalog/mapreduce/TestHCatPartitioned.java =================================================================== --- src/test/org/apache/hcatalog/mapreduce/TestHCatPartitioned.java (revision 1325041) +++ src/test/org/apache/hcatalog/mapreduce/TestHCatPartitioned.java (working copy) @@ -62,7 +62,9 @@ @Override protected List getPartitionKeys() { List fields = new ArrayList(); + //Defining partition names in unsorted order fields.add(new FieldSchema("PaRT1", Constants.STRING_TYPE_NAME, "")); + fields.add(new FieldSchema("part0", Constants.STRING_TYPE_NAME, "")); return fields; } @@ -79,11 +81,13 @@ Map partitionMap = new HashMap(); partitionMap.put("part1", "p1value1"); + partitionMap.put("part0", "p0value1"); runMRCreate(partitionMap, partitionColumns, writeRecords, 10,true); partitionMap.clear(); partitionMap.put("PART1", "p1value2"); + partitionMap.put("PART0", "p0value2"); runMRCreate(partitionMap, partitionColumns, writeRecords, 20,true); @@ -102,7 +106,8 @@ //Test for publish with invalid partition key name exc = null; partitionMap.clear(); - partitionMap.put("px", "p1value2"); + partitionMap.put("px1", "p1value2"); + partitionMap.put("px0", "p0value2"); try { runMRCreate(partitionMap, partitionColumns, writeRecords, 20,true); @@ -114,7 +119,22 @@ assertTrue(exc instanceof HCatException); assertEquals(ErrorType.ERROR_MISSING_PARTITION_KEY, ((HCatException) exc).getErrorType()); + //Test for publish with missing partition key values + exc = null; + partitionMap.clear(); + partitionMap.put("px", "p1value2"); + try { + runMRCreate(partitionMap, partitionColumns, writeRecords, 20,true); + } catch(IOException e) { + exc = e; + } + + assertTrue(exc != null); + assertTrue(exc instanceof HCatException); + assertEquals(ErrorType.ERROR_INVALID_PARTITION_VALUES, ((HCatException) exc).getErrorType()); + + //Test for null partition value map exc = null; try { @@ -135,6 +155,9 @@ runMRRead(10, "part1 = \"p1value1\""); runMRRead(20, "part1 = \"p1value2\""); runMRRead(30, "part1 = \"p1value1\" or part1 = \"p1value2\""); + runMRRead(10, "part0 = \"p0value1\""); + runMRRead(20, "part0 = \"p0value2\""); + runMRRead(30, "part0 = \"p0value1\" or part0 = \"p0value2\""); tableSchemaTest(); columnOrderChangeTest(); @@ -147,7 +170,7 @@ HCatSchema tableSchema = getTableSchema(); - assertEquals(3, tableSchema.getFields().size()); + assertEquals(4, tableSchema.getFields().size()); //Update partition schema to have 3 fields partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c3", Constants.STRING_TYPE_NAME, ""))); @@ -166,21 +189,24 @@ Map partitionMap = new HashMap(); partitionMap.put("part1", "p1value5"); + partitionMap.put("part0", "p0value5"); runMRCreate(partitionMap, partitionColumns, writeRecords, 10,true); tableSchema = getTableSchema(); //assert that c3 has got added to table schema - assertEquals(4, tableSchema.getFields().size()); + assertEquals(5, tableSchema.getFields().size()); assertEquals("c1", tableSchema.getFields().get(0).getName()); assertEquals("c2", tableSchema.getFields().get(1).getName()); assertEquals("c3", tableSchema.getFields().get(2).getName()); assertEquals("part1", tableSchema.getFields().get(3).getName()); + assertEquals("part0", tableSchema.getFields().get(4).getName()); //Test that changing column data type fails partitionMap.clear(); partitionMap.put("part1", "p1value6"); + partitionMap.put("part0", "p0value6"); partitionColumns = new ArrayList(); partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c1", Constants.INT_TYPE_NAME, ""))); @@ -225,13 +251,16 @@ List records= runMRRead(20,"part1 = \"p1value6\""); assertEquals(20, records.size()); + records= runMRRead(20,"part0 = \"p0value6\""); + assertEquals(20, records.size()); Integer i =0; for(HCatRecord rec : records){ - assertEquals(4, rec.size()); + assertEquals(5, rec.size()); assertTrue(rec.get(0).equals(i)); assertTrue(rec.get(1).equals("c2value"+i)); assertTrue(rec.get(2).equals("c3value"+i)); assertTrue(rec.get(3).equals("p1value6")); + assertTrue(rec.get(4).equals("p0value6")); i++; } } @@ -241,7 +270,7 @@ HCatSchema tableSchema = getTableSchema(); - assertEquals(4, tableSchema.getFields().size()); + assertEquals(5, tableSchema.getFields().size()); partitionColumns = new ArrayList(); partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c1", Constants.INT_TYPE_NAME, ""))); @@ -263,8 +292,8 @@ Map partitionMap = new HashMap(); partitionMap.put("part1", "p1value8"); + partitionMap.put("part0", "p0value8"); - Exception exc = null; try { runMRCreate(partitionMap, partitionColumns, writeRecords, 10,true); Index: src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java =================================================================== --- src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java (revision 1325041) +++ src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java (working copy) @@ -50,6 +50,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; @@ -262,8 +263,12 @@ HCatOutputFormat.setSchema(job, new HCatSchema(partitionColumns)); - job.waitForCompletion(true); - new FileOutputCommitterContainer(job,null).cleanupJob(job); + boolean success = job.waitForCompletion(true); + if (success) { + new FileOutputCommitterContainer(job,null).commitJob(job); + } else { + new FileOutputCommitterContainer(job,null).abortJob(job, JobStatus.State.FAILED); + } if (assertWrite){ // we assert only if we expected to assert with this call. Assert.assertEquals(writeCount, MapCreate.writeCount); Index: src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java =================================================================== --- src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java (revision 1325041) +++ src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java (working copy) @@ -19,10 +19,7 @@ package org.apache.hcatalog.mapreduce; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.ql.io.RCFile; @@ -32,13 +29,11 @@ import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputFormat; import org.apache.hcatalog.common.HCatConstants; import org.apache.hcatalog.common.HCatUtil; import java.io.IOException; -import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -95,13 +90,13 @@ } @Override - public void configureInputJobProperties(TableDesc tableDesc, + public void configureInputJobProperties(TableDesc tableDesc, Map jobProperties) { } @Override - public void configureOutputJobProperties(TableDesc tableDesc, + public void configureOutputJobProperties(TableDesc tableDesc, Map jobProperties) { try { OutputJobInfo jobInfo = (OutputJobInfo) @@ -114,7 +109,7 @@ // For dynamic partitioned writes without all keyvalues specified, // we create a temp dir for the associated write job if (dynHash != null){ - parentPath = new Path(parentPath, + parentPath = new Path(parentPath, DYNTEMP_DIR_NAME+dynHash).toString(); } @@ -128,16 +123,13 @@ List cols = new ArrayList(); List values = new ArrayList(); - //sort the cols and vals - for(String name: + //Get the output location in the order partition keys are defined for the table. + for(String name: jobInfo.getTableInfo(). getPartitionColumns().getFieldNames()) { String value = jobInfo.getPartitionValues().get(name); - int i=0; - while(i 0) - i++; - cols.add(i,name); - values.add(i,value); + cols.add(name); + values.add(value); } outputLocation = FileUtils.makePartName(cols, values); } @@ -145,7 +137,7 @@ jobInfo.setLocation(new Path(parentPath,outputLocation).toString()); //only set output dir if partition is fully materialized - if(jobInfo.getPartitionValues().size() + if(jobInfo.getPartitionValues().size() == jobInfo.getTableInfo().getPartitionColumns().size()) { jobProperties.put("mapred.output.dir", jobInfo.getLocation()); } @@ -179,7 +171,7 @@ } @Override - public HiveAuthorizationProvider getAuthorizationProvider() + public HiveAuthorizationProvider getAuthorizationProvider() throws HiveException { return new DefaultHiveAuthorizationProvider(); }