commit 23da064856c6eaafda1aa770f8ebb8f2d1c8b681 Author: Nitay Joffe Date: Wed Jan 7 17:29:43 2015 -0500 Don't create a new Configuration in AvroSerdeUtils#determineSchemaOrThrow diff --git ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java index 3985cd6..633a75b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java @@ -60,7 +60,7 @@ Properties properties, Progressable progressable) throws IOException { Schema schema; try { - schema = AvroSerdeUtils.determineSchemaOrThrowException(properties); + schema = AvroSerdeUtils.determineSchemaOrThrowException(jobConf, properties); } catch (AvroSerdeException e) { throw new IOException(e); } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java index 22e54d3..1381514 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java @@ -109,7 +109,7 @@ private Schema getSchema(JobConf job, FileSplit split) throws AvroSerdeException Properties props = pathsAndParts.getValue().getProperties(); if(props.containsKey(AvroSerdeUtils.SCHEMA_LITERAL) || props.containsKey(AvroSerdeUtils.SCHEMA_URL)) { - return AvroSerdeUtils.determineSchemaOrThrowException(props); + return AvroSerdeUtils.determineSchemaOrThrowException(job, props); } else { return null; // If it's not in this property, it won't be in any others diff --git serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java index 0280b0e..8ba0692 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java @@ -100,7 +100,7 @@ public void initialize(Configuration configuration, Properties properties) throw || properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_URL.getPropName()) != null || columnNameProperty == null || columnNameProperty.isEmpty() || columnTypeProperty == null || columnTypeProperty.isEmpty()) { - schema = determineSchemaOrReturnErrorSchema(properties); + schema = determineSchemaOrReturnErrorSchema(configuration, properties); } else { // Get column names and sort order columnNames = Arrays.asList(columnNameProperty.split(",")); @@ -159,10 +159,10 @@ public static Schema getSchemaFromCols(Properties properties, * any call, including calls to update the serde properties, meaning * if the serde is in a bad state, there is no way to update that state. */ - public Schema determineSchemaOrReturnErrorSchema(Properties props) { + public Schema determineSchemaOrReturnErrorSchema(Configuration conf, Properties props) { try { configErrors = ""; - return AvroSerdeUtils.determineSchemaOrThrowException(props); + return AvroSerdeUtils.determineSchemaOrThrowException(conf, props); } catch(AvroSerdeException he) { LOG.warn("Encountered AvroSerdeException determining schema. Returning " + "signal schema to indicate problem", he); diff --git serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java index fa9222a..4edf654 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java @@ -97,7 +97,7 @@ public String getPropName(){ * @throws IOException if error while trying to read the schema from another location * @throws AvroSerdeException if unable to find a schema or pointer to it in the properties */ - public static Schema determineSchemaOrThrowException(Properties properties) + public static Schema determineSchemaOrThrowException(Configuration conf, Properties properties) throws IOException, AvroSerdeException { String schemaString = properties.getProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName()); if(schemaString != null && !schemaString.equals(SCHEMA_NONE)) @@ -109,7 +109,7 @@ public static Schema determineSchemaOrThrowException(Properties properties) throw new AvroSerdeException(EXCEPTION_MESSAGE); try { - Schema s = getSchemaFromFS(schemaString, new Configuration()); + Schema s = getSchemaFromFS(schemaString, conf); if (s == null) { //in case schema is not a file system return AvroSerdeUtils.getSchemaFor(new URL(schemaString).openStream()); diff --git serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java index af236f7..e07d06b 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java +++ serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerdeUtils.java @@ -117,26 +117,29 @@ public void getTypeFromNullableTypePositiveCase() { @Test(expected=AvroSerdeException.class) public void determineSchemaThrowsExceptionIfNoSchema() throws IOException, AvroSerdeException { + Configuration conf = new Configuration(); Properties prop = new Properties(); - AvroSerdeUtils.determineSchemaOrThrowException(prop); + AvroSerdeUtils.determineSchemaOrThrowException(conf, prop); } @Test public void determineSchemaFindsLiterals() throws Exception { String schema = TestAvroObjectInspectorGenerator.RECORD_SCHEMA; + Configuration conf = new Configuration(); Properties props = new Properties(); props.put(AvroSerdeUtils.SCHEMA_LITERAL, schema); Schema expected = AvroSerdeUtils.getSchemaFor(schema); - assertEquals(expected, AvroSerdeUtils.determineSchemaOrThrowException(props)); + assertEquals(expected, AvroSerdeUtils.determineSchemaOrThrowException(conf, props)); } @Test public void detemineSchemaTriesToOpenUrl() throws AvroSerdeException, IOException { + Configuration conf = new Configuration(); Properties props = new Properties(); props.put(AvroSerdeUtils.SCHEMA_URL, "not:///a.real.url"); try { - AvroSerdeUtils.determineSchemaOrThrowException(props); + AvroSerdeUtils.determineSchemaOrThrowException(conf, props); fail("Should have tried to open that URL"); } catch (AvroSerdeException e) { assertEquals("Unable to read schema from given path: not:///a.real.url", e.getMessage()); @@ -145,13 +148,14 @@ public void detemineSchemaTriesToOpenUrl() throws AvroSerdeException, IOExceptio @Test public void noneOptionWorksForSpecifyingSchemas() throws IOException, AvroSerdeException { + Configuration conf = new Configuration(); Properties props = new Properties(); // Combo 1: Both set to none props.put(SCHEMA_URL, SCHEMA_NONE); props.put(SCHEMA_LITERAL, SCHEMA_NONE); try { - determineSchemaOrThrowException(props); + determineSchemaOrThrowException(conf, props); fail("Should have thrown exception with none set for both url and literal"); } catch(AvroSerdeException he) { assertEquals(EXCEPTION_MESSAGE, he.getMessage()); @@ -161,7 +165,7 @@ public void noneOptionWorksForSpecifyingSchemas() throws IOException, AvroSerdeE props.put(SCHEMA_LITERAL, TestAvroObjectInspectorGenerator.RECORD_SCHEMA); Schema s; try { - s = determineSchemaOrThrowException(props); + s = determineSchemaOrThrowException(conf, props); assertNotNull(s); assertEquals(AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.RECORD_SCHEMA), s); } catch(AvroSerdeException he) { @@ -172,7 +176,7 @@ public void noneOptionWorksForSpecifyingSchemas() throws IOException, AvroSerdeE props.put(SCHEMA_LITERAL, SCHEMA_NONE); props.put(SCHEMA_URL, "not:///a.real.url"); try { - determineSchemaOrThrowException(props); + determineSchemaOrThrowException(conf, props); fail("Should have tried to open that bogus URL"); } catch (AvroSerdeException e) { assertEquals("Unable to read schema from given path: not:///a.real.url", e.getMessage());