[AVRO-2602] Updating breaks backward compatibility by throwing AvroTypeException in some cases - ASF JIRA

Details

Type: Bug
Status: Open
Priority: Major
Resolution: Unresolved
Affects Version/s: 1.8.0, 1.8.1, 1.9.0, 1.8.2, 1.9.1
Fix Version/s: None
Component/s: None
Labels:
None

Description

When I try to update Avro from 1.7.7 to the newer version. The following code:

	@Test
	public void Demo() throws IOException{
		
        Schema schema = Schema.parse("{\"type\": \"enum\", \"name\": \"MyEnum\", \"symbols\": [\"A\", \"B\", \"C\"]}");

        Map<String, String> expectedA = new java.util.HashMap();
        expectedA.put("", "A");
        assertEquals(expectedA, write(schema, "A"));
		
	}
	
    private Map<String, String> write(Schema schema, Object datum) throws IOException {
        DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
        Map<String, String> out = new java.util.HashMap();
        KeyValueEncoder encoder = new KeyValueEncoder(schema, out);
        writer.write(datum, encoder);
        return out;
    }
    

}

class KeyValueEncoder extends ParsingEncoder implements Parser.ActionHandler {
	  final Parser parser;
	  protected BitSet isEmpty = new BitSet();
	  private java.util.Map<String, String> out;
	  
	  KeyValueEncoder(Schema sc, java.util.Map<String, String> out) throws IOException {
	    configure(out);
	    this.parser =
	      new Parser(new JsonGrammarGenerator().generate(sc), this);
	  }
	  


	  public void flush() throws IOException {
	    // Do nothing
	  }

	  public KeyValueEncoder configure(java.util.Map<String, String> newOut) throws IOException {
	    this.out = newOut;
	    return this;
	  }
	  
	  /////////////////////////////////////////////////////////////////////////////

	  @Override
	  public void writeNull() throws IOException {
	    parser.advance(Symbol.NULL);
	  }

	  @Override
	  public void writeBoolean(boolean b) throws IOException {
	    parser.advance(Symbol.BOOLEAN);
	    out.put(getKeyPathString(), Boolean.toString(b));
	  }

	  @Override
	  public void writeInt(int n) throws IOException {
	    parser.advance(Symbol.INT);
	    out.put(getKeyPathString(), Integer.toString(n));
	  }

	  @Override
	  public void writeLong(long n) throws IOException {
	    parser.advance(Symbol.LONG);
	    out.put(getKeyPathString(), Long.toString(n));
	  }

	  @Override
	  public void writeFloat(float f) throws IOException {
	    parser.advance(Symbol.FLOAT);
	    out.put(getKeyPathString(), Float.toString(f));
	  }

	  @Override
	  public void writeDouble(double d) throws IOException {
	    parser.advance(Symbol.DOUBLE);
	    out.put(getKeyPathString(), Double.toString(d));
	  }

	  @Override
	  public void writeString(Utf8 utf8) throws IOException {
	    writeString(utf8.toString());
	  }
	  
	  @Override 
	  public void writeString(String str) throws IOException {
	    parser.advance(Symbol.STRING);
	    trace("writeString(" + str + ")");
	    if (parser.topSymbol() == Symbol.MAP_KEY_MARKER) {
	      parser.advance(Symbol.MAP_KEY_MARKER);
	      pushKeyPathComponent(str);
	      // out.writeFieldName(str);
	    } else {
	      out.put(getKeyPathString(), str);
	    }
	  }

	  @Override
	  public void writeBytes(ByteBuffer bytes) throws IOException {
	    if (bytes.hasArray()) {
	      writeBytes(bytes.array(), bytes.position(), bytes.remaining());
	    } else {
	      byte[] b = new byte[bytes.remaining()];
	      for (int i = 0; i < b.length; i++) {
	        b[i] = bytes.get();
	      }
	      writeBytes(b);
	    }
	  }

	  @Override
	  public void writeBytes(byte[] bytes, int start, int len) throws IOException {
	    parser.advance(Symbol.BYTES);
	    writeByteArray(bytes, start, len);
	  }

	  private void writeByteArray(byte[] bytes, int start, int len) throws IOException {
	    out.put(getKeyPathString(), new String(bytes, start, len, "UTF-8"));
	  }

	  @Override
	  public void writeFixed(byte[] bytes, int start, int len) throws IOException {
	    throw new IOException("Fixed encoding is not implemented");
	  }

	  @Override
	  public void writeEnum(int e) throws IOException {
	    parser.advance(Symbol.ENUM);
	    Symbol.EnumLabelsAction top = (Symbol.EnumLabelsAction) parser.popSymbol();
	    trace("writeEnum(" + e + " : " + top.getLabel(e) + ")");
	    if (e < 0 || e >= top.size) {
	      throw new AvroTypeException(
	          "Enumeration out of range: max is " +
	          top.size + " but received " + e);
	    }
	    out.put(getKeyPathString(), top.getLabel(e));
	  }

	  @Override
	  public void writeArrayStart() throws IOException {
	    throw new IOException("Array encoding is not implemented");
	  }

	  @Override
	  public void writeArrayEnd() throws IOException {
	    if (! isEmpty.get(pos)) {
	      parser.advance(Symbol.ITEM_END);
	    }
	    pop();
	    parser.advance(Symbol.ARRAY_END);
	    // out.writeEndArray();
	  }

	  @Override
	  public void writeMapStart() throws IOException {
	    push();
	    isEmpty.set(depth());

	    parser.advance(Symbol.MAP_START);
//	     out.writeStartObject();
	  }

	  @Override
	  public void writeMapEnd() throws IOException {
	    if (! isEmpty.get(pos)) {
	      parser.advance(Symbol.ITEM_END);
	      popKeyPathComponent();
	    }
	    pop();

	    parser.advance(Symbol.MAP_END);
	    // out.writeEndObject();
	  }

	  @Override
	  public void startItem() throws IOException {
	    trace("startItem");
	    if (! isEmpty.get(pos)) {
	      parser.advance(Symbol.ITEM_END);
	      popKeyPathComponent();
	    }
	    super.startItem();
	    isEmpty.clear(depth());
	  }

	  @Override
	  public void writeIndex(int unionIndex) throws IOException {
	    parser.advance(Symbol.UNION);
	    Symbol.Alternative top = (Symbol.Alternative) parser.popSymbol();
	    Symbol symbol = top.getSymbol(unionIndex);
	    if (symbol != Symbol.NULL) {
	      pushKeyPathComponent(top.getLabel(unionIndex));
	      parser.pushSymbol(Symbol.UNION_END);
	    }
	    parser.pushSymbol(symbol);
	  }

	  public Symbol doAction(Symbol input, Symbol top) throws IOException {
	    if (top instanceof Symbol.FieldAdjustAction) {
	      Symbol.FieldAdjustAction fa = (Symbol.FieldAdjustAction) top;
	      pushKeyPathComponent(fa.fname);
	    } else if (top == Symbol.RECORD_START) {
	      // out.writeStartObject();
	    } else if (top == Symbol.RECORD_END || top == Symbol.UNION_END) {
	      // out.writeEndObject();
	    } else if (top == Symbol.FIELD_END) {
	      popKeyPathComponent();
	    } else {
	      throw new AvroTypeException("Unknown action symbol " + top);
	    }
	    return null;
	  }

	  /////////////////////////////////////////////////////////////////////////////
	  // Key path

	  private java.util.ArrayList<String> keyPath = new java.util.ArrayList();

	  private void pushKeyPathComponent(String component) {
	    keyPath.add(component);
	  }

	  private void popKeyPathComponent() {
	    keyPath.remove(keyPath.size() - 1);
	  }

	  private String getKeyPathString() {
	    return StringUtils.join(keyPath, '|');
	  }

	  /////////////////////////////////////////////////////////////////////////////

	  private void trace(String s) {
	    System.out.println(s + ":\t topSymbol=" + parser.topSymbol() + " keyPath=" + getKeyPathString());
	  }
	
}

Throws an AvroTypeException error:

org.apache.avro.AvroTypeException: Not an enum: A for schema: {"type":"enum","name":"MyEnum","symbols":["A","B","C"]}
    at org.apache.avro.generic.GenericDatumWriter.writeEnum(GenericDatumWriter.java:218)
    at org.apache.avro.generic.GenericDatumWriter.writeWithoutConversion(GenericDatumWriter.java:133)
    at org.apache.avro.generic.GenericDatumWriter.write(GenericDatumWriter.java:82)
    at org.apache.avro.generic.GenericDatumWriter.write(GenericDatumWriter.java:72)
    at UTD.SeLab.BBI.BugDetection.TestAvro.write(TestAvro.java:42)
    at UTD.SeLab.BBI.BugDetection.TestAvro.Demo(TestAvro.java:34)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
    at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
    at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
    at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
    at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:86)
    at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:459)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:678)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:382)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:192)

Attachments

Issue Links

is related to

AVRO-997 Generic API should require GenericEnumSymbol when writing Avro Enums

Closed

Updating breaks backward compatibility by throwing AvroTypeException in some cases

Details

Description

Attachments

Issue Links

Activity

People

Dates