diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/AccumuloStorageHandler.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/AccumuloStorageHandler.java index d2aecc17e5..cb5d5f0d81 100644 --- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/AccumuloStorageHandler.java +++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/AccumuloStorageHandler.java @@ -93,7 +93,7 @@ public void configureTableJobProperties(TableDesc desc, Map jobP protected String getTableName(Table table) throws MetaException { // Use TBLPROPERTIES - String tableName = table.getParameters().get(AccumuloSerDeParameters.TABLE_NAME); + String tableName = table.getParameters().get(AccumuloConnectionParameters.TABLE_NAME); if (null != tableName) { return tableName; @@ -101,7 +101,7 @@ protected String getTableName(Table table) throws MetaException { // Then try SERDEPROPERTIES tableName = table.getSd().getSerdeInfo().getParameters() - .get(AccumuloSerDeParameters.TABLE_NAME); + .get(AccumuloConnectionParameters.TABLE_NAME); if (null != tableName) { return tableName; @@ -132,7 +132,7 @@ protected String getIndexTableName(Table table) { protected String getTableName(TableDesc tableDesc) { Properties props = tableDesc.getProperties(); - String tableName = props.getProperty(AccumuloSerDeParameters.TABLE_NAME); + String tableName = props.getProperty(AccumuloConnectionParameters.TABLE_NAME); if (null != tableName) { return tableName; } @@ -191,11 +191,11 @@ public void configureInputJobProperties(TableDesc tableDesc, Map jobProperties.put(AccumuloSerDeParameters.COLUMN_MAPPINGS, props.getProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS)); - String tableName = props.getProperty(AccumuloSerDeParameters.TABLE_NAME); + String tableName = props.getProperty(AccumuloConnectionParameters.TABLE_NAME); if (null == tableName) { tableName = getTableName(tableDesc); } - jobProperties.put(AccumuloSerDeParameters.TABLE_NAME, + jobProperties.put(AccumuloConnectionParameters.TABLE_NAME, tableName); String useIterators = props.getProperty(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY); @@ -259,11 +259,11 @@ public void configureOutputJobProperties(TableDesc tableDesc, Map getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { try { @@ -108,14 +109,14 @@ protected static Boolean getStringEncoding(JobConf job) { protected AccumuloRecordWriter(JobConf job) throws AccumuloException, AccumuloSecurityException, IOException { this.isStringEncoded = AccumuloIndexedOutputFormat.getStringEncoding(job).booleanValue(); - this.simulate = AccumuloIndexedOutputFormat.getSimulationMode(job).booleanValue(); - this.createTables = AccumuloIndexedOutputFormat.canCreateTables(job).booleanValue(); + this.simulate = AccumuloOutputFormat.getSimulationMode(job).booleanValue(); + this.createTables = AccumuloOutputFormat.canCreateTables(job).booleanValue(); if (this.simulate) { LOG.info("Simulating output only. No writes to tables will occur"); } this.bws = new HashMap(); - String tname = AccumuloIndexedOutputFormat.getDefaultTableName(job); + String tname = AccumuloOutputFormat.getDefaultTableName(job); this.defaultTableName = tname == null ? null : new Text(tname); String iname = AccumuloIndexedOutputFormat.getIndexTableName(job); @@ -125,11 +126,11 @@ protected AccumuloRecordWriter(JobConf job) this.indexDef = createIndexDefinition(job, tname, iname); } if (!this.simulate) { - this.conn = AccumuloIndexedOutputFormat.getInstance(job) - .getConnector(AccumuloIndexedOutputFormat.getPrincipal(job), - AccumuloIndexedOutputFormat.getAuthenticationToken(job)); + this.conn = AccumuloOutputFormat.getInstance(job) + .getConnector(AccumuloOutputFormat.getPrincipal(job), + AccumuloOutputFormat.getAuthenticationToken(job)); this.mtbw = this.conn.createMultiTableBatchWriter( - AccumuloIndexedOutputFormat.getBatchWriterOptions(job)); + AccumuloOutputFormat.getBatchWriterOptions(job)); } } @@ -141,6 +142,7 @@ AccumuloIndexDefinition createIndexDefinition(JobConf job, String tname, String return def; } + @Override public void write(Text table, Mutation mutation) throws IOException { if(table == null || table.toString().isEmpty()) { table = this.defaultTableName; @@ -150,7 +152,7 @@ public void write(Text table, Mutation mutation) throws IOException { throw new IOException("No table or default table specified. Try simulation mode next time"); } else { ++this.mutCount; - this.valCount += (long)mutation.size(); + this.valCount += mutation.size(); this.printMutation(table, mutation); if(!this.simulate) { if(!this.bws.containsKey(table)) { @@ -171,7 +173,7 @@ public void write(Text table, Mutation mutation) throws IOException { } try { - ((BatchWriter)this.bws.get(table)).addMutation(mutation); + this.bws.get(table).addMutation(mutation); } catch (MutationsRejectedException var4) { throw new IOException(var4); } @@ -291,6 +293,7 @@ private String hexDump(byte[] ba) { return sb.toString(); } + @Override public void close(Reporter reporter) throws IOException { LOG.debug("mutations written: {}, values written: {}", this.mutCount, this.valCount); if(!this.simulate) { @@ -305,7 +308,7 @@ public void close(Reporter reporter) throws IOException { for(Iterator itr = var7.getAuthorizationFailuresMap().entrySet().iterator(); itr.hasNext(); ((Set)secCodes).addAll((Collection)ke.getValue())) { ke = (Map.Entry)itr.next(); - secCodes = (Set)tables.get(((KeyExtent)ke.getKey()).getTableId().toString()); + secCodes = tables.get(((KeyExtent)ke.getKey()).getTableId().toString()); if(secCodes == null) { secCodes = new HashSet(); tables.put(((KeyExtent)ke.getKey()).getTableId().toString(), secCodes); diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloRecordReader.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloRecordReader.java index 45607cbecf..3a21604ebd 100644 --- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloRecordReader.java +++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloRecordReader.java @@ -26,6 +26,7 @@ import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.WholeRowIterator; import org.apache.accumulo.core.util.PeekingIterator; import org.apache.hadoop.hive.accumulo.AccumuloHiveRow; import org.apache.hadoop.hive.accumulo.predicate.PrimitiveComparisonFilter; @@ -94,7 +95,7 @@ public boolean next(Text rowKey, AccumuloHiveRow row) throws IOException { pushToValue(keys, values, row); } else { for (int i = 0; i < iteratorCount; i++) { // each iterator creates a level of encoding. - SortedMap decoded = PrimitiveComparisonFilter.decodeRow(keys.get(0), + SortedMap decoded = WholeRowIterator.decodeRow(keys.get(0), values.get(0)); keys = Lists.newArrayList(decoded.keySet()); values = Lists.newArrayList(decoded.values()); diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableInputFormat.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableInputFormat.java index af64eac1fc..7512473a13 100644 --- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableInputFormat.java +++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableInputFormat.java @@ -31,8 +31,10 @@ import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.mapred.AbstractInputFormat; import org.apache.accumulo.core.client.mapred.AccumuloInputFormat; import org.apache.accumulo.core.client.mapred.AccumuloRowInputFormat; +import org.apache.accumulo.core.client.mapred.InputFormatBase; import org.apache.accumulo.core.client.mapred.RangeInputSplit; import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase; import org.apache.accumulo.core.client.mock.MockInstance; @@ -343,25 +345,25 @@ protected void configure(JobConf conf, Instance instance, Connector connector, // verify were correctly called via Mockito protected void setInputTableName(JobConf conf, String tableName) { - AccumuloInputFormat.setInputTableName(conf, tableName); + InputFormatBase.setInputTableName(conf, tableName); } protected void setScanAuthorizations(JobConf conf, Authorizations auths) { - AccumuloInputFormat.setScanAuthorizations(conf, auths); + AbstractInputFormat.setScanAuthorizations(conf, auths); } protected void addIterators(JobConf conf, List iterators) { for (IteratorSetting is : iterators) { - AccumuloInputFormat.addIterator(conf, is); + InputFormatBase.addIterator(conf, is); } } protected void setRanges(JobConf conf, Collection ranges) { - AccumuloInputFormat.setRanges(conf, ranges); + InputFormatBase.setRanges(conf, ranges); } protected void fetchColumns(JobConf conf, Set> cfCqPairs) { - AccumuloInputFormat.fetchColumns(conf, cfCqPairs); + InputFormatBase.fetchColumns(conf, cfCqPairs); } /** diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableOutputFormat.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableOutputFormat.java index b399bc6262..e7c4fa5f2f 100644 --- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableOutputFormat.java +++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableOutputFormat.java @@ -21,7 +21,6 @@ import java.io.IOException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.mapred.AccumuloOutputFormat; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Mutation; import org.apache.hadoop.fs.FileSystem; @@ -64,7 +63,7 @@ public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException protected void configureAccumuloOutputFormat(JobConf job) throws IOException { AccumuloConnectionParameters cnxnParams = getConnectionParams(job); - final String tableName = job.get(AccumuloSerDeParameters.TABLE_NAME); + final String tableName = job.get(AccumuloConnectionParameters.TABLE_NAME); // Make sure we actually go the table name Preconditions.checkNotNull(tableName, @@ -110,7 +109,7 @@ protected void configureAccumuloOutputFormat(JobConf job) throws IOException { // Non-static methods to wrap the static AccumuloOutputFormat methods to enable testing protected void setDefaultAccumuloTableName(JobConf conf, String tableName) { - AccumuloIndexedOutputFormat.setDefaultTableName(conf, tableName); + AccumuloOutputFormat.setDefaultTableName(conf, tableName); } protected void setAccumuloIndexTableName(JobConf conf, String indexTableName) { diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloRangeGenerator.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloRangeGenerator.java index 17963820ed..e9979788db 100644 --- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloRangeGenerator.java +++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloRangeGenerator.java @@ -45,7 +45,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.UTF8; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/DoubleCompare.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/DoubleCompare.java index 53fd6e34b0..d4a478114c 100644 --- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/DoubleCompare.java +++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/DoubleCompare.java @@ -33,6 +33,7 @@ /** * */ + @Override public void init(byte[] constant) { this.constant = serialize(constant); } @@ -40,6 +41,7 @@ public void init(byte[] constant) { /** * @return BigDecimal holding double byte [] value */ + @Override public BigDecimal serialize(byte[] value) { try { return new BigDecimal(ByteBuffer.wrap(value).asDoubleBuffer().get()); diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/IntCompare.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/IntCompare.java index 826355bde5..c35889321c 100644 --- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/IntCompare.java +++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/IntCompare.java @@ -69,6 +69,7 @@ public boolean like(byte[] value) { throw new UnsupportedOperationException("Like not supported for " + getClass().getName()); } + @Override public Integer serialize(byte[] value) { try { return ByteBuffer.wrap(value).asIntBuffer().get(); diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/LongCompare.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/LongCompare.java index afd03d6238..1061840ada 100644 --- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/LongCompare.java +++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/LongCompare.java @@ -70,6 +70,7 @@ public boolean like(byte[] value) { throw new UnsupportedOperationException("Like not supported for " + getClass().getName()); } + @Override public Long serialize(byte[] value) { try { return ByteBuffer.wrap(value).asLongBuffer().get(); diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/StringCompare.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/StringCompare.java index 3d6d55c05e..6184248754 100644 --- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/StringCompare.java +++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/compare/StringCompare.java @@ -73,6 +73,7 @@ public boolean like(byte[] value) { return match; } + @Override public String serialize(byte[] value) { return new String(value); } diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/CompositeAccumuloRowIdFactory.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/CompositeAccumuloRowIdFactory.java index a3be5a3319..6101ec3482 100644 --- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/CompositeAccumuloRowIdFactory.java +++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/CompositeAccumuloRowIdFactory.java @@ -64,7 +64,7 @@ public void addDependencyJars(Configuration jobConf) throws IOException { @Override public T createRowId(ObjectInspector inspector) throws SerDeException { try { - return (T) constructor.newInstance(inspector, this.properties, + return constructor.newInstance(inspector, this.properties, this.accumuloSerDeParams.getConf()); } catch (Exception e) { throw new SerDeException(e); diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloIndexLexicoder.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloIndexLexicoder.java index b23ed494ca..996feb585f 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloIndexLexicoder.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloIndexLexicoder.java @@ -20,9 +20,7 @@ import org.apache.accumulo.core.client.lexicoder.BigIntegerLexicoder; import org.apache.accumulo.core.client.lexicoder.DoubleLexicoder; import org.apache.accumulo.core.client.lexicoder.IntegerLexicoder; -import org.apache.accumulo.core.client.lexicoder.LongLexicoder; import org.apache.hadoop.hive.serde.serdeConstants; -import org.junit.Assert; import org.junit.Test; import java.math.BigInteger; @@ -30,7 +28,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; /** * diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java index 7b12362682..e4af4c8724 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java @@ -66,7 +66,7 @@ public void testTablePropertiesPassedToOutputJobProperties() { props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq1,cf:cq2,cf:cq3"); props.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string:int:string"); props.setProperty(serdeConstants.LIST_COLUMNS, "name,age,email"); - props.setProperty(AccumuloSerDeParameters.TABLE_NAME, "table"); + props.setProperty(AccumuloConnectionParameters.TABLE_NAME, "table"); props.setProperty(AccumuloSerDeParameters.VISIBILITY_LABEL_KEY, "foo"); Mockito.when(tableDesc.getProperties()).thenReturn(props); @@ -80,9 +80,9 @@ public void testTablePropertiesPassedToOutputJobProperties() { jobProperties.get(AccumuloSerDeParameters.COLUMN_MAPPINGS)); Assert.assertTrue("Job properties did not contain accumulo table name", - jobProperties.containsKey(AccumuloSerDeParameters.TABLE_NAME)); - Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.TABLE_NAME), - jobProperties.get(AccumuloSerDeParameters.TABLE_NAME)); + jobProperties.containsKey(AccumuloConnectionParameters.TABLE_NAME)); + Assert.assertEquals(props.getProperty(AccumuloConnectionParameters.TABLE_NAME), + jobProperties.get(AccumuloConnectionParameters.TABLE_NAME)); Assert.assertTrue("Job properties did not contain visibility label", jobProperties.containsKey(AccumuloSerDeParameters.VISIBILITY_LABEL_KEY)); @@ -97,7 +97,7 @@ public void testTablePropertiesPassedToInputJobProperties() { Map jobProperties = new HashMap(); props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq1,cf:cq2,cf:cq3"); - props.setProperty(AccumuloSerDeParameters.TABLE_NAME, "table"); + props.setProperty(AccumuloConnectionParameters.TABLE_NAME, "table"); props.setProperty(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY, "true"); props .setProperty(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE, ColumnEncoding.BINARY.getName()); @@ -113,9 +113,9 @@ public void testTablePropertiesPassedToInputJobProperties() { Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS), jobProperties.get(AccumuloSerDeParameters.COLUMN_MAPPINGS)); - Assert.assertTrue(jobProperties.containsKey(AccumuloSerDeParameters.TABLE_NAME)); - Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.TABLE_NAME), - jobProperties.get(AccumuloSerDeParameters.TABLE_NAME)); + Assert.assertTrue(jobProperties.containsKey(AccumuloConnectionParameters.TABLE_NAME)); + Assert.assertEquals(props.getProperty(AccumuloConnectionParameters.TABLE_NAME), + jobProperties.get(AccumuloConnectionParameters.TABLE_NAME)); Assert.assertTrue(jobProperties.containsKey(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY)); Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY), @@ -137,7 +137,7 @@ public void testNonBooleanIteratorPushdownValue() { Map jobProperties = new HashMap(); props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq1,cf:cq2,cf:cq3"); - props.setProperty(AccumuloSerDeParameters.TABLE_NAME, "table"); + props.setProperty(AccumuloConnectionParameters.TABLE_NAME, "table"); props.setProperty(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY, "foo"); Mockito.when(tableDesc.getProperties()).thenReturn(props); @@ -152,7 +152,7 @@ public void testEmptyIteratorPushdownValue() { Map jobProperties = new HashMap(); props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq1,cf:cq2,cf:cq3"); - props.setProperty(AccumuloSerDeParameters.TABLE_NAME, "table"); + props.setProperty(AccumuloConnectionParameters.TABLE_NAME, "table"); props.setProperty(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY, ""); Mockito.when(tableDesc.getProperties()).thenReturn(props); diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestHiveAccumuloHelper.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestHiveAccumuloHelper.java index af561e0c3d..359ef71ff1 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestHiveAccumuloHelper.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestHiveAccumuloHelper.java @@ -19,12 +19,10 @@ import static org.junit.Assert.assertEquals; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Collection; import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.mapred.AccumuloInputFormat; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTableInputFormat.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTableInputFormat.java index 95cc27d9da..f2cfa17bfd 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTableInputFormat.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTableInputFormat.java @@ -47,6 +47,7 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.WholeRowIterator; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.util.Pair; @@ -85,6 +86,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; +import org.mockito.Matchers; import org.mockito.Mockito; import com.google.common.collect.Sets; @@ -117,12 +119,12 @@ public void createMockKeyValues() throws Exception { mockInstance = new MockInstance(test.getMethodName()); inputformat = new HiveAccumuloTableInputFormat(); conf = new JobConf(); - conf.set(AccumuloSerDeParameters.TABLE_NAME, TEST_TABLE); - conf.set(AccumuloSerDeParameters.USE_MOCK_INSTANCE, "true"); - conf.set(AccumuloSerDeParameters.INSTANCE_NAME, test.getMethodName()); - conf.set(AccumuloSerDeParameters.USER_NAME, USER); - conf.set(AccumuloSerDeParameters.USER_PASS, PASS); - conf.set(AccumuloSerDeParameters.ZOOKEEPERS, "localhost:2181"); // not used for mock, but + conf.set(AccumuloConnectionParameters.TABLE_NAME, TEST_TABLE); + conf.set(AccumuloConnectionParameters.USE_MOCK_INSTANCE, "true"); + conf.set(AccumuloConnectionParameters.INSTANCE_NAME, test.getMethodName()); + conf.set(AccumuloConnectionParameters.USER_NAME, USER); + conf.set(AccumuloConnectionParameters.USER_PASS, PASS); + conf.set(AccumuloConnectionParameters.ZOOKEEPERS, "localhost:2181"); // not used for mock, but // required by input format. columnNames = Arrays.asList("name", "sid", "dgrs", "mills"); @@ -273,9 +275,9 @@ public void testDegreesAndMillis() throws Exception { boolean foundSid = false; boolean foundDegrees = false; boolean foundMillis = false; - SortedMap items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue()); + SortedMap items = WholeRowIterator.decodeRow(kv.getKey(), kv.getValue()); for (Map.Entry item : items.entrySet()) { - SortedMap nestedItems = PrimitiveComparisonFilter.decodeRow(item.getKey(), + SortedMap nestedItems = WholeRowIterator.decodeRow(item.getKey(), item.getValue()); for (Map.Entry nested : nestedItems.entrySet()) { if (nested.getKey().getRow().toString().equals("r3")) { @@ -319,7 +321,7 @@ public void testGreaterThan1Sid() throws Exception { boolean foundSid = false; boolean foundDegrees = false; boolean foundMillis = false; - SortedMap items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue()); + SortedMap items = WholeRowIterator.decodeRow(kv.getKey(), kv.getValue()); for (Map.Entry item : items.entrySet()) { if (item.getKey().getRow().toString().equals("r2")) { foundMark = true; @@ -360,7 +362,7 @@ public void testNameEqualBrian() throws Exception { boolean foundDegrees = false; boolean foundMillis = false; for (Map.Entry kv : scan) { - SortedMap items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue()); + SortedMap items = WholeRowIterator.decodeRow(kv.getKey(), kv.getValue()); for (Map.Entry item : items.entrySet()) { assertEquals(item.getKey().getRow().toString(), "r1"); if (item.getKey().getColumnQualifier().equals(NAME)) { @@ -413,7 +415,7 @@ public void testIteratorNotInSplitsCompensation() throws Exception { // Mock out the predicate handler because it's just easier AccumuloPredicateHandler predicateHandler = Mockito.mock(AccumuloPredicateHandler.class); Mockito.when( - predicateHandler.getIterators(Mockito.any(JobConf.class), Mockito.any(ColumnMapper.class))) + predicateHandler.getIterators(Matchers.any(JobConf.class), Matchers.any(ColumnMapper.class))) .thenReturn(Arrays.asList(is)); // Set it on our inputformat diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTableOutputFormat.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTableOutputFormat.java index 1dd2b8cac5..11fca54233 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTableOutputFormat.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTableOutputFormat.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map.Entry; @@ -60,7 +59,6 @@ import org.apache.hadoop.mapred.RecordWriter; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -422,7 +420,7 @@ public void testWriteMap() throws Exception { stringOI, stringOI, (byte) ',', (byte) ':', serDeParams.getNullSequence(), serDeParams.isEscaped(), serDeParams.getEscapeChar()); - LazySimpleStructObjectInspector structOI = (LazySimpleStructObjectInspector) LazyObjectInspectorFactory + LazySimpleStructObjectInspector structOI = LazyObjectInspectorFactory .getLazySimpleStructObjectInspector(Arrays.asList("row", "data"), Arrays.asList(stringOI, mapOI), (byte) ' ', serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(), diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTypes.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTypes.java index 2eeb7deecb..040e0adfa5 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTypes.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTypes.java @@ -35,6 +35,7 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.accumulo.AccumuloConnectionParameters; import org.apache.hadoop.hive.accumulo.AccumuloHiveConstants; import org.apache.hadoop.hive.accumulo.AccumuloHiveRow; import org.apache.hadoop.hive.accumulo.serde.AccumuloSerDeParameters; @@ -123,12 +124,12 @@ public void testBinaryTypes() throws Exception { HiveAccumuloTableInputFormat inputformat = new HiveAccumuloTableInputFormat(); JobConf conf = new JobConf(); - conf.set(AccumuloSerDeParameters.TABLE_NAME, tableName); - conf.set(AccumuloSerDeParameters.USE_MOCK_INSTANCE, "true"); - conf.set(AccumuloSerDeParameters.INSTANCE_NAME, test.getMethodName()); - conf.set(AccumuloSerDeParameters.USER_NAME, user); - conf.set(AccumuloSerDeParameters.USER_PASS, pass); - conf.set(AccumuloSerDeParameters.ZOOKEEPERS, "localhost:2181"); // not used for mock, but + conf.set(AccumuloConnectionParameters.TABLE_NAME, tableName); + conf.set(AccumuloConnectionParameters.USE_MOCK_INSTANCE, "true"); + conf.set(AccumuloConnectionParameters.INSTANCE_NAME, test.getMethodName()); + conf.set(AccumuloConnectionParameters.USER_NAME, user); + conf.set(AccumuloConnectionParameters.USER_PASS, pass); + conf.set(AccumuloConnectionParameters.ZOOKEEPERS, "localhost:2181"); // not used for mock, but // required by input format. conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, AccumuloHiveConstants.ROWID @@ -474,12 +475,12 @@ public void testUtf8Types() throws Exception { HiveAccumuloTableInputFormat inputformat = new HiveAccumuloTableInputFormat(); JobConf conf = new JobConf(); - conf.set(AccumuloSerDeParameters.TABLE_NAME, tableName); - conf.set(AccumuloSerDeParameters.USE_MOCK_INSTANCE, "true"); - conf.set(AccumuloSerDeParameters.INSTANCE_NAME, test.getMethodName()); - conf.set(AccumuloSerDeParameters.USER_NAME, user); - conf.set(AccumuloSerDeParameters.USER_PASS, pass); - conf.set(AccumuloSerDeParameters.ZOOKEEPERS, "localhost:2181"); // not used for mock, but + conf.set(AccumuloConnectionParameters.TABLE_NAME, tableName); + conf.set(AccumuloConnectionParameters.USE_MOCK_INSTANCE, "true"); + conf.set(AccumuloConnectionParameters.INSTANCE_NAME, test.getMethodName()); + conf.set(AccumuloConnectionParameters.USER_NAME, user); + conf.set(AccumuloConnectionParameters.USER_PASS, pass); + conf.set(AccumuloConnectionParameters.ZOOKEEPERS, "localhost:2181"); // not used for mock, but // required by input format. conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, AccumuloHiveConstants.ROWID diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/predicate/TestAccumuloRangeGenerator.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/predicate/TestAccumuloRangeGenerator.java index 4975fa0d5e..6fc691ac2b 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/predicate/TestAccumuloRangeGenerator.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/predicate/TestAccumuloRangeGenerator.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hive.accumulo.TestAccumuloDefaultIndexScanner; import org.apache.hadoop.hive.accumulo.columns.ColumnEncoding; import org.apache.hadoop.hive.accumulo.columns.HiveAccumuloRowIdColumnMapping; -import org.apache.hadoop.hive.accumulo.serde.AccumuloSerDeParameters; import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; @@ -39,7 +38,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToString; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan; @@ -47,7 +45,6 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPPlus; -import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.junit.Assert; import org.junit.Before; diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java index ed28e18140..22d706544b 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java @@ -49,7 +49,7 @@ public Object getField(int fieldID) { // The separator for the hive row would be using \x02, so the separator for this struct would be // \x02 + 1 = \x03 - char separator = (char) ((int) oi.getSeparator() + 1); + char separator = (char) (oi.getSeparator() + 1); log.info("Separator: " + String.format("%04x", (int) separator)); diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloRowSerializer.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloRowSerializer.java index 85bc960237..9769a8cdc0 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloRowSerializer.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloRowSerializer.java @@ -50,6 +50,7 @@ import org.apache.hadoop.io.Text; import org.junit.Assert; import org.junit.Test; +import org.mockito.Matchers; import org.mockito.Mockito; import com.google.common.base.Joiner; @@ -74,13 +75,13 @@ public void testBufferResetBeforeUse() throws IOException { String object = "hello"; Mockito.when( - serializer.getSerializedValue(Mockito.any(ObjectInspector.class), Mockito.any(), - Mockito.any(ByteStream.Output.class), Mockito.any(ColumnMapping.class))) + serializer.getSerializedValue(Matchers.any(ObjectInspector.class), Matchers.any(), + Matchers.any(ByteStream.Output.class), Matchers.any(ColumnMapping.class))) .thenCallRealMethod(); Mockito.when(fieldObjectInspector.getCategory()).thenReturn(ObjectInspector.Category.PRIMITIVE); Mockito.when(fieldObjectInspector.getPrimitiveCategory()).thenReturn(PrimitiveCategory.STRING); - Mockito.when(fieldObjectInspector.getPrimitiveWritableObject(Mockito.any(Object.class))) + Mockito.when(fieldObjectInspector.getPrimitiveWritableObject(Matchers.any(Object.class))) .thenReturn(new Text(object)); Mockito.when(mapping.getEncoding()).thenReturn(ColumnEncoding.STRING); @@ -128,7 +129,7 @@ public void testBinarySerialization() throws IOException, SerDeException { 'l', 'u', 'e'}); obj.init(byteRef, 0, byteRef.getData().length); - Mutation m = (Mutation) serializer.serialize(obj, oi); + Mutation m = serializer.serialize(obj, oi); Assert.assertArrayEquals("row1".getBytes(), m.getRow()); @@ -197,7 +198,7 @@ public void testVisibilityLabel() throws IOException, SerDeException { 'l', 'u', 'e'}); obj.init(byteRef, 0, byteRef.getData().length); - Mutation m = (Mutation) serializer.serialize(obj, oi); + Mutation m = serializer.serialize(obj, oi); Assert.assertArrayEquals("row1".getBytes(), m.getRow()); @@ -262,7 +263,7 @@ public void testMapSerialization() throws IOException, SerDeException { stringOI, stringOI, (byte) ',', (byte) ':', serDeParams.getNullSequence(), serDeParams.isEscaped(), serDeParams.getEscapeChar()); - LazySimpleStructObjectInspector structOI = (LazySimpleStructObjectInspector) LazyObjectInspectorFactory + LazySimpleStructObjectInspector structOI = LazyObjectInspectorFactory .getLazySimpleStructObjectInspector(columns, Arrays.asList(stringOI, mapOI), (byte) ' ', serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(), serDeParams.getEscapeChar()); @@ -278,7 +279,7 @@ public void testMapSerialization() throws IOException, SerDeException { byteRef.setData("row1 cq1:10,cq2:20,cq3:value".getBytes()); obj.init(byteRef, 0, byteRef.getData().length); - Mutation m = (Mutation) serializer.serialize(obj, structOI); + Mutation m = serializer.serialize(obj, structOI); Assert.assertArrayEquals("row1".getBytes(), m.getRow()); diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloSerDe.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloSerDe.java index 58cac882da..897ea9e2b3 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloSerDe.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloSerDe.java @@ -401,13 +401,13 @@ public void testStructOfMapSerialization() throws IOException, SerDeException { stringOI, stringOI, seps[3], seps[4], serDeParams.getNullSequence(), serDeParams.isEscaped(), serDeParams.getEscapeChar()); - LazySimpleStructObjectInspector rowStructOI = (LazySimpleStructObjectInspector) LazyObjectInspectorFactory + LazySimpleStructObjectInspector rowStructOI = LazyObjectInspectorFactory .getLazySimpleStructObjectInspector(structColNames, - Arrays. asList(mapOI, mapOI), (byte) seps[2], + Arrays. asList(mapOI, mapOI), seps[2], serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(), serDeParams.getEscapeChar()); - LazySimpleStructObjectInspector structOI = (LazySimpleStructObjectInspector) LazyObjectInspectorFactory + LazySimpleStructObjectInspector structOI = LazyObjectInspectorFactory .getLazySimpleStructObjectInspector(columns, Arrays.asList(rowStructOI, stringOI), seps[1], serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(), serDeParams.getEscapeChar()); diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestDefaultAccumuloRowIdFactory.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestDefaultAccumuloRowIdFactory.java index e0a8db9df3..5f5e3d99d8 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestDefaultAccumuloRowIdFactory.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestDefaultAccumuloRowIdFactory.java @@ -93,11 +93,11 @@ public void testCorrectComplexInspectors() throws SerDeException { Assert.assertEquals(LazyMapObjectInspector.class, OIs.get(1).getClass()); LazySimpleStructObjectInspector structOI = (LazySimpleStructObjectInspector) OIs.get(0); - Assert.assertEquals(2, (int) structOI.getSeparator()); + Assert.assertEquals(2, structOI.getSeparator()); LazyMapObjectInspector mapOI = (LazyMapObjectInspector) OIs.get(1); - Assert.assertEquals(2, (int) mapOI.getItemSeparator()); - Assert.assertEquals(3, (int) mapOI.getKeyValueSeparator()); + Assert.assertEquals(2, mapOI.getItemSeparator()); + Assert.assertEquals(3, mapOI.getKeyValueSeparator()); } @Test