Index: src/test/org/apache/hadoop/hbase/thrift/DisabledTestThriftServer.java =================================================================== --- src/test/org/apache/hadoop/hbase/thrift/DisabledTestThriftServer.java (revision 785929) +++ src/test/org/apache/hadoop/hbase/thrift/DisabledTestThriftServer.java (working copy) @@ -1,380 +0,0 @@ -/** - * Copyright 2008-2009 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.thrift; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hbase.HBaseClusterTestCase; -import org.apache.hadoop.hbase.thrift.generated.BatchMutation; -import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor; -import org.apache.hadoop.hbase.thrift.generated.IllegalArgument; -import org.apache.hadoop.hbase.thrift.generated.Mutation; -import org.apache.hadoop.hbase.thrift.generated.TCell; -import org.apache.hadoop.hbase.thrift.generated.TRowResult; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * Unit testing for ThriftServer.HBaseHandler, a part of the - * org.apache.hadoop.hbase.thrift package. - */ -public class DisabledTestThriftServer extends HBaseClusterTestCase { - - // Static names for tables, columns, rows, and values - private static byte[] tableAname = Bytes.toBytes("tableA"); - private static byte[] tableBname = Bytes.toBytes("tableB"); - private static byte[] columnAname = Bytes.toBytes("columnA:"); - private static byte[] columnBname = Bytes.toBytes("columnB:"); - private static byte[] badColumnName = Bytes.toBytes("noColon:"); - private static byte[] rowAname = Bytes.toBytes("rowA"); - private static byte[] rowBname = Bytes.toBytes("rowB"); - private static byte[] valueAname = Bytes.toBytes("valueA"); - private static byte[] valueBname = Bytes.toBytes("valueB"); - private static byte[] valueCname = Bytes.toBytes("valueC"); - private static byte[] valueDname = Bytes.toBytes("valueD"); - - /** - * Runs all of the tests under a single JUnit test method. We - * consolidate all testing to one method because HBaseClusterTestCase - * is prone to OutOfMemoryExceptions when there are three or more - * JUnit test methods. - * - * @throws Exception - */ - public void testAll() throws Exception { - // Run all tests - doTestTableCreateDrop(); - doTestTableMutations(); - doTestTableTimestampsAndColumns(); - doTestTableScanners(); - } - - /** - * Tests for creating, enabling, disabling, and deleting tables. Also - * tests that creating a table with an invalid column name yields an - * IllegalArgument exception. - * - * @throws Exception - */ - public void doTestTableCreateDrop() throws Exception { - ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler(); - - // Create/enable/disable/delete tables, ensure methods act correctly - assertEquals(handler.getTableNames().size(), 0); - handler.createTable(tableAname, getColumnDescriptors()); - assertEquals(handler.getTableNames().size(), 1); - assertEquals(handler.getColumnDescriptors(tableAname).size(), 2); - assertTrue(handler.isTableEnabled(tableAname)); - handler.createTable(tableBname, new ArrayList()); - assertEquals(handler.getTableNames().size(), 2); - handler.disableTable(tableBname); - assertFalse(handler.isTableEnabled(tableBname)); - handler.deleteTable(tableBname); - assertEquals(handler.getTableNames().size(), 1); - handler.disableTable(tableAname); - assertFalse(handler.isTableEnabled(tableAname)); - handler.enableTable(tableAname); - assertTrue(handler.isTableEnabled(tableAname)); - handler.disableTable(tableAname); - handler.deleteTable(tableAname); - } - - /** - * Tests adding a series of Mutations and BatchMutations, including a - * delete mutation. Also tests data retrieval, and getting back multiple - * versions. - * - * @throws Exception - */ - public void doTestTableMutations() throws Exception { - // Setup - ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler(); - handler.createTable(tableAname, getColumnDescriptors()); - - // Apply a few Mutations to rowA - // mutations.add(new Mutation(false, columnAname, valueAname)); - // mutations.add(new Mutation(false, columnBname, valueBname)); - handler.mutateRow(tableAname, rowAname, getMutations()); - - // Assert that the changes were made - assertTrue(Bytes.equals(valueAname, - handler.get(tableAname, rowAname, columnAname).get(0).value)); - TRowResult rowResult1 = handler.getRow(tableAname, rowAname).get(0); - assertTrue(Bytes.equals(rowAname, rowResult1.row)); - assertTrue(Bytes.equals(valueBname, - rowResult1.columns.get(columnBname).value)); - - // Apply a few BatchMutations for rowA and rowB - // rowAmutations.add(new Mutation(true, columnAname, null)); - // rowAmutations.add(new Mutation(false, columnBname, valueCname)); - // batchMutations.add(new BatchMutation(rowAname, rowAmutations)); - // Mutations to rowB - // rowBmutations.add(new Mutation(false, columnAname, valueCname)); - // rowBmutations.add(new Mutation(false, columnBname, valueDname)); - // batchMutations.add(new BatchMutation(rowBname, rowBmutations)); - handler.mutateRows(tableAname, getBatchMutations()); - - // Assert that changes were made to rowA - List cells = handler.get(tableAname, rowAname, columnAname); - assertFalse(cells.size() > 0); - assertTrue(Bytes.equals(valueCname, handler.get(tableAname, rowAname, columnBname).get(0).value)); - List versions = handler.getVer(tableAname, rowAname, columnBname, MAXVERSIONS); - assertTrue(Bytes.equals(valueCname, versions.get(0).value)); - assertTrue(Bytes.equals(valueBname, versions.get(1).value)); - - // Assert that changes were made to rowB - TRowResult rowResult2 = handler.getRow(tableAname, rowBname).get(0); - assertTrue(Bytes.equals(rowBname, rowResult2.row)); - assertTrue(Bytes.equals(valueCname, rowResult2.columns.get(columnAname).value)); - assertTrue(Bytes.equals(valueDname, rowResult2.columns.get(columnBname).value)); - - // Apply some deletes - handler.deleteAll(tableAname, rowAname, columnBname); - handler.deleteAllRow(tableAname, rowBname); - - // Assert that the deletes were applied - int size = handler.get(tableAname, rowAname, columnBname).size(); - assertEquals(0, size); - size = handler.getRow(tableAname, rowBname).size(); - assertEquals(0, size); - - // Teardown - handler.disableTable(tableAname); - handler.deleteTable(tableAname); - } - - /** - * Similar to testTableMutations(), except Mutations are applied with - * specific timestamps and data retrieval uses these timestamps to - * extract specific versions of data. - * - * @throws Exception - */ - public void doTestTableTimestampsAndColumns() throws Exception { - // Setup - ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler(); - handler.createTable(tableAname, getColumnDescriptors()); - - // Apply timestamped Mutations to rowA - long time1 = System.currentTimeMillis(); - handler.mutateRowTs(tableAname, rowAname, getMutations(), time1); - - // Sleep to assure that 'time1' and 'time2' will be different even with a - // coarse grained system timer. - Thread.sleep(1000); - - // Apply timestamped BatchMutations for rowA and rowB - long time2 = System.currentTimeMillis(); - handler.mutateRowsTs(tableAname, getBatchMutations(), time2); - - // Apply an overlapping timestamped mutation to rowB - handler.mutateRowTs(tableAname, rowBname, getMutations(), time2); - - // Assert that the timestamp-related methods retrieve the correct data - assertEquals(handler.getVerTs(tableAname, rowAname, columnBname, time2, - MAXVERSIONS).size(), 2); - assertEquals(handler.getVerTs(tableAname, rowAname, columnBname, time1, - MAXVERSIONS).size(), 1); - - TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1).get(0); - TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2).get(0); - assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname)); - assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueBname)); - assertTrue(Bytes.equals(rowResult2.columns.get(columnBname).value, valueCname)); - - // Maybe I'd reading this wrong but at line #187 above, the BatchMutations - // are adding a columnAname at time2 so the below should be true not false - // -- St.Ack - assertTrue(rowResult2.columns.containsKey(columnAname)); - - List columns = new ArrayList(); - columns.add(columnBname); - - rowResult1 = handler.getRowWithColumns(tableAname, rowAname, columns).get(0); - assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueCname)); - assertFalse(rowResult1.columns.containsKey(columnAname)); - - rowResult1 = handler.getRowWithColumnsTs(tableAname, rowAname, columns, time1).get(0); - assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueBname)); - assertFalse(rowResult1.columns.containsKey(columnAname)); - - // Apply some timestamped deletes - handler.deleteAllTs(tableAname, rowAname, columnBname, time1); - handler.deleteAllRowTs(tableAname, rowBname, time2); - - // Assert that the timestamp-related methods retrieve the correct data - int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS).size(); - assertFalse(size > 0); - assertTrue(Bytes.equals(handler.get(tableAname, rowAname, columnBname).get(0).value, valueCname)); - assertFalse(handler.getRow(tableAname, rowBname).size() > 0); - - // Teardown - handler.disableTable(tableAname); - handler.deleteTable(tableAname); - } - - /** - * Tests the four different scanner-opening methods (with and without - * a stoprow, with and without a timestamp). - * - * @throws Exception - */ - public void doTestTableScanners() throws Exception { - // Setup - ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler(); - handler.createTable(tableAname, getColumnDescriptors()); - - // Apply timestamped Mutations to rowA - long time1 = System.currentTimeMillis(); - handler.mutateRowTs(tableAname, rowAname, getMutations(), time1); - - // Sleep to assure that 'time1' and 'time2' will be different even with a - // coarse grained system timer. - Thread.sleep(1000); - - // Apply timestamped BatchMutations for rowA and rowB - long time2 = System.currentTimeMillis(); - handler.mutateRowsTs(tableAname, getBatchMutations(), time2); - - // Test a scanner on all rows and all columns, no timestamp - int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true)); - TRowResult rowResult1a = handler.scannerGet(scanner1).get(0); - assertTrue(Bytes.equals(rowResult1a.row, rowAname)); - // This used to be '1'. I don't know why when we are asking for two columns - // and when the mutations above would seem to add two columns to the row. - // -- St.Ack 05/12/2009 - assertEquals(rowResult1a.columns.size(), 2); - assertTrue(Bytes.equals(rowResult1a.columns.get(columnBname).value, valueCname)); - TRowResult rowResult1b = handler.scannerGet(scanner1).get(0); - assertTrue(Bytes.equals(rowResult1b.row, rowBname)); - assertEquals(rowResult1b.columns.size(), 2); - assertTrue(Bytes.equals(rowResult1b.columns.get(columnAname).value, valueCname)); - assertTrue(Bytes.equals(rowResult1b.columns.get(columnBname).value, valueDname)); - closeScanner(scanner1, handler); - - // Test a scanner on all rows and all columns, with timestamp - int scanner2 = handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1); - TRowResult rowResult2a = handler.scannerGet(scanner2).get(0); - assertEquals(rowResult2a.columns.size(), 2); - assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname)); - assertTrue(Bytes.equals(rowResult2a.columns.get(columnBname).value, valueBname)); - closeScanner(scanner2, handler); - - // Test a scanner on the first row and first column only, no timestamp - int scanner3 = handler.scannerOpenWithStop(tableAname, rowAname, rowBname, - getColumnList(true, false)); - closeScanner(scanner3, handler); - - // Test a scanner on the first row and second column only, with timestamp - int scanner4 = handler.scannerOpenWithStopTs(tableAname, rowAname, rowBname, - getColumnList(false, true), time1); - TRowResult rowResult4a = handler.scannerGet(scanner4).get(0); - assertEquals(rowResult4a.columns.size(), 1); - assertTrue(Bytes.equals(rowResult4a.columns.get(columnBname).value, valueBname)); - - // Teardown - handler.disableTable(tableAname); - handler.deleteTable(tableAname); - } - - /** - * - * @return a List of ColumnDescriptors for use in creating a table. Has one - * default ColumnDescriptor and one ColumnDescriptor with fewer versions - */ - private List getColumnDescriptors() { - ArrayList cDescriptors = new ArrayList(); - - // A default ColumnDescriptor - ColumnDescriptor cDescA = new ColumnDescriptor(); - cDescA.name = columnAname; - cDescriptors.add(cDescA); - - // A slightly customized ColumnDescriptor (only 2 versions) - ColumnDescriptor cDescB = new ColumnDescriptor(columnBname, 2, "NONE", - false, 2147483647, "NONE", 0, 0, false, -1); - cDescriptors.add(cDescB); - - return cDescriptors; - } - - /** - * - * @param includeA whether or not to include columnA - * @param includeB whether or not to include columnB - * @return a List of column names for use in retrieving a scanner - */ - private List getColumnList(boolean includeA, boolean includeB) { - List columnList = new ArrayList(); - if (includeA) columnList.add(columnAname); - if (includeB) columnList.add(columnBname); - return columnList; - } - - /** - * - * @return a List of Mutations for a row, with columnA having valueA - * and columnB having valueB - */ - private List getMutations() { - List mutations = new ArrayList(); - mutations.add(new Mutation(false, columnAname, valueAname)); - mutations.add(new Mutation(false, columnBname, valueBname)); - return mutations; - } - - /** - * - * @return a List of BatchMutations with the following effects: - * (rowA, columnA): delete - * (rowA, columnB): place valueC - * (rowB, columnA): place valueC - * (rowB, columnB): place valueD - */ - private List getBatchMutations() { - List batchMutations = new ArrayList(); - // Mutations to rowA. You can't mix delete and put anymore. - List rowAmutations = new ArrayList(); - rowAmutations.add(new Mutation(true, columnAname, null)); - batchMutations.add(new BatchMutation(rowAname, rowAmutations)); - rowAmutations = new ArrayList(); - rowAmutations.add(new Mutation(false, columnBname, valueCname)); - batchMutations.add(new BatchMutation(rowAname, rowAmutations)); - // Mutations to rowB - List rowBmutations = new ArrayList(); - rowBmutations.add(new Mutation(false, columnAname, valueCname)); - rowBmutations.add(new Mutation(false, columnBname, valueDname)); - batchMutations.add(new BatchMutation(rowBname, rowBmutations)); - return batchMutations; - } - - /** - * Asserts that the passed scanner is exhausted, and then closes - * the scanner. - * - * @param scannerId the scanner to close - * @param handler the HBaseHandler interfacing to HBase - * @throws Exception - */ - private void closeScanner(int scannerId, ThriftServer.HBaseHandler handler) throws Exception { - handler.scannerGet(scannerId); - handler.scannerClose(scannerId); - } -} Index: src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java =================================================================== --- src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java (revision 0) +++ src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java (revision 0) @@ -0,0 +1,397 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.thrift; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.HBaseClusterTestCase; +import org.apache.hadoop.hbase.thrift.generated.BatchMutation; +import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor; +import org.apache.hadoop.hbase.thrift.generated.IllegalArgument; +import org.apache.hadoop.hbase.thrift.generated.Mutation; +import org.apache.hadoop.hbase.thrift.generated.TCell; +import org.apache.hadoop.hbase.thrift.generated.TRowResult; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Unit testing for ThriftServer.HBaseHandler, a part of the + * org.apache.hadoop.hbase.thrift package. + */ +public class TestThriftServer extends HBaseClusterTestCase { + + // Static names for tables, columns, rows, and values + private static byte[] tableAname = Bytes.toBytes("tableA"); + private static byte[] tableBname = Bytes.toBytes("tableB"); + private static byte[] columnAname = Bytes.toBytes("columnA:"); + private static byte[] columnBname = Bytes.toBytes("columnB:"); + private static byte[] rowAname = Bytes.toBytes("rowA"); + private static byte[] rowBname = Bytes.toBytes("rowB"); + private static byte[] valueAname = Bytes.toBytes("valueA"); + private static byte[] valueBname = Bytes.toBytes("valueB"); + private static byte[] valueCname = Bytes.toBytes("valueC"); + private static byte[] valueDname = Bytes.toBytes("valueD"); + + /** + * Runs all of the tests under a single JUnit test method. We + * consolidate all testing to one method because HBaseClusterTestCase + * is prone to OutOfMemoryExceptions when there are three or more + * JUnit test methods. + * + * @throws Exception + */ + public void testAll() throws Exception { + // Run all tests + doTestTableCreateDrop(); + doTestTableMutations(); + doTestTableTimestampsAndColumns(); + doTestTableScanners(); + } + + /** + * Tests for creating, enabling, disabling, and deleting tables. Also + * tests that creating a table with an invalid column name yields an + * IllegalArgument exception. + * + * @throws Exception + */ + public void doTestTableCreateDrop() throws Exception { + ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler(); + + // Create/enable/disable/delete tables, ensure methods act correctly + assertEquals(handler.getTableNames().size(), 0); + handler.createTable(tableAname, getColumnDescriptors()); + assertEquals(handler.getTableNames().size(), 1); + assertEquals(handler.getColumnDescriptors(tableAname).size(), 2); + assertTrue(handler.isTableEnabled(tableAname)); + handler.createTable(tableBname, new ArrayList()); + assertEquals(handler.getTableNames().size(), 2); + handler.disableTable(tableBname); + assertFalse(handler.isTableEnabled(tableBname)); + handler.deleteTable(tableBname); + assertEquals(handler.getTableNames().size(), 1); + handler.disableTable(tableAname); + assertFalse(handler.isTableEnabled(tableAname)); + handler.enableTable(tableAname); + assertTrue(handler.isTableEnabled(tableAname)); + handler.disableTable(tableAname); + handler.deleteTable(tableAname); + } + + /** + * Tests adding a series of Mutations and BatchMutations, including a + * delete mutation. Also tests data retrieval, and getting back multiple + * versions. + * + * @throws Exception + */ + public void doTestTableMutations() throws Exception { + // Setup + ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler(); + handler.createTable(tableAname, getColumnDescriptors()); + + // Apply a few Mutations to rowA + // mutations.add(new Mutation(false, columnAname, valueAname)); + // mutations.add(new Mutation(false, columnBname, valueBname)); + handler.mutateRow(tableAname, rowAname, getMutations()); + + // Assert that the changes were made + assertTrue(Bytes.equals(valueAname, + handler.get(tableAname, rowAname, columnAname).get(0).value)); + TRowResult rowResult1 = handler.getRow(tableAname, rowAname).get(0); + assertTrue(Bytes.equals(rowAname, rowResult1.row)); + assertTrue(Bytes.equals(valueBname, + rowResult1.columns.get(columnBname).value)); + + // Apply a few BatchMutations for rowA and rowB + // rowAmutations.add(new Mutation(true, columnAname, null)); + // rowAmutations.add(new Mutation(false, columnBname, valueCname)); + // batchMutations.add(new BatchMutation(rowAname, rowAmutations)); + // Mutations to rowB + // rowBmutations.add(new Mutation(false, columnAname, valueCname)); + // rowBmutations.add(new Mutation(false, columnBname, valueDname)); + // batchMutations.add(new BatchMutation(rowBname, rowBmutations)); + handler.mutateRows(tableAname, getBatchMutations()); + + // Assert that changes were made to rowA + List cells = handler.get(tableAname, rowAname, columnAname); + assertFalse(cells.size() > 0); + assertTrue(Bytes.equals(valueCname, handler.get(tableAname, rowAname, columnBname).get(0).value)); + List versions = handler.getVer(tableAname, rowAname, columnBname, MAXVERSIONS); + assertTrue(Bytes.equals(valueCname, versions.get(0).value)); + assertTrue(Bytes.equals(valueBname, versions.get(1).value)); + + // Assert that changes were made to rowB + TRowResult rowResult2 = handler.getRow(tableAname, rowBname).get(0); + assertTrue(Bytes.equals(rowBname, rowResult2.row)); + assertTrue(Bytes.equals(valueCname, rowResult2.columns.get(columnAname).value)); + assertTrue(Bytes.equals(valueDname, rowResult2.columns.get(columnBname).value)); + + // Apply some deletes + handler.deleteAll(tableAname, rowAname, columnBname); + handler.deleteAllRow(tableAname, rowBname); + + // Assert that the deletes were applied + int size = handler.get(tableAname, rowAname, columnBname).size(); + assertEquals(0, size); + size = handler.getRow(tableAname, rowBname).size(); + assertEquals(0, size); + + // Teardown + handler.disableTable(tableAname); + handler.deleteTable(tableAname); + } + + /** + * Similar to testTableMutations(), except Mutations are applied with + * specific timestamps and data retrieval uses these timestamps to + * extract specific versions of data. + * + * @throws Exception + */ + public void doTestTableTimestampsAndColumns() throws Exception { + // Setup + ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler(); + handler.createTable(tableAname, getColumnDescriptors()); + + // Apply timestamped Mutations to rowA + long time1 = System.currentTimeMillis(); + handler.mutateRowTs(tableAname, rowAname, getMutations(), time1); + + Thread.sleep(1000); + + // Apply timestamped BatchMutations for rowA and rowB + long time2 = System.currentTimeMillis(); + handler.mutateRowsTs(tableAname, getBatchMutations(), time2); + + // Apply an overlapping timestamped mutation to rowB + handler.mutateRowTs(tableAname, rowBname, getMutations(), time2); + + // the getVerTs is [inf, ts) so you need to increment one. + time1 += 1; + time2 += 2; + + // Assert that the timestamp-related methods retrieve the correct data + assertEquals(2, handler.getVerTs(tableAname, rowAname, columnBname, time2, + MAXVERSIONS).size()); + assertEquals(1, handler.getVerTs(tableAname, rowAname, columnBname, time1, + MAXVERSIONS).size()); + + TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1).get(0); + TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2).get(0); + // columnA was completely deleted + //assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname)); + assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueBname)); + assertTrue(Bytes.equals(rowResult2.columns.get(columnBname).value, valueCname)); + + // ColumnAname has been deleted, and will never be visible even with a getRowTs() + assertFalse(rowResult2.columns.containsKey(columnAname)); + + List columns = new ArrayList(); + columns.add(columnBname); + + rowResult1 = handler.getRowWithColumns(tableAname, rowAname, columns).get(0); + assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueCname)); + assertFalse(rowResult1.columns.containsKey(columnAname)); + + rowResult1 = handler.getRowWithColumnsTs(tableAname, rowAname, columns, time1).get(0); + assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueBname)); + assertFalse(rowResult1.columns.containsKey(columnAname)); + + // Apply some timestamped deletes + // this actually deletes _everything_. + // nukes everything in columnB: forever. + handler.deleteAllTs(tableAname, rowAname, columnBname, time1); + handler.deleteAllRowTs(tableAname, rowBname, time2); + + // Assert that the timestamp-related methods retrieve the correct data + int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS).size(); + assertEquals(0, size); + + size = handler.getVerTs(tableAname, rowAname, columnBname, time2, MAXVERSIONS).size(); + assertEquals(1, size); + + // should be available.... + assertTrue(Bytes.equals(handler.get(tableAname, rowAname, columnBname).get(0).value, valueCname)); + + assertEquals(0, handler.getRow(tableAname, rowBname).size()); + + // Teardown + handler.disableTable(tableAname); + handler.deleteTable(tableAname); + } + + /** + * Tests the four different scanner-opening methods (with and without + * a stoprow, with and without a timestamp). + * + * @throws Exception + */ + public void doTestTableScanners() throws Exception { + // Setup + ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler(); + handler.createTable(tableAname, getColumnDescriptors()); + + // Apply timestamped Mutations to rowA + long time1 = System.currentTimeMillis(); + handler.mutateRowTs(tableAname, rowAname, getMutations(), time1); + + // Sleep to assure that 'time1' and 'time2' will be different even with a + // coarse grained system timer. + Thread.sleep(1000); + + // Apply timestamped BatchMutations for rowA and rowB + long time2 = System.currentTimeMillis(); + handler.mutateRowsTs(tableAname, getBatchMutations(), time2); + + time1 += 1; + time2 += 1; + + // Test a scanner on all rows and all columns, no timestamp + int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true)); + TRowResult rowResult1a = handler.scannerGet(scanner1).get(0); + assertTrue(Bytes.equals(rowResult1a.row, rowAname)); + // This used to be '1'. I don't know why when we are asking for two columns + // and when the mutations above would seem to add two columns to the row. + // -- St.Ack 05/12/2009 + assertEquals(rowResult1a.columns.size(), 1); + assertTrue(Bytes.equals(rowResult1a.columns.get(columnBname).value, valueCname)); + + TRowResult rowResult1b = handler.scannerGet(scanner1).get(0); + assertTrue(Bytes.equals(rowResult1b.row, rowBname)); + assertEquals(rowResult1b.columns.size(), 2); + assertTrue(Bytes.equals(rowResult1b.columns.get(columnAname).value, valueCname)); + assertTrue(Bytes.equals(rowResult1b.columns.get(columnBname).value, valueDname)); + closeScanner(scanner1, handler); + + // Test a scanner on all rows and all columns, with timestamp + int scanner2 = handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1); + TRowResult rowResult2a = handler.scannerGet(scanner2).get(0); + assertEquals(rowResult2a.columns.size(), 1); + // column A deleted, does not exist. + //assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname)); + assertTrue(Bytes.equals(rowResult2a.columns.get(columnBname).value, valueBname)); + closeScanner(scanner2, handler); + + // Test a scanner on the first row and first column only, no timestamp + int scanner3 = handler.scannerOpenWithStop(tableAname, rowAname, rowBname, + getColumnList(true, false)); + closeScanner(scanner3, handler); + + // Test a scanner on the first row and second column only, with timestamp + int scanner4 = handler.scannerOpenWithStopTs(tableAname, rowAname, rowBname, + getColumnList(false, true), time1); + TRowResult rowResult4a = handler.scannerGet(scanner4).get(0); + assertEquals(rowResult4a.columns.size(), 1); + assertTrue(Bytes.equals(rowResult4a.columns.get(columnBname).value, valueBname)); + + // Teardown + handler.disableTable(tableAname); + handler.deleteTable(tableAname); + } + + /** + * + * @return a List of ColumnDescriptors for use in creating a table. Has one + * default ColumnDescriptor and one ColumnDescriptor with fewer versions + */ + private List getColumnDescriptors() { + ArrayList cDescriptors = new ArrayList(); + + // A default ColumnDescriptor + ColumnDescriptor cDescA = new ColumnDescriptor(); + cDescA.name = columnAname; + cDescriptors.add(cDescA); + + // A slightly customized ColumnDescriptor (only 2 versions) + ColumnDescriptor cDescB = new ColumnDescriptor(columnBname, 2, "NONE", + false, 2147483647, "NONE", 0, 0, false, -1); + cDescriptors.add(cDescB); + + return cDescriptors; + } + + /** + * + * @param includeA whether or not to include columnA + * @param includeB whether or not to include columnB + * @return a List of column names for use in retrieving a scanner + */ + private List getColumnList(boolean includeA, boolean includeB) { + List columnList = new ArrayList(); + if (includeA) columnList.add(columnAname); + if (includeB) columnList.add(columnBname); + return columnList; + } + + /** + * + * @return a List of Mutations for a row, with columnA having valueA + * and columnB having valueB + */ + private List getMutations() { + List mutations = new ArrayList(); + mutations.add(new Mutation(false, columnAname, valueAname)); + mutations.add(new Mutation(false, columnBname, valueBname)); + return mutations; + } + + /** + * + * @return a List of BatchMutations with the following effects: + * (rowA, columnA): delete + * (rowA, columnB): place valueC + * (rowB, columnA): place valueC + * (rowB, columnB): place valueD + */ + private List getBatchMutations() { + List batchMutations = new ArrayList(); + + // Mutations to rowA. You can't mix delete and put anymore. + List rowAmutations = new ArrayList(); + rowAmutations.add(new Mutation(true, columnAname, null)); + batchMutations.add(new BatchMutation(rowAname, rowAmutations)); + + rowAmutations = new ArrayList(); + rowAmutations.add(new Mutation(false, columnBname, valueCname)); + batchMutations.add(new BatchMutation(rowAname, rowAmutations)); + + // Mutations to rowB + List rowBmutations = new ArrayList(); + rowBmutations.add(new Mutation(false, columnAname, valueCname)); + rowBmutations.add(new Mutation(false, columnBname, valueDname)); + batchMutations.add(new BatchMutation(rowBname, rowBmutations)); + + return batchMutations; + } + + /** + * Asserts that the passed scanner is exhausted, and then closes + * the scanner. + * + * @param scannerId the scanner to close + * @param handler the HBaseHandler interfacing to HBase + * @throws Exception + */ + private void closeScanner(int scannerId, ThriftServer.HBaseHandler handler) throws Exception { + handler.scannerGet(scannerId); + handler.scannerClose(scannerId); + } +} Index: src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java =================================================================== --- src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java (revision 785929) +++ src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java (working copy) @@ -506,9 +506,12 @@ KeyValue put1 = new KeyValue(row, fam, qf1, ts, val); KeyValue put2 = new KeyValue(row, fam, qf2, ts, val); KeyValue put3 = new KeyValue(row, fam, qf3, ts, val); + KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val); + memcache.add(put1); memcache.add(put2); memcache.add(put3); + memcache.add(put4); KeyValue del = new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val); @@ -516,8 +519,9 @@ List expected = new ArrayList(); expected.add(del); + expected.add(put4); - assertEquals(1, memcache.memcache.size()); + assertEquals(2, memcache.memcache.size()); int i=0; for(KeyValue actual : memcache.memcache) { assertEquals(expected.get(i++), actual); Index: src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java =================================================================== --- src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java (revision 785929) +++ src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java (working copy) @@ -140,8 +140,13 @@ List results = new ArrayList(); assertEquals(true, scan.next(results)); + assertEquals(0, results.size()); + + assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[2], results.get(0)); + + assertEquals(false, scan.next(results)); } public void testDeleteVersionMaskingMultiplePuts() throws IOException { @@ -272,8 +277,12 @@ null, scanners); List results = new ArrayList(); assertEquals(true, scan.next(results)); + assertEquals(0, results.size()); + assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[kvs.length-1], results.get(0)); + + assertEquals(false, scan.next(results)); } public void testDeleteColumn() throws IOException { Index: src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java =================================================================== --- src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java (revision 785929) +++ src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java (working copy) @@ -285,10 +285,45 @@ get = new Get(row).addColumn(fam, splitB); result = region.get(get, null); assertEquals(1, result.size()); + } - + public void testScanner_DeleteOneFamilyNotAnother() throws IOException { + byte [] tableName = Bytes.toBytes("test_table"); + byte [] fam1 = Bytes.toBytes("columnA"); + byte [] fam2 = Bytes.toBytes("columnB"); + initHRegion(tableName, getName(), fam1, fam2); - + byte [] rowA = Bytes.toBytes("rowA"); + byte [] rowB = Bytes.toBytes("rowB"); + + byte [] value = Bytes.toBytes("value"); + + Delete delete = new Delete(rowA); + delete.deleteFamily(fam1); + + region.delete(delete, null, true); + + // now create data. + Put put = new Put(rowA); + put.add(fam2, null, value); + region.put(put); + + put = new Put(rowB); + put.add(fam1, null, value); + put.add(fam2, null, value); + region.put(put); + + Scan scan = new Scan(); + scan.addFamily(fam1).addFamily(fam2); + InternalScanner s = region.getScanner(scan); + List results = new ArrayList(); + s.next(results); + assertTrue(Bytes.equals(rowA, results.get(0).getRow())); + + results.clear(); + s.next(results); + assertTrue(Bytes.equals(rowB, results.get(0).getRow())); + } //Visual test, since the method doesn't return anything Index: src/java/org/apache/hadoop/hbase/thrift/ThriftServer.java =================================================================== --- src/java/org/apache/hadoop/hbase/thrift/ThriftServer.java (revision 785929) +++ src/java/org/apache/hadoop/hbase/thrift/ThriftServer.java (working copy) @@ -298,12 +298,15 @@ HTable table = getTable(tableName); Get get = new Get(row); get.addColumn(family, qualifier); - get.setTimeStamp(timestamp); + get.setTimeRange(Long.MIN_VALUE, timestamp); get.setMaxVersions(numVersions); Result result = table.get(get); List cells = new ArrayList(); - for(KeyValue kv : result.sorted()) { - cells.add(new Cell(kv.getValue(), kv.getTimestamp())); + KeyValue [] kvs = result.sorted(); + if (kvs != null) { + for(KeyValue kv : kvs) { + cells.add(new Cell(kv.getValue(), kv.getTimestamp())); + } } return ThriftUtilities.cellFromHBase(cells.toArray(new Cell[0])); } catch (IOException e) { @@ -335,7 +338,7 @@ HTable table = getTable(tableName); if (columns == null) { Get get = new Get(row); - get.setTimeStamp(timestamp); + get.setTimeRange(Long.MIN_VALUE, timestamp); Result result = table.get(get); return ThriftUtilities.rowResultFromHBase(result.getRowResult()); } @@ -345,7 +348,7 @@ byte [][] famAndQf = KeyValue.parseColumn(column); get.addColumn(famAndQf[0], famAndQf[1]); } - get.setTimeStamp(timestamp); + get.setTimeRange(Long.MIN_VALUE, timestamp); Result result = table.get(get); return ThriftUtilities.rowResultFromHBase(result.getRowResult()); } catch (IOException e) { @@ -362,12 +365,12 @@ long timestamp) throws IOError { try { HTable table = getTable(tableName); - Delete delete = new Delete(row, timestamp, null); + Delete delete = new Delete(row); byte [][] famAndQf = KeyValue.parseColumn(column); - if(famAndQf[1].length == 0){ - delete.deleteFamily(famAndQf[0]); + if (famAndQf[1].length == 0) { + delete.deleteFamily(famAndQf[0], timestamp); } else { - delete.deleteColumns(famAndQf[0], famAndQf[1]); + delete.deleteColumns(famAndQf[0], famAndQf[1], timestamp); } table.delete(delete); @@ -457,23 +460,36 @@ public void mutateRowsTs(byte[] tableName, List rowBatches, long timestamp) throws IOError, IllegalArgument, TException { List puts = new ArrayList(); - + List deletes = new ArrayList(); + for (BatchMutation batch : rowBatches) { byte[] row = batch.row; List mutations = batch.mutations; + Delete delete = new Delete(row); Put put = new Put(row); put.setTimeStamp(timestamp); for (Mutation m : mutations) { byte [][] famAndQf = KeyValue.parseColumn(m.column); - put.add(famAndQf[0], famAndQf[1], m.value); + if (m.isDelete) { + delete.deleteColumns(famAndQf[0], famAndQf[1]); + } else { + put.add(famAndQf[0], famAndQf[1], m.value); + } } - puts.add(put); + if (!delete.isEmpty()) + deletes.add(delete); + if (!put.isEmpty()) + puts.add(put); } HTable table = null; try { table = getTable(tableName); - table.put(puts); + if (!puts.isEmpty()) + table.put(puts); + for (Delete del : deletes) { + table.delete(del); + } } catch (IOException e) { throw new IOError(e.getMessage()); } catch (IllegalArgumentException e) { @@ -534,19 +550,19 @@ public int scannerOpen(byte[] tableName, byte[] startRow, List columns) throws IOError { try { - HTable table = getTable(tableName); - byte[][] columnsArray = null; - if ((columns == null) || (columns.size() == 0)) { - columnsArray = getAllColumns(table); - } else { - columnsArray = columns.toArray(new byte[0][]); + HTable table = getTable(tableName); + byte[][] columnsArray = null; + if ((columns == null) || (columns.size() == 0)) { + columnsArray = getAllColumns(table); + } else { + columnsArray = columns.toArray(new byte[0][]); + } + Scan scan = new Scan(startRow); + scan.addColumns(columnsArray); + return addScanner(table.getScanner(scan)); + } catch (IOException e) { + throw new IOError(e.getMessage()); } - Scan scan = new Scan(startRow); - scan.addColumns(columnsArray); - return addScanner(table.getScanner(scan)); - } catch (IOException e) { - throw new IOError(e.getMessage()); - } } public int scannerOpenWithStop(byte[] tableName, byte[] startRow, @@ -579,7 +595,7 @@ } Scan scan = new Scan(startRow); scan.addColumns(columnsArray); - scan.setTimeRange(0, timestamp); + scan.setTimeRange(Long.MIN_VALUE, timestamp); return addScanner(table.getScanner(scan)); } catch (IOException e) { throw new IOError(e.getMessage()); @@ -599,7 +615,7 @@ } Scan scan = new Scan(startRow, stopRow); scan.addColumns(columnsArray); - scan.setTimeRange(0, timestamp); + scan.setTimeRange(Long.MIN_VALUE, timestamp); return addScanner(table.getScanner(scan)); } catch (IOException e) { throw new IOError(e.getMessage()); Index: src/java/org/apache/hadoop/hbase/regionserver/Memcache.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/Memcache.java (revision 785929) +++ src/java/org/apache/hadoop/hbase/regionserver/Memcache.java (working copy) @@ -251,13 +251,13 @@ byte deleteType = deleteBuffer[deleteOffset]; //Comparing with tail from memcache - for(KeyValue mem : tailSet) { + for (KeyValue mem : tailSet) { DeleteCode res = DeleteCompare.deleteCompare(mem, deleteBuffer, deleteRowOffset, deleteRowLen, deleteQualifierOffset, deleteQualifierLen, deleteTimestampOffset, deleteType, comparator.getRawComparator()); - if(res == DeleteCode.DONE) { + if (res == DeleteCode.DONE) { break; } else if (res == DeleteCode.DELETE) { deletes.add(mem); Index: src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (revision 785929) +++ src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (working copy) @@ -123,7 +123,8 @@ * @param result * @return true if there are more rows, false if scanner is done */ - public synchronized boolean next(List result) throws IOException { + public synchronized boolean next(List outResult) throws IOException { + List results = new ArrayList(); KeyValue peeked = this.heap.peek(); if (peeked == null) { close(); @@ -136,27 +137,25 @@ switch(qcode) { case INCLUDE: KeyValue next = this.heap.next(); - result.add(next); + results.add(next); continue; case DONE: - // what happens if we have 0 results? - if (result.isEmpty()) { - // try the next one. - matcher.setRow(this.heap.peek().getRow()); - continue; - } if (matcher.filterEntireRow()) { - // wow, well, um, reset the result and continue. - result.clear(); - matcher.setRow(heap.peek().getRow()); - continue; + // nuke all results, and then return. + results.clear(); } + // copy jazz + outResult.addAll(results); return true; case DONE_SCAN: close(); + + // copy jazz + outResult.addAll(results); + return false; case SEEK_NEXT_ROW: @@ -178,9 +177,14 @@ throw new RuntimeException("UNEXPECTED"); } } - if(result.size() > 0) { + + if (!results.isEmpty()) { + // copy jazz + outResult.addAll(results); + return true; } + // No more keys close(); return false; Index: src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java (revision 785929) +++ src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java (working copy) @@ -89,8 +89,8 @@ int timeRes = Bytes.compareTo(memBuffer, tsOffset, Bytes.SIZEOF_LONG, deleteBuffer, deleteTimeOffset, Bytes.SIZEOF_LONG); - if(deleteType == KeyValue.Type.DeleteFamily.getCode()) { - if(timeRes <= 0){ + if (deleteType == KeyValue.Type.DeleteFamily.getCode()) { + if (timeRes <= 0) { return DeleteCode.DELETE; } return DeleteCode.SKIP; @@ -99,16 +99,16 @@ //Compare columns res = Bytes.compareTo(memBuffer, memOffset, memQualifierLen, deleteBuffer, deleteQualifierOffset, deleteQualifierLength); - if(res < 0) { + if (res < 0) { return DeleteCode.SKIP; } else if(res > 0) { return DeleteCode.DONE; } // same column, compare the time. - if(timeRes == 0) { + if (timeRes == 0) { return DeleteCode.DELETE; } else if (timeRes < 0) { - if(deleteType == KeyValue.Type.DeleteColumn.getCode()) { + if (deleteType == KeyValue.Type.DeleteColumn.getCode()) { return DeleteCode.DELETE; } return DeleteCode.DONE; Index: src/java/org/apache/hadoop/hbase/regionserver/HRegion.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (revision 785929) +++ src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (working copy) @@ -1710,7 +1710,7 @@ if(stopRow != null && comparator.compareRows(stopRow, 0, stopRow.length, currentRow, 0, currentRow.length) - <= 0){ + <= 0) { return false; } this.storeHeap.next(results); @@ -1721,6 +1721,21 @@ } byte [] row = kv.getRow(); if(!Bytes.equals(currentRow, row)) { + // Next row: + + // what happens if there are _no_ results: + if (results.isEmpty()) { + // Continue on the next row: + currentRow = row; + + // But did we pass the stop row? + if (stopRow != null && + comparator.compareRows(stopRow, 0, stopRow.length, + currentRow, 0, currentRow.length) <= 0) { + return false; + } + continue; + } return true; } this.storeHeap.next(results); Index: src/java/org/apache/hadoop/hbase/client/Delete.java =================================================================== --- src/java/org/apache/hadoop/hbase/client/Delete.java (revision 785929) +++ src/java/org/apache/hadoop/hbase/client/Delete.java (working copy) @@ -105,6 +105,14 @@ } /** + * Method to check if the familyMap is empty + * @return true if empty, false otherwise + */ + public boolean isEmpty() { + return familyMap.isEmpty(); + } + + /** * Delete all versions of all columns of the specified family. *

* Overrides previous calls to deleteColumn and deleteColumns for the Index: src/java/org/apache/hadoop/hbase/client/Result.java =================================================================== --- src/java/org/apache/hadoop/hbase/client/Result.java (revision 785929) +++ src/java/org/apache/hadoop/hbase/client/Result.java (working copy) @@ -235,6 +235,8 @@ public Cell getCellValue(byte[] family, byte[] qualifier) { Map.Entry val = getKeyValue(family, qualifier); + if (val == null) + return null; return new Cell(val.getValue(), val.getKey()); }