diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index 13f1bd9..3d4aad6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -191,6 +191,14 @@ public class HTableDescriptor implements Comparable { /** Default durability for HTD is USE_DEFAULT, which defaults to HBase-global default value */ private static final Durability DEFAULT_DURABLITY = Durability.USE_DEFAULT; + /** + * Timestamp type for the table which defines the semantics and format for the cell timestamps + * in the table. + */ + public static final String TIMESTAMP_TYPE = "TIMESTAMP_TYPE"; + private static final Bytes TIMESTAMP_TYPE_KEY = + new Bytes(Bytes.toBytes(TIMESTAMP_TYPE)); + /* * The below are ugly but better than creating them each time till we * replace booleans being saved as Strings with plain booleans. Need a @@ -229,6 +237,13 @@ public class HTableDescriptor implements Comparable { public static final boolean DEFAULT_REGION_MEMSTORE_REPLICATION = true; + public static final Timestamp DEFAULT_TIMESTAMP_TYPE = Timestamp.CUSTOM; + + /** If timestamp is not specified in the HTableDescription, than it means that it is a table + * before Timestamp type is introduced. We default to CUSTOM type for these tables. + */ + public static final Timestamp DEFAULT_UNSET_TIMESTAMP_TYPE = Timestamp.CUSTOM; + private final static Map DEFAULT_VALUES = new HashMap(); private final static Set RESERVED_KEYWORDS @@ -842,6 +857,29 @@ public class HTableDescriptor implements Comparable { } /** + * Sets the timestamp type for this table. + * @param timestamp the timestamp implementation + * @see Timestamp + */ + public HTableDescriptor setTimestampType(Timestamp timestamp) { + setValue(TIMESTAMP_TYPE, timestamp.getType()); + return this; + } + + /** + * Returns the timestamp type for this table. + * @return a Timestamp implementation. + * @see Timestamp + */ + public Timestamp getTimestampType() { + String value = getValue(TIMESTAMP_TYPE); + if (value == null || value.isEmpty()) { + return DEFAULT_UNSET_TIMESTAMP_TYPE; + } + return Timestamp.getByType(value); + } + + /** * Adds a column family. * For the updating purpose please use {@link #modifyFamily(HColumnDescriptor)} instead. * @param family HColumnDescriptor of family to add. diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/Timestamp.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/Timestamp.java new file mode 100644 index 0000000..1605732 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/Timestamp.java @@ -0,0 +1,463 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.time.FastDateFormat; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.util.TimeZone; +import com.google.common.annotations.VisibleForTesting; + +/** + * Timestamp is an abstraction for different representations of algorithms for representing time + * in HBase. Time is usually encoded as a 64-bit long in cells timestamps and used for sorting + * cells, ordering writes, snapshot in time queries, as well as TTL feature. The clock type defines + * the way that HBase servers assign timestamps to cells, while Timestamp type defines the way that + * time given by the clock is converted to a 64-bit long. A clock type is always bound to a + * timestamp type of the same name. + * + *

The clock/timestamp implementation is configurable per table. + * A clock can have physical time component(PT) and a logical time component(LT). The physical + * component usually represents the wall clock time. + * The Timestamp class encapsulates the logic to "encode" the physical and logical components from + * the given clock to a signed long. Timestamp offers methods to convert a given + * timestamp (long) in its format from and back to wall clock time based on epoch. Timestamps from + * cells should always be interpreted by the table's Timestamp type so that timestamps given by + * HBase servers can be converted to wall clock. + * + * + * HBase defines three different clock/timestamp implementations, CUSTOM, + *
+ * SYSTEM and HLC: + *

  • Custom clock (CUSTOM): This clock is the default clock for HBase versions prior to + * 2.0. It uses System.currentTimeMillis() as the physical component, and does not + * have a logical component. Custom clock is not monotonic (in case of leap seconds, NTP + * corrections, or manual time adjustments from the underlying OS). Also CUSTOM time does not keep + * track of previously given timestamps and does not assume any format for the timestamps. + * Custom timestamps should only be used if NTP or any other clock synchronization is not used or + * client supplied timestamps not representing milliseconds-since-epoch are used in the table. + * + *
  • + *
    + *
  • System clock (SYSTEM): This clock provides timestamps that are + * monotonically non-decreasing. Similar to CUSTOM time, it uses + * System.currentTimeMillis() as the physical component and does not have a logical component. + * SYSTEM time can be used if NTP or other clock synchronization mechanism is used within the + * cluster. In case that timestamps are also provided from client side, SYSTEM clock makes the + * clients a part of the cluster as well. Thus any clock skews from the client can cause rejection + * of requests or cause the clocks of region servers to drift. HLC clock should be preferred over + * SYSTEM clock since HLC provides monotonically increasing timestamps. SYSTEM clock should be used + * in cases where the table have data with CUSTOM timestamp types and TTL is set for the table or + * in case the client application cannot be used Timestamp class to interpret the timestamps from + * cells. + *
  • + *
    + *
  • Hybrid logical clock (HLC): This clock implements a clock with physical (PT) and + * logical (LT) components + * as defined in paper [1]. Each node has an HLC clock that sits on top of the systems physical + * clock that is usually NTP based. HLC captures causality relationship while maintaining the clock + * value to be close to NTP time. Timestamps given out by HLC are monotonically increasing at all + * times (region move, NTP adjustments, leap seconds, etc) except in cases where LT overflows. + * 22 bits are used for LT and 41 bits are used for PT in millisecond resolution. PT values from + * unix epoch (1/1/1970) till year 2039 can be represented with signed comparison. With unsigned + * timestamp comparison (which is not implemented in HBase yet), PT values up to year 2109 can be + * represented in PT. LT values up to 4,194,304 can be represented in LT. LT can overflow in the + * unlikely event of very big clock skews and high throughput. If that happens, maxLT is repeated + * which will make the clock monotonically non-decreasing (as opposed to monotonically increasing). + * HLC timestamps can also be used with TTL and other wall-clock time based features given that + * {@link #fromEpochTimeMillis(long)} and {@link #toEpochTimeMillis(long)} is properly used. + *
  • + * + *

    + * All tables created in HBase-2.0+ default to using HLC. All tables created before HBase-2.0 + * default to using CUSTOM clock. Existing tables will continue using CUSTOM timestamp format + * unless explicitly altered to have the timestamp type changed. + * + *

    + * CUSTOM and SYSTEM timestamps encode the timestamp as milliseconds-since + * epoch, making it compatible with earlier versions of HBase and client code can interpret + * Cell.getTimestamp() directly as wall clock time from System.currentTimeMillis() + * without a need to use the Timestamp class. However, we recommend always using HLC + * timestamp type together with the APIs in this class. + * + *

    + * HBase makes timestamps visible, quaryable and also settable for cells from client side. If not + * set, HBase populates the timestamp of cells from the given clock implementation for the table. + * All of the three clocks allows the client to set the timestamp of writes if needed. However, in + * case of SYSTEM and HLC, the client supplied timestamps will also update the clocks of the region + * servers making the client be a part of the cluster as well. If the correct timestamp format is + * not used or the client has a big clock skew, the request may be rejected. In case of client + * supplied timestamps, the monotonicity guarantees for updates will not hold for writes. It + * is recommended to always use the time from the server side unless there is special logic from the + * client side that provides monotonic guarantees. + * + *

    + * A table of CUSTOM timestamp can be converted to SYSTEM type if and only if the timestamps for + * all existing cells have been populated with System.currentTimeMillis(). In case of + * client-supplied timestamps are used, the table SHOULD NOT be converted to SYSTEM type if + * client-supplied timestamps are not representing milliseconds passed since epoch. Converting a + * table from CUSTOM to SYSTEM timestamp makes the timestamps given out to regions monotonically + * non-decreasing. Thus care should be taken so that client clocks are in sync with the cluster. + * CUSTOM and SYSTEM clocks uses the same timestamp representation to tables with or without TTL + * can be altered between these two types. + * + *

    + * A table of CUSTOM or SYSTEM type can be converted to HLC type by altering the table if and only + * if the timestamps for all existing cells have been populated with + * System.currentTimeMillis() and TTL per table or TTL-per-cell is not used. All + * existing cells will have the old timestamp format, and all new existing timestamps will be in HLC + * format. Since usually HLC timestmaps are greater than SYSTEM timestamps, write ordering should + * not be affected. If client application interprets the timestamps, {@link #isLikelyOfType(long)} + * can be used to differentiate between different timestamp types. + * + *

    + * A table with HLC timestamp type SHOULD NOT be converted to any other timestamp type if there is + * data written in the table. + *

    + * References:
    + * [1] http://www.cse.buffalo.edu/tech-reports/2014-04.pdf + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Timestamp { + + private static final FastDateFormat dateFormat + = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss:SSS", TimeZone.getTimeZone("UTC")); + + /** + * Returns the unique type name for the Timestamp implementation. + * @return the unique type name for the timestamp implementation + */ + public abstract String getType(); + + /** + * Converts the given timestamp to the unix epoch timestamp with millisecond resolution. + * Returned timestamp is compatible with System.currentTimeMillis(). + * @return number of milliseconds from epoch + */ + public abstract long toEpochTimeMillis(long timestamp); + + /** + * Converts the given timestamp to the timestamp representation for the Timestamp implementation. + * @param timestamp epoch time in milliseconds + * @return a timestamp representation for the Timestamp implementation. + */ + public abstract long fromEpochTimeMillis(long timestamp); + + /** + * Converts the given physical clock in the given timeunit and logical clock to a 64-bit timestamp + * @param timeUnit the timeunit of physical clock + * @param pt physical clock + * @param lt logical clock + * @return a timestamp in 64 bits + */ + public abstract long toTimestamp(TimeUnit timeUnit, long pt, long lt); + + /** + * Extracts and returns the physical time from the timestamp + * @param timestamp the timestamp + * @return physical time + */ + public abstract long getPhysicalTime(long timestamp); + + /** + * Extracts and returns the logical time from the timestamp + * @param timestamp the timestamp + * @return logical time + */ + public abstract long getLogicalTime(long timestamp); + + /** + * Returns the maximum time representable by the physical clock + * @return maximum timestamp + */ + public abstract long getMaxPhysicalTime(); + + /** + * Returns the maximum time representable by the logical clock + * @return maximum timestamp + */ + public abstract long getMaxLogicalTime(); + + /** + * Returns whether the given timestamp is "likely" of the given timestamp type. Timestamp + * implementations can use the full range of 64bit longs to represent physical and logical + * components of time. However, this method returns whether the given timestamp is a likely + * representation depending on heuristics for the clock implementation. + * + *

  • HLC: Timestamps of HLC type are checked whether they belong to HLC range assuming + * that HLC timestamps will only have > 0 lt component for timestamps corresponding to years after + * 2015. This method will return false if lt > 0 and year is before 2015. + * Due to left shifting for HLC, all millisecond-since-epoch timestamps from SYSTEM type for + * years 1970-10K fall into year 1970 when interpretted as HLC timestamps. Thus, isLikelyOfType() + * will return false for timestamps which are in the year 1970 and lt = 0 when interpreted as of + * type HLC. + *
  • + *
  • SYSTEM: Timestamps of SYSTEM type are assumed to be from milliseconds since + * epoch 1970, till year 3000. This method will return false if the timestamp value falls out + * of the year range 1970 - 3000.
  • + *
  • CUSTOM: Timestamps of CUSTOM type have no assumptions for timestamp formatting. + * This method will always return true.
  • + * + *

    Note that this method uses heuristics which may not hold + * if custom timestamps are intermixed from client side and server side or timestamp + * sources other than system clock are used.

    + * @return whether the given timestamp is heuristically falls within the expected range and format + * for the Timestamp implementation. + */ + public abstract boolean isLikelyOfType(long timestamp); + + /** + * Returns a string for human parsing and debugging + */ + public abstract String toString(long timestamp); + + /** + * + */ + public static Timestamp HLC = new HLTimestamp(); + + /** + * TODO + */ + public static Timestamp SYSTEM = new SystemTimestamp(); + + /** + * + */ + public static Timestamp CUSTOM = new CustomTimestamp(); + + + private static Timestamp[] values = new Timestamp[] {HLC, SYSTEM, CUSTOM}; + + /** + * Returns the Timestamp implementation by its type name. Possible values are: + * "HLC", "SYSTEM", "SYSTEM_MONOTONIC". + * @param type type of the timestamp + * @return a Timestamp implementation + */ + public static Timestamp getByType(String type) { + if (type.equals(HLC.getType())) { + return HLC; + } else if (type.equals(SYSTEM.getType())) { + return SYSTEM; + } else if (type.equals(CUSTOM.getType())) { + return CUSTOM; + } else { + throw new IllegalArgumentException("No Timestamp implementation found for " + type); + } + } + + public static Timestamp[] values() { + return values; + } + + @InterfaceAudience.Private + @InterfaceStability.Evolving + static class HLTimestamp extends Timestamp { + + private static final String NAME = "HLC"; + /** + * Hard coded 42-bits for physical time, allowing us to represent all the dates between + * unix epoch (1970) and year 2039 with signed timestamp comparison. With unsigned comparison + * all dates up to 2109 can be represented. Please fix the timestamp comparison to be unsigned + * in the next 24 years! + */ + @SuppressWarnings("unused") + private static final int BITS_FOR_PT = 42; + + /** + * Remaining 22-bits for logical time, allowing values up to 4,194,304. LT is at the least + * significant part of the 64 bit timestamp, so unsigned comparison can be used for LT. + */ + @VisibleForTesting + static final int BITS_FOR_LT = 22; + + /** + * Max value for physical time, inclusive. This assumes signed comparison. + */ + private static final long PT_MAX_VALUE = 0x1ffffffffffL; + + /** + * Max value for logical time, inclusive. + */ + private static final long LT_MAX_VALUE = 0x3fffffL; + + @Override + public String getType() { + return NAME; + } + + @Override + public long toEpochTimeMillis(long timestamp) { + return getPhysicalTime(timestamp); + } + + @Override + public long fromEpochTimeMillis(long timestamp) { + return toTimestamp(TimeUnit.MILLISECONDS, timestamp, 0); + } + + @Override + public long toTimestamp(TimeUnit timeUnit, long pt, long lt) { + pt = TimeUnit.MILLISECONDS.convert(pt, timeUnit); + return (pt << BITS_FOR_LT) + lt; + } + + @Override + public long getPhysicalTime(long timestamp) { + return (timestamp >>> BITS_FOR_LT); // assume unsigned timestamp + } + + @Override + public long getLogicalTime(long timestamp) { + return timestamp & LT_MAX_VALUE; + } + + @Override + public long getMaxPhysicalTime() { + return PT_MAX_VALUE; + } + + @Override + public long getMaxLogicalTime() { + return LT_MAX_VALUE; + } + + @Override + public boolean isLikelyOfType(long timestamp) { + long pt = getPhysicalTime(timestamp); + long lt = getLogicalTime(timestamp); + + // heuristic 1: Up until year 2015 (1420070400000), lt component cannot be non-zero. + if (pt < 1420070400000L && lt != 0) { + return false; + } else if (pt < 31536000000L) { + // heuristic 2: Even if lt = 0, pt should be before year 1971 (31536000000L). + // Due to left shifting by 22, all epoch ms timestamps from SYSTEM timestamp + // end up in year 1970, even for epoch time for the year 10000. + // this assumes HLC is not used to represent timestamps for year 1970 UTC. + return false; + } + return true; + } + /** + * Returns a string representation for PT and LT components. The format is: + * <yyyy-MM-dd HH:mm:ss:SSS(PT),LT> + * Example: <2015-07-17 16:56:35:891(1437177395891), 0> + */ + @Override + public String toString(long timestamp) { + long pt = getPhysicalTime(timestamp); + long lt = getLogicalTime(timestamp); + return new StringBuilder("<") + .append(dateFormat.format(pt)) + .append("(").append(pt).append(")") + .append(", ").append(lt) + .append(">").toString(); + } + } + + /** + * Implementation of Timestamp interface backed by System.currentTimeMillis(). This is compatible + * with timestamps in HBase that are currentTimeMillis() based. + */ + @InterfaceAudience.Private + @InterfaceStability.Evolving + static class SystemTimestamp extends Timestamp { + private static final String NAME = "SYSTEM"; + + @Override + public String getType() { + return NAME; + } + + @Override + public long toEpochTimeMillis(long timestamp) { + return timestamp; + } + + @Override + public long fromEpochTimeMillis(long timestamp) { + return timestamp; + } + + @Override + public long toTimestamp(TimeUnit timeUnit, long pt, long lt) { + return TimeUnit.MILLISECONDS.convert(pt, timeUnit); + } + + @Override + public long getPhysicalTime(long timestamp) { + return timestamp; + } + + @Override + public long getLogicalTime(long timestamp) { + return 0; + } + + @Override + public long getMaxPhysicalTime() { + return Long.MAX_VALUE; + } + + @Override + public long getMaxLogicalTime() { + return 0; + } + + @Override + public boolean isLikelyOfType(long timestamp) { + // heuristic: the timestamp should be up to year 3K (32503680000000L). + return timestamp < 32503680000000L; + } + + @Override + public String toString(long timestamp) { + long pt = timestamp; + return new StringBuilder("<") + .append(dateFormat.format(pt)) + .append("(").append(pt).append(")") + .append(">").toString(); + } + } + + /** + * Implementation of Timestamp interface for custom timestamps. + */ + @InterfaceAudience.Private + @InterfaceStability.Evolving + static class CustomTimestamp extends SystemTimestamp { + private static final String NAME = "SYSTEM_MONOTONIC"; + + @Override + public String getType() { + return NAME; + } + + @Override + public boolean isLikelyOfType(long timestamp) { + // custom timestamps do not assume any format + return true; + } + } +} \ No newline at end of file diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestTimestamp.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestTimestamp.java new file mode 100644 index 0000000..164fe06 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestTimestamp.java @@ -0,0 +1,310 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.apache.hadoop.hbase.Timestamp.*; +import static org.junit.Assert.*; + +@Category(SmallTests.class) +public class TestTimestamp { + + private static long testPt = 1234567890123L; + private static long testLt = 12; + + /* + * Tests for class methods and for all clocks + */ + + @Test + public void testFromToEpoch() { + for (Timestamp timestamp : Timestamp.values()) { + long wallTime = System.currentTimeMillis(); + long converted = timestamp.toEpochTimeMillis( + timestamp.fromEpochTimeMillis(wallTime)); + + assertEquals(wallTime, converted); + } + } + + @Test + public void testGetByType() { + for (Timestamp timestamp : Timestamp.values()) { + assertEquals(timestamp, Timestamp.getByType(timestamp.getType())); + } + } + + /* + * Tests for HL Clock + */ + @Test + public void testHLCMaxValues() { + // assert 42-bit PT with signed comparison (actual 41 bits) + assertEquals( + (1L << 41) - 1, + HLC.getMaxPhysicalTime()); + + // assert 22-bit LT + assertEquals( + (1L << 22) - 1, + HLC.getMaxLogicalTime()); + + // assert that maximum representable timestamp is Long.MAX_VALUE (assuming signed comparison). + assertEquals( + Long.MAX_VALUE, + HLC.toTimestamp(TimeUnit.MILLISECONDS, + HLC.getMaxPhysicalTime(), + HLC.getMaxLogicalTime()) + ); + } + + @Test + public void testHLCGetPhysicalTime() { + long ts = HLC.toTimestamp(TimeUnit.MILLISECONDS, testPt, testLt); + assertEquals(testPt, HLC.getPhysicalTime(ts)); + } + + @Test + public void testHLCGetLogicalTime() { + long ts = HLC.toTimestamp(TimeUnit.MILLISECONDS, testPt, testLt); + assertEquals(testLt, HLC.getLogicalTime(ts)); + } + + @Test + public void testHLCToString() { + long ts = HLC.toTimestamp(TimeUnit.MILLISECONDS, testPt, testLt); + + assertEquals("<2009-02-13 23:31:30:123(1234567890123), 12>", HLC.toString(ts)); + } + + @Test + public void testHLCToTimestamp() { + long expected = (testPt << 22) + testLt; + // test millisecond + long ts = HLC.toTimestamp(TimeUnit.MILLISECONDS, testPt, testLt); + assertEquals(ts, expected); + + // test nanosecond + ts = HLC.toTimestamp(TimeUnit.NANOSECONDS, TimeUnit.MILLISECONDS.toNanos(testPt), testLt); + assertEquals(ts, expected); + } + + @Test + public void testHLCIsLikelyOfType() throws ParseException { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS Z"); + + // test timestamps of HLC type from year 1971 to 2039 where lt = 0 + System.out.println("==="); + for (int year = 1971; year <= 2039; year += 1) { + Date date = dateFormat.parse(year + "-01-01 11:22:33:444 UTC"); + + // HLC type ts with pt = date and lt = 0 + long ts = HLC.toTimestamp(TimeUnit.MILLISECONDS, date.getTime(), 0); + System.out.println(HLC.toString(ts)); + + assertTrue(HLC.isLikelyOfType(ts)); + } + + // test timestamps of HLC type from year 2015 to 2039 where lt > 0 + System.out.println("==="); + for (int year = 2015; year <= 2039; year += 1) { + Date date = dateFormat.parse(year + "-01-01 11:22:33:444 UTC"); + + // HLC type ts with pt = date and lt = 123 + long ts = HLC.toTimestamp(TimeUnit.MILLISECONDS, date.getTime(), 123); + System.out.println(HLC.toString(ts)); + + assertTrue(HLC.isLikelyOfType(ts)); + } + + // test that timestamps from different years are not HLC type + System.out.println("==="); + for (int year = 1970; year <= 10000 ;year += 10) { + // Stardate 1970 to 10000 + Date date = dateFormat.parse(year + "-01-01 00:00:00:000 UTC"); + long ts = date.getTime(); + System.out.println(SYSTEM.toString(ts)); + System.out.println(SYSTEM.toString(HLC.getPhysicalTime(ts))); + + assertFalse(HLC.isLikelyOfType(ts)); + } + + // test that timestamps up to 2015 are not HLC even if lt = 0 + System.out.println("==="); + for (int year = 1970; year <= 2015; year += 1) { + Date date = dateFormat.parse(year + "-01-01 11:22:33:444 UTC"); + + // reset lt = 0 + long ts = ((date.getTime() + >> HLTimestamp.BITS_FOR_LT) << HLTimestamp.BITS_FOR_LT); + System.out.println(Long.toHexString(ts)); + + System.out.println(SYSTEM.toString(ts)); + System.out.println(SYSTEM.toString(HLC.getPhysicalTime(ts))); + + assertFalse(HLC.isLikelyOfType(ts)); + } + + // test that timestamps from currentTime epoch are not HLC type + System.out.println("==="); + long systemTimeNow = System.currentTimeMillis(); + System.out.println(SYSTEM.toString(systemTimeNow)); + System.out.println(SYSTEM.toString((HLC.getPhysicalTime(systemTimeNow)))); + assertFalse(HLC.isLikelyOfType(systemTimeNow)); + } + + /* + * Tests for System Clock + */ + + @Test + public void testSystemClockMaxValues() { + assertEquals( + (1L << 63) - 1, + SYSTEM.getMaxPhysicalTime()); + + assertEquals(0, SYSTEM.getMaxLogicalTime()); + } + + @Test + public void testSystemClockGetPhysicalTime() { + long ts = SYSTEM.toTimestamp(TimeUnit.MILLISECONDS, testPt, testLt); + assertEquals(testPt, SYSTEM.getPhysicalTime(ts)); + } + + @Test + public void testSystemClockGetLogicalTime() { + long ts = SYSTEM.toTimestamp(TimeUnit.MILLISECONDS, testPt, testLt); + assertEquals(0, SYSTEM.getLogicalTime(ts)); + } + + @Test + public void testSystemClockToString() { + long ts = SYSTEM.toTimestamp(TimeUnit.MILLISECONDS, testPt, testLt); + + assertEquals("<2009-02-13 23:31:30:123(1234567890123)>", SYSTEM.toString(ts)); + } + + @Test + public void testSystemClockToTimestamp() { + // test millisecond + long ts = SYSTEM.toTimestamp(TimeUnit.MILLISECONDS, testPt, testLt); + assertEquals(ts, testPt); + + // test nanosecond + ts = SYSTEM.toTimestamp(TimeUnit.NANOSECONDS, TimeUnit.MILLISECONDS.toNanos(testPt), testLt); + assertEquals(ts, testPt); + } + + @Test + public void testSystemClockIsLikelyOfType() throws ParseException { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS Z"); + + // test that timestamps from 1970 to 3K epoch are of SYSTEM type + System.out.println("==="); + for (int year = 1970; year < 3000 ;year += 10) { + // Stardate 1970 to 10000 + Date date = dateFormat.parse(year + "-01-01 00:00:00:000 UTC"); + long ts = date.getTime(); + System.out.println(SYSTEM.toString(ts)); + System.out.println(SYSTEM.toString(HLC.getPhysicalTime(ts))); + + assertTrue(SYSTEM.isLikelyOfType(ts)); + } + + // test that timestamps from currentTime epoch are of SYSTEM type + System.out.println("==="); + long systemTimeNow = System.currentTimeMillis(); + System.out.println(SYSTEM.toString(systemTimeNow)); + assertTrue(SYSTEM.isLikelyOfType(systemTimeNow)); + + // test timestamps of HLC type from year 1970 to 2039 are not of SYSTEM type + System.out.println("==="); + for (int year = 1970; year <= 2039; year += 1) { + Date date = dateFormat.parse(year + "-01-01 11:22:33:444 UTC"); + + // HLC type ts with pt = date and lt = 0 + long ts = HLC.toTimestamp(TimeUnit.MILLISECONDS, date.getTime(), 0); + System.out.println(HLC.toString(ts)); + System.out.println(SYSTEM.toString(ts)); + + assertFalse(SYSTEM.isLikelyOfType(ts)); + } + } + + /* + * Tests for Custom Clock + */ + + @Test + public void testCustomClockMaxValues() { + assertEquals( + (1L << 63) - 1, + CUSTOM.getMaxPhysicalTime()); + + assertEquals(0, CUSTOM.getMaxLogicalTime()); + } + + @Test + public void testCustomClockGetPhysicalTime() { + long ts = CUSTOM.toTimestamp(TimeUnit.MILLISECONDS, testPt, testLt); + assertEquals(testPt, CUSTOM.getPhysicalTime(ts)); + } + + @Test + public void testCustomClockGetLogicalTime() { + long ts = CUSTOM.toTimestamp(TimeUnit.MILLISECONDS, testPt, testLt); + assertEquals(0, CUSTOM.getLogicalTime(ts)); + } + + @Test + public void testCustomClockToString() { + long ts = CUSTOM.toTimestamp(TimeUnit.MILLISECONDS, testPt, testLt); + + assertEquals("<2009-02-13 23:31:30:123(1234567890123)>", SYSTEM.toString(ts)); + } + + @Test + public void testCustomClockToTimestamp() { + // test millisecond + long ts = CUSTOM.toTimestamp(TimeUnit.MILLISECONDS, testPt, testLt); + assertEquals(ts, testPt); + + // test nanosecond + ts = CUSTOM.toTimestamp(TimeUnit.NANOSECONDS, TimeUnit.MILLISECONDS.toNanos(testPt), testLt); + assertEquals(ts, testPt); + } + + @Test + public void testCustomClockIsLikelyOfType() throws ParseException { + // test that custom clock isLikelyOfType() always returns true + for (int offset = 0; offset < 64; offset++) { + assertTrue(CUSTOM.isLikelyOfType(1L << offset)); + assertTrue(CUSTOM.isLikelyOfType(0x8000000000000000L >> offset)); + } + } +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java index d0aeb6c..c7cb70b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableDescriptors; +import org.apache.hadoop.hbase.Timestamp; import org.apache.hadoop.hbase.TableInfoMissingException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -160,6 +161,7 @@ public class FSTableDescriptors implements TableDescriptors { metaDescriptor.addCoprocessor( "org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint", null, Coprocessor.PRIORITY_SYSTEM, null); + metaDescriptor.setTimestampType(Timestamp.CUSTOM); return metaDescriptor; }