diff --git a/src/docbkx/book.xml b/src/docbkx/book.xml index 3c12169..82b9098 100644 --- a/src/docbkx/book.xml +++ b/src/docbkx/book.xml @@ -863,6 +863,11 @@ System.out.println("md5 digest as string length: " + sbDigest.length); // ret +
Constraints + HBase currently supports 'constraints' in traditional (SQL) database parlance. The advised usage for Constraints is in enforcing business rules for attributes in the table (eg. make sure values are in the range 1-10). + Constraints could be also be used to enforce referential integrity, but this is strongly discouraged as it will dramatically decrease the write throughput of the tables where that checking enabled. +
+ diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index 84a0d1a..c745f9c 100644 --- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import java.util.regex.Matcher; @@ -39,6 +40,9 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableComparable; +import com.google.common.base.Function; +import com.google.common.collect.Iterables; + /** * HTableDescriptor contains the details about an HBase table such as the descriptors of * all the column families, is the table a catalog table, -ROOT- or @@ -412,10 +416,10 @@ public class HTableDescriptor implements WritableComparable { * @see #values */ public String getValue(String key) { - byte[] value = getValue(Bytes.toBytes(key)); + byte[] value = getValue(SERIALIZE_KEY.apply(key)); if (value == null) return null; - return Bytes.toString(value); + return DESERIALIZE_VALUE.apply(value); } /** @@ -465,7 +469,18 @@ public class HTableDescriptor implements WritableComparable { * @see #values */ public void setValue(String key, String value) { - setValue(Bytes.toBytes(key), Bytes.toBytes(value)); + setValue(SERIALIZE_KEY.apply(key), SERIALIZE_VALUE.apply(value)); + } + + /** + * Setter for storing metadata as a (key, value) pair in {@link #values} map + * + * @param key The key. + * @param value The value to store verbatim + * @see #values + */ + public void setValue(String key, byte[] value) { + setValue(SERIALIZE_KEY.apply(key), value); } /** @@ -479,6 +494,16 @@ public class HTableDescriptor implements WritableComparable { } /** + * Remove metadata represented by the key from the {@link #values} map + * + * @param key Key whose key and value we're to remove from the HTableDecriptor + * parameters + */ + public void remove(final String key) { + this.remove(SERIALIZE_KEY.apply(key)); + } + + /** * Check if the readOnly flag of the table is set. If the readOnly flag is * set then the contents of the table can only be read from but not modified. * @@ -711,8 +736,8 @@ public class HTableDescriptor implements WritableComparable { s.append("'"); for (Map.Entry e: values.entrySet()) { - String key = Bytes.toString(e.getKey().get()); - String value = Bytes.toString(e.getValue().get()); + String key = DESERIALIZE_KEY.apply(e.getKey().get()); + String value = DESERIALIZE_VALUE.apply(e.getValue().get()); if (key == null) { continue; } @@ -788,6 +813,7 @@ public class HTableDescriptor implements WritableComparable { * INTERNAL This method is a part of {@link WritableComparable} interface * and is used for de-serialization of the HTableDescriptor over RPC */ + @Override public void readFields(DataInput in) throws IOException { int version = in.readInt(); if (version < 3) @@ -822,6 +848,7 @@ public class HTableDescriptor implements WritableComparable { * INTERNAL This method is a part of {@link WritableComparable} interface * and is used for serialization of the HTableDescriptor over RPC */ + @Override public void write(DataOutput out) throws IOException { out.writeInt(TABLE_DESCRIPTOR_VERSION); Bytes.writeByteArray(out, name); @@ -850,6 +877,7 @@ public class HTableDescriptor implements WritableComparable { * @return 0 if the contents of the descriptors are exactly matching, * 1 if there is a mismatch in the contents */ + @Override public int compareTo(final HTableDescriptor other) { int result = Bytes.compareTo(this.name, other.name); if (result == 0) { @@ -999,11 +1027,9 @@ public class HTableDescriptor implements WritableComparable { // generate a coprocessor key int maxCoprocessorNumber = 0; Matcher keyMatcher; - for (Map.Entry e: - this.values.entrySet()) { - keyMatcher = - HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher( - Bytes.toString(e.getKey().get())); + for (Map.Entry e : new StringifiedIterable( + this.values.entrySet())) { + keyMatcher = HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher(e.getKey()); if (!keyMatcher.matches()) { continue; } @@ -1029,17 +1055,14 @@ public class HTableDescriptor implements WritableComparable { public boolean hasCoprocessor(String className) { Matcher keyMatcher; Matcher valueMatcher; - for (Map.Entry e: - this.values.entrySet()) { - keyMatcher = - HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher( - Bytes.toString(e.getKey().get())); + for (Map.Entry e : new StringifiedIterable( + this.values.entrySet())) { + keyMatcher = HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher(e.getKey()); if (!keyMatcher.matches()) { continue; } - valueMatcher = - HConstants.CP_HTD_ATTR_VALUE_PATTERN.matcher( - Bytes.toString(e.getValue().get())); + valueMatcher = HConstants.CP_HTD_ATTR_VALUE_PATTERN + .matcher((e.getValue())); if (!valueMatcher.matches()) { continue; } @@ -1052,6 +1075,36 @@ public class HTableDescriptor implements WritableComparable { return false; } + /** + * Remove a coprocessor from those set on the table + * @param className Class name of the co-processor + */ + public void removeCoprocessor(String className) { + String match = null; + Matcher keyMatcher; + Matcher valueMatcher; + for (Map.Entry e : new StringifiedIterable( + this.values.entrySet())) { + keyMatcher = HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher(e.getKey()); + if (!keyMatcher.matches()) { + continue; + } + valueMatcher = HConstants.CP_HTD_ATTR_VALUE_PATTERN.matcher(e.getValue()); + if (!valueMatcher.matches()) { + continue; + } + // get className and compare + String clazz = valueMatcher.group(2).trim(); // classname is the 2nd field + // remove the CP if it is present + if (clazz.equals(className.trim())) { + match = e.getKey(); + break; + } + } + // if we found a match, remove it + if (match != null) + this.remove(match); + } /** * Returns the {@link Path} object representing the table directory under @@ -1083,7 +1136,6 @@ public class HTableDescriptor implements WritableComparable { HConstants.FOREVER, StoreFile.BloomType.NONE.toString(), HConstants.REPLICATION_SCOPE_LOCAL)}); - public void setOwner(User owner) { setOwnerString(owner != null ? owner.getShortName() : null); } @@ -1105,4 +1157,99 @@ public class HTableDescriptor implements WritableComparable { // .META. and -ROOT- should return system user as owner, not null (see MasterFileSystem.java:bootstrap()). return null; } + + // Serialization and deserialization of keys and values in metainformation + // about the table + public static final Function SERIALIZE_KEY = new Function() { + @Override + public byte[] apply(String input) { + return Bytes.toBytes(input); + } + }; + + public static final Function DESERIALIZE_KEY = new Function() { + @Override + public String apply(byte[] input) { + return Bytes.toString(input); + } + }; + + public static final Function SERIALIZE_VALUE = new Function() { + @Override + public byte[] apply(String input) { + return Bytes.toBytes(input); + } + }; + + public static final Function DESERIALIZE_VALUE = new Function() { + @Override + public String apply(byte[] input) { + return Bytes.toString(input); + } + }; + + // Maybe this would make more sense using a static method that just wraps the + // creation? + @SuppressWarnings("synthetic-access") + private static class StringifiedIterable implements + Iterable> { + private final Iterable> delegate; + private static final MutableEntry entry = new MutableEntry(); + private static final Function, Entry> transformToStrings = new Function, Entry>() { + + @Override + public Entry apply( + Entry e) { + entry.setKey(DESERIALIZE_KEY.apply(e.getKey().get())); + entry.setValue(DESERIALIZE_VALUE.apply(e.getValue().get())); + return entry; + } + }; + + /** + * Convert the iterable of bytes to the string repesentation of each entry. + * @param iterable to be wrapped + */ + public StringifiedIterable( + Iterable> iterable) { + delegate = Iterables.transform(iterable, transformToStrings); + } + + @Override + public Iterator> iterator() { + return delegate.iterator(); + } + } + + /** + * Entry that has a fully mutable key and value. Saves on object reuse. + * + * @param Key class to store + * @param Value class to store + */ + private static class MutableEntry implements Map.Entry { + private K key; + private V value; + + @Override + public K getKey() { + return key; + } + + public K setKey(K key) { + this.key = key; + return this.key; + } + + @Override + public V getValue() { + return value; + } + + @Override + public V setValue(V value) { + this.value = value; + return this.value; + } + } } diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/BaseConstraint.java b/src/main/java/org/apache/hadoop/hbase/constraint/BaseConstraint.java new file mode 100644 index 0000000..b432c65 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/BaseConstraint.java @@ -0,0 +1,43 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.conf.Configuration; + +/** + * Base class to use when actually implementing a {@link Constraint}. It takes + * care of getting and setting of configuration for the constraint. + */ +public abstract class BaseConstraint implements + Constraint { + + private Configuration conf; + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return this.conf; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java b/src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java new file mode 100644 index 0000000..8c09770 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java @@ -0,0 +1,77 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Put; + +/** + * Apply a {@link Constraint} (in traditional database terminology) to a HTable. + * Any number of {@link Constraint Constraints} can be added to the table, in + * any order. + *

+ * A {@link Constraint} must be added to a table before the table is loaded via + * {@link Constraints#add(HTableDescriptor, Class...)} or + * {@link Constraints#add(HTableDescriptor, org.apache.hadoop.hbase.util.Pair...)} + * (if you want to add a configuration with the {@link Constraint}). Constraints + * will be run in the order that they are added. Further, a Constraint will be + * configured before it is run (on load). + *

+ * See {@link Constraints#enableConstraint(HTableDescriptor, Class)} and + * {@link Constraints#disableConstraint(HTableDescriptor, Class)} for + * enabling/disabling of a given {@link Constraint} after it has been added. + *

+ * If a {@link Put} is not valid, the Constraint should throw some sort + * {@link ConstraintException} indicating that the {@link Put} has failed. When + * this exception is thrown, not further retries of the {@link Put} are + * attempted nor are any other {@link Constraint Constraints} attempted (the + * {@link Put} is clearly not valid). Therefore, there are performance + * implications in the order in which {@link BaseConstraint Constraints} are + * specified. + *

+ * If a {@link Constraint} fails to fail the {@link Put} via a + * {@link ConstraintException}, but instead throws a {@link RuntimeException}, + * the entire constraint processing mechanism ({@link ConstraintProcessor}) will + * be unloaded from the table. This ensures that the region server is still + * functional, but not more {@link Put Puts} will be checked via + * {@link Constraint Constraints}. + *

+ * Further, {@link Constraint Constraints} probably not be used to enforce + * cross-table references as it will cause tremendous write slowdowns, but it is + * possible. + *

+ * NOTE: Implementing classes must have a nullary (no-args) constructor + */ +public interface Constraint extends Configurable { + + /** + * Check a {@link Put} to ensure it is valid for the table. If the {@link Put} + * is valid, then just return from the method. Otherwise, throw an + * {@link Exception} specifying what happened. This {@link Exception} is + * propagated back to the client so you can see what caused the {@link Put} to + * fail. + * @param p {@link Put} to check + * @throws ConstraintException when the {@link Put} does not match the + * constraint. + */ + public void check(Put p) throws ConstraintException; + +} diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/ConstraintException.java b/src/main/java/org/apache/hadoop/hbase/constraint/ConstraintException.java new file mode 100644 index 0000000..4ca8442 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/ConstraintException.java @@ -0,0 +1,48 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.client.Put; + +/** + * Exception that a user defined constraint throws on failure of a {@link Put}. + *

+ * Does NOT attempt the {@link Put} multiple times, since the constraint + * should fail every time for the same {@link Put} (it should be + * idempotent). + */ +public class ConstraintException extends DoNotRetryIOException { + private static final long serialVersionUID = 1197446454511704140L; + + public ConstraintException() { + super(); + } + + public ConstraintException(String msg) + { + super(msg); + } + + + public ConstraintException(String msg, Throwable cause) { + super(msg, cause); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/ConstraintProcessor.java b/src/main/java/org/apache/hadoop/hbase/constraint/ConstraintProcessor.java new file mode 100644 index 0000000..1323d1b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/ConstraintProcessor.java @@ -0,0 +1,90 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; + +/*** + * Processes multiple {@link Constraint Constraints} on a given table. + *

+ * This is an ease of use mechanism - all the functionality here could be + * implemented on any given system by a coprocessor. + */ +public class ConstraintProcessor extends BaseRegionObserver { + + private static final Log LOG = LogFactory.getLog(ConstraintProcessor.class); + + private final ClassLoader classloader; + + private List constraints = new ArrayList(); + + /** + * Create the constraint processor. + *

+ * Stores the current classloader. + */ + public ConstraintProcessor() { + classloader = this.getClass().getClassLoader(); + } + + @Override + public void start(CoprocessorEnvironment environment) { + if (!(environment instanceof RegionCoprocessorEnvironment)) + throw new IllegalArgumentException( + "Constraints only act on regions - started in an environment that was not a region"); + RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) environment; + HTableDescriptor desc = env.getRegion().getTableDesc(); + try { + this.constraints = Constraints.getConstraints(desc, classloader); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Added " + constraints.size() + " constraints"); + } + if (LOG.isInfoEnabled()) { + LOG.info("Finished loading user Constraints on table: " + + new String(desc.getName())); + } + + } + + @Override + public void prePut(ObserverContext e, Put put, + WALEdit edit, boolean writeToWAL) throws IOException { + // check the put against the stored constraints + for (Constraint c : constraints) + c.check(put); + // if we made it here, then the Put is valid + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/Constraints.java b/src/main/java/org/apache/hadoop/hbase/constraint/Constraints.java new file mode 100644 index 0000000..795dcce --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/Constraints.java @@ -0,0 +1,468 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import static org.apache.hadoop.hbase.HTableDescriptor.DESERIALIZE_KEY; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Pair; + +/** + * Utilities for adding/removing constraints from a table. + *

+ * Constraints can be added on table load time, via the {@link HTableDescriptor}. + *

+ * NOTE: this class is NOT thread safe. Concurrent setting/enabling/disabling of + * constraints can cause constraints to be run at incorrect times or not at all. + */ +public final class Constraints { + private Constraints() { + } + + private static final Log LOG = LogFactory.getLog(Constraints.class); + private static final String CONSTRAINT_HTD_KEY_PREFIX = "constraint $"; + private static final Pattern CONSTRAINT_HTD_ATTR_KEY_PATTERN = Pattern + .compile(CONSTRAINT_HTD_KEY_PREFIX, Pattern.LITERAL); + + // Has a Configuration bytes + private static final byte HAS_CONFIGURATION = 1; + private static final byte DOES_N0T_HAVE_CONFIGURATION = 0; + + // If a constraint is enabled bytes + private static final byte ENABLED = 1; + private static final byte DISABLED = 0; + + private static String COUNTER_KEY = "hbase.constraint.counter"; + + /** + * Enable constraints on a table. + *

+ * Currently, if you attempt to add a constraint to the table, then + * Constraints will automatically be turned on. + * @param desc table description to add the processor + * @throws IOException If the {@link ConstraintProcessor} CP couldn't be added + * to the table. + */ + public static void enable(HTableDescriptor desc) throws IOException { + // if the CP has already been loaded, do nothing + String clazz = ConstraintProcessor.class.getName(); + if (desc.hasCoprocessor(clazz)) + return; + + // add the constrain processor CP to the table + desc.addCoprocessor(clazz); + } + + /** + * Turn off processing constraints for a given table, even if constraints have + * been turned on or added. + * @param desc {@link HTableDescriptor} where to disable {@link Constraint + * Constraints}. + */ + public static void disable(HTableDescriptor desc) { + desc.removeCoprocessor(ConstraintProcessor.class.getName()); + } + + /** + * Remove all {@link Constraint Constraints} that have been added to the table + * and turn off the constraint processing. + *

+ * All {@link Configuration Configurations} and their associated + * {@link Constraint} are removed. + * @param desc {@link HTableDescriptor} to remove {@link Constraint + * Constraints} from. + */ + public static void remove(HTableDescriptor desc) { + disable(desc); + List keys = new ArrayList(); + // loop through all the key, values looking for constraints + for (Map.Entry e : desc + .getValues().entrySet()) { + String key = DESERIALIZE_KEY.apply(e.getKey().get()); + String[] className = CONSTRAINT_HTD_ATTR_KEY_PATTERN.split(key); + if (className.length == 2) { + keys.add(e.getKey()); + } + } + // now remove all the keys we found + for (ImmutableBytesWritable key : keys) { + desc.remove(key.get()); + } + } + + /** + * Check to see if the Constraint is currently set. + * @param desc {@link HTableDescriptor} to check + * @param clazz {@link Constraint} class to check for. + * @return true if the {@link Constraint} is present, even if it is + * disabled. false otherwise. + */ + public static boolean has(HTableDescriptor desc, + Class clazz) { + return getKeyValueForClass(desc, clazz) != null; + } + + /** + * Get the kv {@link Entry} in the descriptor for the specified class + * @param desc {@link HTableDescriptor} to read + * @param clazz to search for + * @return the {@link Pair} of in the table, if that class is + * present. null otherwise. + */ + private static Pair getKeyValueForClass( + HTableDescriptor desc, Class clazz) { + // get the serialized version of the constraint + String key = serializeConstraintClass(clazz); + String value = desc.getValue(key); + if( value == null) + return null; + return new Pair(key, HTableDescriptor.SERIALIZE_VALUE.apply(value)); + } + + /** + * Add configuration-less constraints to the table. + *

+ * This will overwrite any configuration associated with the previous + * constraint of the same class. + * @param desc {@link HTableDescriptor} to add a {@link Constraint} + * @param constraints {@link Constraint Constraints} to add. All constraints + * are considered automatically enabled on add + * @throws IOException If constraint could not be serialized/added to table + */ + public static void add(HTableDescriptor desc, + Class... constraints) throws IOException { + // make sure constraints are enabled + enable(desc); + long priority = getNextPriority(desc); + for (Class clazz : constraints) { + writeConstraint(desc, clazz, null, priority++); + } + updateLatestPriority(desc, priority); + } + + /** + * Add constraints and their associated configurations to the table. + *

+ * Adding the same constraint class twice will overwrite the first + * constraint's configuration + * @param desc {@link HTableDescriptor} to add a {@link Constraint} + * @param constraints {@link Pair} of a {@link Constraint} and its associated + * {@link Configuration}. The Constraint will be configured on load + * with the specified configuration.All constraints are considered + * automatically enabled on add + * @throws IOException if any constraint could not be deserialized. Assumes if + * 1 constraint is not loaded properly, something has gone terribly + * wrong and that all constraints need to be enforced. + */ + public static void add(HTableDescriptor desc, + Pair, Configuration>... constraints) + throws IOException { + enable(desc); + long priority = getNextPriority(desc); + for (Pair, Configuration> pair : constraints) { + writeConstraint(desc, pair.getFirst(), pair.getSecond(), priority++); + } + updateLatestPriority(desc, priority); + } + + private static void writeConstraint(HTableDescriptor desc, + Class clazz, Configuration conf, long priority) + throws IOException { + writeConstraint(desc, serializeConstraintClass(clazz), ENABLED, priority, + conf); + } + + /** + * Just write the class to the byte [] we are expecting + * @param clazz + * @return + */ + private static String serializeConstraintClass( + Class clazz) { + String constraintClazz = clazz.getName(); + return CONSTRAINT_HTD_KEY_PREFIX + constraintClazz; + } + + private static void writeConstraint(HTableDescriptor desc, String key, + byte enabled, long priority, Configuration conf) throws IOException { + // write the data associated with the class + byte[] data = new byte[0]; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(bos); + + // enable the constraint + dos.writeByte(ENABLED); + // write store the priority of the constraint + dos.writeLong(priority); + // note that it has a configuration + if (conf == null) + dos.writeByte(DOES_N0T_HAVE_CONFIGURATION); + else { + dos.writeByte(HAS_CONFIGURATION); + conf.write(dos); + } + dos.flush(); + data = bos.toByteArray(); + desc.setValue(key, data); + } + + + private static long getNextPriority(HTableDescriptor desc) { + String value = desc.getValue(COUNTER_KEY); + + long priority; + // get the current priority + if (value == null) + priority = 0L; + else + priority = Long.parseLong(value) + 1; + + return priority; + } + + private static void updateLatestPriority(HTableDescriptor desc, long priority) { + // update the max priority + desc.setValue(COUNTER_KEY, Long.toString(priority)); + } + + /** + * Update the configuration for the {@link Constraint}. Does not change the + * order in which the constraint is run. If the + * @param desc {@link HTableDescriptor} to update + * @param clazz {@link Constraint} to update + * @param configuration to update the {@link Constraint} with. + * @throws IOException if the Constraint was not stored correctly + * @throws IllegalArgumentException if the Constraint was not present on this + * table. + */ + public static void setConfiguration(HTableDescriptor desc, + Class clazz, Configuration configuration) + throws IOException, IllegalArgumentException { + // get the entry for this class + Pair e = getKeyValueForClass( + desc, clazz); + if(e == null) + throw new IllegalArgumentException("Constraint: " + clazz.getName() + + " is not associated with this table."); + + // read in the info about the constraint + byte[] values = e.getSecond(); + DataInputStream is = new DataInputStream(new ByteArrayInputStream(values)); + byte enabled = is.readByte(); + long priority = is.readLong(); + + // update the current value + writeConstraint(desc, e.getFirst(), enabled, priority, configuration); + } + + /** + * Remove the constraint (and associated information) for the table + * descriptor. + * @param desc {@link HTableDescriptor} to modify + * @param clazz {@link Constraint} class to remove + */ + public static void remove(HTableDescriptor desc, + Class clazz) { + String key = serializeConstraintClass(clazz); + desc.remove(key); + } + + /** + * Enable the given {@link Constraint}. Retains all the information (e.g. + * Configuration) for the {@link Constraint}, but makes sure that it gets + * loaded on the table. + * @param desc {@link HTableDescriptor} to modify + * @param clazz {@link Constraint} to enable + */ + public static void enableConstraint(HTableDescriptor desc, + Class clazz) { + Pair entry = getKeyValueForClass( + desc, clazz); + if (entry == null) + throw new IllegalArgumentException("Constraint: " + clazz.getName() + + " is not associated with this table. You can't enable it!"); + + byte[] values = entry.getSecond(); + values[0] = ENABLED; + desc.setValue(entry.getFirst(), values); + } + + /** + * Disable the given {@link Constraint}. Retains all the information (e.g. + * Configuration) for the {@link Constraint}, but it just doesn't load the + * {@link Constraint} on the table. + * @param desc {@link HTableDescriptor} to modify + * @param clazz {@link Constraint} to disable. + */ + public static void disableConstraint(HTableDescriptor desc, + Class clazz) { + Pair entry = getKeyValueForClass( + desc, clazz); + if(entry == null) + throw new IllegalArgumentException("Constraint: " + clazz.getName() + + " is not associated with this table. You can't disable it"); + + byte[] values = entry.getSecond(); + values[0] = DISABLED; + desc.setValue(entry.getFirst(), values); + } + + /** + * Check to see if the given constraint is enabled. + * @param desc {@link HTableDescriptor} to check. + * @param clazz {@link Constraint} to check for + * @return true if the {@link Constraint} is present and enabled. + * false otherwise. + * @throws IOException If the constraint has improperly stored in the table + */ + public static boolean enabled(HTableDescriptor desc, + Class clazz) throws IOException { + // get the kv + Pair entry = getKeyValueForClass( + desc, clazz); + // its not enabled so just return false. In fact, its not even present! + if (entry == null) + return false; + // get the info about the constraint + byte[] values = entry.getSecond(); + DataInputStream is = new DataInputStream(new ByteArrayInputStream(values)); + return is.readByte() == ENABLED; + } + + /** + * Get the constraints stored in the table descriptor + * @param desc To read from + * @param classloader To use when loading classes + * @return List of configured {@link Constraint Constraints} + * @throws IOException if any part of reading/arguments fails + */ + static List getConstraints(HTableDescriptor desc, + ClassLoader classloader) throws IOException { + List constraints = new ArrayList(); + // loop through all the key, values looking for constraints + for (Map.Entry e : desc + .getValues().entrySet()) { + String key = DESERIALIZE_KEY.apply(e.getKey().get()).trim(); + String[] className = CONSTRAINT_HTD_ATTR_KEY_PATTERN.split(key); + if (className.length == 2) { + key = className[1]; + if (LOG.isDebugEnabled()) { + LOG.debug("Loading constraint:" + key); + } + + // read in the rest of the constraint + byte[] values = e.getValue().get(); + DataInputStream is = new DataInputStream(new ByteArrayInputStream( + values)); + + // if it is disabled, skip it + if (is.readByte() == DISABLED) { + if (LOG.isDebugEnabled()) + LOG.debug("Constraint: " + key + " is DISABLED - skipping it"); + continue; + } + + long priority = is.readLong(); + + // if it has a configuration, read it in + Configuration conf = new Configuration(); + if (is.readByte() == HAS_CONFIGURATION) { + LOG.debug("Loading configuration for constraint"); + conf.readFields(is); + } + + try { + Class clazz = classloader.loadClass(key) + .asSubclass(Constraint.class); + Constraint constraint = clazz.newInstance(); + constraint.setConf(conf); + constraints.add(new OrderedConstraint(constraint, priority)); + } catch (ClassNotFoundException e1) { + throw new IOException(e1); + } catch (InstantiationException e1) { + throw new IOException(e1); + } catch (IllegalAccessException e1) { + throw new IOException(e1); + } + } + } + Collections.sort(constraints); + return constraints; + } + + /** + * Helper class to enforce ordering of constraints on load. + *

+ * Just delegates through to the constraint for checking, but keeps track of + * the priority for sorting. + */ + static class OrderedConstraint implements + Comparable, Constraint, Configurable { + + private final long priority; + final Constraint delegate; + + public OrderedConstraint(Constraint delegate, long priority) { + this.delegate = delegate; + this.priority = priority; + } + + @Override + public void check(Put p) throws ConstraintException { + delegate.check(p); + } + + @Override + public int compareTo(OrderedConstraint arg0) { + return Long.valueOf(priority).compareTo(arg0.priority); + } + + @Override + public Configuration getConf() { + throw new UnsupportedOperationException( + "Ordered Constraints don't support configuration"); + } + + @Override + public void setConf(Configuration conf) { + throw new UnsupportedOperationException( + "Ordered Constraints don't support configuration"); + } + + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/IntegerConstraint.java b/src/main/java/org/apache/hadoop/hbase/constraint/IntegerConstraint.java new file mode 100644 index 0000000..347833e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/IntegerConstraint.java @@ -0,0 +1,51 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Put; + +/** + * Check to make sure that the value in the {@link Put} is an integer, otherwise + * reject it. + */ +public class IntegerConstraint extends BaseConstraint { + + @Override + public void check(Put p) throws ConstraintException { + Map> familyMap = p.getFamilyMap(); + for (List kvs : familyMap.values()) { + for (KeyValue kv : kvs) { + // just make sure that we can actually pull out an int + // this will automatically throw a NumberFormatException if we try to + // store something that isn't an Integer. + try { + Integer.parseInt(new String(kv.getBuffer())); + } catch (NumberFormatException e) { + throw new ConstraintException("Value in Put (" + p + + ") was not a String-encoded integer", e); + } + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/package-info.java b/src/main/java/org/apache/hadoop/hbase/constraint/package-info.java new file mode 100644 index 0000000..b50b062 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/package-info.java @@ -0,0 +1,190 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Restrict the domain of a data attribute, often times to fulfill business rules/requirements. + * +

+

Table of Contents

+ +

+ +

Overview

+ Constraints are used to enforce business rules in a database. + By checking all {@link org.apache.hadoop.hbase.client.Put Puts} on a given table, you can enforce very specific data policies. + For instance, you can ensure that a certain column family-column qualifier pair always has a value between 1 and 10. + Otherwise, the {@link org.apache.hadoop.hbase.client.Put} is rejected and the data integrity is maintained. +

+ Constraints are designed to be configurable, so a constraints can be used across different tables, but implement different behavior depending on the specific configuration given to that constraint. +

+ By adding a constraint to a table (see Example Usage), constraints will automatically enabled. + You also then have the option of disabling (just 'turn off') or remove (delete all associated information) constraints on a table. + If you remove all constraints (see {@link org.apache.hadoop.hbase.constraint.Constraints#remove(org.apache.hadoop.hbase.HTableDescriptor)}, you must re-add any {@link org.apache.hadoop.hbase.constraint.Constraint} you want on that table. + However, if they are just disabled (see {@link org.apache.hadoop.hbase.constraint.Constraints#disable(org.apache.hadoop.hbase.HTableDescriptor)}, all you need to do is enable constraints again, and everything will be turned back on. + Individual constraints can also be individually enabled, disabled or removed without affecting others. +

+ By default, constraints are disabled on a table. + This means you will not see any slow down on a table if constraints are not enabled. +

+ NOTE Constraints are run in the order that they are added to a table. This has implications for what order constraints should be added to a table. +

+ Under the hood, constraints are implemented as a Coprocessor (see {@link org.apache.hadoop.hbase.constraint.ConstraintProcessor} if you are interested). + +

Example usage

+ First, you must define a {@link org.apache.hadoop.hbase.constraint.Constraint}. + The best way to do this is to extend {@link org.apache.hadoop.hbase.constraint.BaseConstraint}, which takes care of some of the more mundane details of using a {@link org.apache.hadoop.hbase.constraint.Constraint}. +

+ Let's look at one of the available, built-in constraints, {@link org.apache.hadoop.hbase.constraint.IntegerConstraint} (there are also several simple examples in the tests). + The {@link org.apache.hadoop.hbase.constraint.IntegerConstraint} checks to make sure that the value is a String-encoded int. + It is really simple to implement this kind of constraint, the only method needs to be implemented is {@link org.apache.hadoop.hbase.constraint.Constraint#check(org.apache.hadoop.hbase.client.Put)}: + +

+
+ public void check(Put p) throws ConstraintException {
+ 
+ Map<byte[], List<KeyValue>> familyMap = p.getFamilyMap();
+ 
+ for (List <KeyValue> kvs : familyMap.values()) {
+   for (KeyValue kv : kvs) {
+   
+   // just make sure that we can actually pull out an int
+   // this will automatically throw a NumberFormatException if we try to
+   // store something that isn't an Integer.
+    
+   try {
+   Integer.parseInt(new String(kv.getBuffer()));
+   } catch (NumberFormatException e) {
+   throw new ConstraintException("Value in Put (" + p
+   + ") was not a String-encoded integer", e);
+ } } } 
+ 
+
+

+ Note that all exceptions that you expect to be thrown must be caught and then rethrown as a {@link org.apache.hadoop.hbase.constraint.ConstraintException}. + This way, you can be sue that a {@link org.apache.hadoop.hbase.client.Put} fails for an expected reason, rather than for any reason. + For example, an {@link java.lang.OutOfMemoryError} is probably indicative of an inherent problem in the {@link org.apache.hadoop.hbase.constraint.Constraint}, rather than a failed {@link org.apache.hadoop.hbase.client.Put}. +

+ If an unexpected exception is thrown (for example, any kind of uncaught {@link java.lang.RuntimeException}), constraint-checking will be 'unloaded' from the regionserver where that error occurred. + This means no further {@link org.apache.hadoop.hbase.constraint.Constraint Constraints} will be checked on that server until it is reloaded. This is done to ensure the system remains as available as possible. + Therefore, be careful when writing your own Constraint. +

+ So now that we have a Constraint, we want to add it to a table. It's as easy as: + +

+
+ HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
+ ...
+ Constraints.add(desc, IntegerConstraint.class);
+ 
+

+ Once we added the IntegerConstraint, constraints will be enabled on the table (once it is created) and we will always check to make sure that the value is an String-encoded integer. +

+ However, suppose we also write our own constraint, MyConstraint.java. + First, you need to make sure this class-files are in the classpath (in a jar) on the regionserver where that constraint will be run. +

+ Suppose that MyConstraint also uses a Configuration (see {@link org.apache.hadoop.hbase.constraint.Constraint#getConf()}). + Then adding MyConstraint looks like this: + +

+
+ HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
+ Configuration conf = new Configuration();
+ ...
+ (add values to the conf)
+ (modify the table descriptor)
+ ...
+ Constraints.add(desc, new Pair(MyConstraint.class, conf));
+ 
+

+ At this point we added both the {@link org.apache.hadoop.hbase.constraint.IntegerConstraint} and MyConstraint to the table, the {@link org.apache.hadoop.hbase.constraint.IntegerConstraint} will be run first, followed by MyConstraint. +

+ Suppose we realize that the {@link org.apache.hadoop.conf.Configuration} for MyConstraint is actually wrong when it was added to the table. Note, when it is added to the table, it is not added by reference, but is instead copied into the {@link org.apache.hadoop.hbase.HTableDescriptor}. + Thus, to change the {@link org.apache.hadoop.conf.Configuration} we are using for MyConstraint, we need to do this: + +

+
+ (add/modify the conf)
+ ...
+ Constraints.setConfiguration(desc, MyConstraint.class, conf);
+ 
+

+ This will overwrite the previous configuration for MyConstraint, but not change the order of the constraint nor if it is enabled/disabled. +

+ Note that the same constraint class can be added multiple times to a table without repercussion. + A use case for this is the same constraint working differently based on its configuration. + +

+ Suppose then we want to disable just MyConstraint. Its as easy as: +

+
+ Constraints.disable(desc, MyConstraint.class);
+ 
+

+ This just turns off MyConstraint, but retains the position and the configuration associated with MyConstraint. + Now, if we want to re-enable the constraint, its just another one-liner: +

+
+ Constraints.enable(desc, MyConstraint.class);
+ 
+

+ Similarly, constraints on the entire table are disabled via: +

+
+ Constraints.disable(desc);
+ 
+

+ Or enabled via: + +

+
+ Constraints.enable(desc);
+ 
+

+ Lastly, suppose you want to remove MyConstraint from the table, including with position it should be run at and its configuration. + This is similarly simple: +

+
+ Constraints.remove(desc, MyConstraint.class);
+ 
+

+ Also, removing all constraints from a table is similarly simple: +

+
+ Constraints.remove(desc);
+ 
+ This will remove all constraints (and associated information) from the table and turn off the constraint processing. + +

Caveats

+ In traditional (SQL) databases, Constraints are often used to enforce referential integrity. + However, in HBase, this will likely cause significant overhead and dramatically decrease the number of {@link org.apache.hadoop.hbase.client.Put Puts}/second possible on a table. + This is because to check the referential integrity when making a {@link org.apache.hadoop.hbase.client.Put}, one must block on a scan for the 'remote' table, checking for the valid reference. + For millions of {@link org.apache.hadoop.hbase.client.Put Puts} a second, this will breakdown very quickly. + There are several options around the blocking behavior including, but not limited to: +
    +
  • Create a 'pre-join' table where the keys are already denormalized
  • +
  • Designing for 'incorrect' references
  • +
  • Using an external enforcement mechanism
  • +
+ */ +package org.apache.hadoop.hbase.constraint; \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java b/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java new file mode 100644 index 0000000..98106ce --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java @@ -0,0 +1,47 @@ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.junit.Test; + +/** + * Test setting values in the descriptor + */ +public class TestHTableDescriptor { + + /** + * Test cps in the table description + * @throws Exception + */ + @Test + public void testGetSetRemoveCP() throws Exception { + HTableDescriptor desc = new HTableDescriptor("table"); + // simple CP + String className = BaseRegionObserver.class.getName(); + // add and check that it is present + desc.addCoprocessor(className); + assertTrue(desc.hasCoprocessor(className)); + // remove it and check that it is gone + desc.removeCoprocessor(className); + assertFalse(desc.hasCoprocessor(className)); + } + + /** + * Test that we add and remove strings from settings properly. + * @throws Exception + */ + @Test + public void testRemoveString() throws Exception { + HTableDescriptor desc = new HTableDescriptor("table"); + String key = "Some"; + String value = "value"; + desc.setValue(key, value); + assertEquals(value, desc.getValue(key)); + desc.remove(key); + assertEquals(null, desc.getValue(key)); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/AllFailConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/AllFailConstraint.java new file mode 100644 index 0000000..308e3c3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/AllFailConstraint.java @@ -0,0 +1,33 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.client.Put; + +/** + * Always fail the put. + */ +public class AllFailConstraint extends BaseConstraint { + + @Override + public void check(Put p) throws ConstraintException { + throw new ConstraintException("AllFailConstraint fails for all puts"); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/AllPassConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/AllPassConstraint.java new file mode 100644 index 0000000..1710282 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/AllPassConstraint.java @@ -0,0 +1,34 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.client.Put; + +/** + * Simple test constraint that always allows the put to pass + */ +public class AllPassConstraint extends BaseConstraint { + + @Override + public void check(Put p) { + // Do nothing - it passes + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/CheckConfigurationConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/CheckConfigurationConstraint.java new file mode 100644 index 0000000..b6f7b1d --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/CheckConfigurationConstraint.java @@ -0,0 +1,58 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Put; + +/** + * Test Constraint to check to make sure the configuration is set + */ +public class CheckConfigurationConstraint implements Constraint { + + + private static String key = "testKey"; + private static String value = "testValue"; + + public static Configuration getConfiguration() { + Configuration conf = new Configuration(); + conf.set(key, value); + return conf; + } + + @Override + public Configuration getConf() { + return null; + } + + @Override + public void check(Put p) { + // NOOP + } + + @Override + public void setConf(Configuration conf) { + String val = conf.get(key); + if (val == null || !val.equals(value)) + throw new IllegalArgumentException( + "Configuration was not passed correctly"); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/IntegrationTestConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/IntegrationTestConstraint.java new file mode 100644 index 0000000..f51fc31 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/IntegrationTestConstraint.java @@ -0,0 +1,259 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Do the complex integration tests of constraints + */ +public class IntegrationTestConstraint { + private static final Log LOG = LogFactory + .getLog(IntegrationTestConstraint.class); + + private static HBaseTestingUtility util; + private static final byte[] tableName = Bytes.toBytes("test"); + private static final byte[] dummy = Bytes.toBytes("dummy"); + private static final byte[] row1 = Bytes.toBytes("r1"); + private static final byte[] test = Bytes.toBytes("test"); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + util = new HBaseTestingUtility(); + util.startMiniCluster(); + } + + /** + * Test that we run a passing constraint + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test + public void testConstraintPasses() throws Exception { + // create the table + // it would be nice if this was also a method on the util + HTableDescriptor desc = new HTableDescriptor(tableName); + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + // add a constraint + Constraints.add(desc, CheckWasRunConstraint.class); + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we don't fail on a valid put + Put put = new Put(row1); + byte[] value = Integer.toString(10).getBytes(); + put.add(dummy, new byte[0], value); + table.put(put); + + assertTrue(CheckWasRunConstraint.wasRun); + } + + /** + * Test that constraints will fail properly + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test(timeout = 10000) + public void testConstraintFails() throws Exception { + + // create the table + // it would be nice if this was also a method on the util + HTableDescriptor desc = new HTableDescriptor(tableName); + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + + // add a constraint that is sure to fail + Constraints.add(desc, AllFailConstraint.class); + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we do fail on violation + Put put = new Put(row1); + put.add(dummy, new byte[0], "fail".getBytes()); + LOG.warn("Doing put in table"); + try { + table.put(put); + fail("This put should not have suceeded - AllFailConstraint was not run!"); + } catch (RetriesExhaustedWithDetailsException e) { + List causes = e.getCauses(); + assertEquals( + "More than one failure cause - should only be the failure constraint exception", + 1, causes.size()); + Throwable t = causes.get(0); + assertEquals(ConstraintException.class, t.getClass()); + } + } + + /** + * Check that if we just disable one constraint, then + * @throws Throwable + */ + @SuppressWarnings("unchecked") + @Test + public void testDisableConstraint() throws Throwable { + // create the table + HTableDescriptor desc = new HTableDescriptor(tableName); + // add a family to the table + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + // add a constraint to make sure it others get run + Constraints.add(desc, CheckWasRunConstraint.class); + + // Add Constraint to check + Constraints.add(desc, AllFailConstraint.class); + + // and then disable the failing constraint + Constraints.disableConstraint(desc, AllFailConstraint.class); + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we don't fail because its disabled + Put put = new Put(row1); + put.add(dummy, new byte[0], "pass".getBytes()); + table.put(put); + + assertTrue(CheckWasRunConstraint.wasRun); + } + + /** + * Test that if we disable all constraints, then nothing gets run + * @throws Throwable + */ + @SuppressWarnings("unchecked") + @Test + public void testDisableConstraints() throws Throwable { + // create the table + HTableDescriptor desc = new HTableDescriptor(tableName); + // add a family to the table + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + // add a constraint to check to see if is run + Constraints.add(desc, CheckWasRunConstraint.class); + + // then disable all the constraints + Constraints.disable(desc); + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we do fail on violation + Put put = new Put(row1); + put.add(dummy, new byte[0], "pass".getBytes()); + LOG.warn("Doing put in table"); + table.put(put); + + assertFalse(CheckWasRunConstraint.wasRun); + } + + /** + * Check to make sure a constraint is unloaded when it fails + * @throws Exception + */ + @Test + public void testIsUnloaded() throws Exception { + // create the table + HTableDescriptor desc = new HTableDescriptor(tableName); + // add a family to the table + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + // make sure that constraints are unloaded + Constraints.add(desc, RuntimeFailConstraint.class); + // add a constraint to check to see if is run + Constraints.add(desc, CheckWasRunConstraint.class); + CheckWasRunConstraint.wasRun = false; + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we do fail on violation + Put put = new Put(row1); + put.add(dummy, new byte[0], "pass".getBytes()); + + try{ + table.put(put); + fail("RuntimeFailConstraint wasn't triggered - this put shouldn't work!"); + } catch (Exception e) {// NOOP + } + + // try the put again, this time constraints are not used, so it works + table.put(put); + // and we make sure that constraints were not run... + assertFalse(CheckWasRunConstraint.wasRun); + } + + @After + public void cleanup() throws Exception { + // cleanup + CheckWasRunConstraint.wasRun = false; + util.getHBaseAdmin().disableTable(tableName); + util.getHBaseAdmin().deleteTable(tableName); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + util.shutdownMiniCluster(); + } + + /** + * Constraint to check that it was actually run (or not) + */ + public static class CheckWasRunConstraint extends BaseConstraint { + public static boolean wasRun = false; + + @Override + public void check(Put p) { + wasRun = true; + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/RuntimeFailConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/RuntimeFailConstraint.java new file mode 100644 index 0000000..6164370 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/RuntimeFailConstraint.java @@ -0,0 +1,16 @@ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.client.Put; + +/** + * Always non-gracefully fail on attempt + */ +public class RuntimeFailConstraint extends BaseConstraint { + + @Override + public void check(Put p) throws ConstraintException { + throw new RuntimeException( + "RuntimeFailConstraint always throws a runtime exception"); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraints.java b/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraints.java new file mode 100644 index 0000000..e92b7b5 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraints.java @@ -0,0 +1,182 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.constraint.Constraints.OrderedConstraint; +import org.apache.hadoop.hbase.constraint.IntegrationTestConstraint.CheckWasRunConstraint; +import org.apache.hadoop.hbase.constraint.WorksConstraint.NameConstraint; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Test; + +/** + * Test reading/writing the constraints into the {@link HTableDescriptor} + */ +public class TestConstraints { + + @SuppressWarnings("unchecked") + @Test + public void testSimpleReadWrite() throws Throwable { + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.add(desc, WorksConstraint.class); + + List constraints = (List) Constraints + .getConstraints(desc, this + .getClass().getClassLoader()); + assertEquals(1, constraints.size()); + + assertEquals(WorksConstraint.class, constraints.get(0).delegate.getClass()); + + // Check that we can add more than 1 constraint and that ordering is + // preserved + Constraints.add(desc, AlsoWorks.class, NameConstraint.class); + constraints = (List) Constraints.getConstraints(desc, + this.getClass().getClassLoader()); + assertEquals(3, constraints.size()); + + assertEquals(WorksConstraint.class, constraints.get(0).delegate.getClass()); + assertEquals(AlsoWorks.class, constraints.get(1).delegate.getClass()); + assertEquals(NameConstraint.class, constraints.get(2).delegate.getClass()); + + } + + @SuppressWarnings("unchecked") + @Test + public void testReadWriteWithConf() throws Throwable { + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.add(desc, + new Pair, Configuration>( + CheckConfigurationConstraint.class, CheckConfigurationConstraint + .getConfiguration())); + + List c = (List) Constraints + .getConstraints(desc, this + .getClass().getClassLoader()); + assertEquals(1, c.size()); + + assertEquals(CheckConfigurationConstraint.class, + c.get(0).delegate.getClass()); + + // check to make sure that we overwrite configurations + Constraints.add(desc, + new Pair, Configuration>( + CheckConfigurationConstraint.class, new Configuration())); + + try + {Constraints.getConstraints(desc, this + .getClass().getClassLoader()); + assertTrue("No exception thrown - configuration not overwritten", false); + } catch (IllegalArgumentException e) { + } + } + + /** + * Test that Constraints are properly enabled, disabled, and removed + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test + public void testEnableDisableRemove() throws Exception { + HTableDescriptor desc = new HTableDescriptor("table"); + // check general enabling/disabling of constraints + // first add a constraint + Constraints.add(desc, AllPassConstraint.class); + // make sure everything is enabled + assertTrue(Constraints.enabled(desc, AllPassConstraint.class)); + assertTrue(desc.hasCoprocessor(ConstraintProcessor.class.getName())); + + // check disabling + Constraints.disable(desc); + assertFalse(desc.hasCoprocessor(ConstraintProcessor.class.getName())); + // make sure the added constraints are still present + assertTrue(Constraints.enabled(desc, AllPassConstraint.class)); + + // check just removing the single constraint + Constraints.remove(desc, AllPassConstraint.class); + assertFalse(Constraints.has(desc, AllPassConstraint.class)); + + // Add back the single constraint + Constraints.add(desc, AllPassConstraint.class); + + //and now check that when we remove constraints, all are gone + Constraints.remove(desc); + assertFalse(desc.hasCoprocessor(ConstraintProcessor.class.getName())); + assertFalse(Constraints.has(desc, AllPassConstraint.class)); + + } + + /** + * Test that when we update a constraint the ordering is not modified. + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test + public void testUpdateConstraint() throws Exception { + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.add(desc, CheckConfigurationConstraint.class, + CheckWasRunConstraint.class); + Constraints.setConfiguration(desc, CheckConfigurationConstraint.class, + CheckConfigurationConstraint.getConfiguration()); + + List constraints = (List) Constraints + .getConstraints(desc, this.getClass().getClassLoader()); + + assertEquals(2, constraints.size()); + + // check to make sure the order didn't change + assertEquals(CheckConfigurationConstraint.class, + constraints.get(0).delegate.getClass()); + assertEquals(CheckWasRunConstraint.class, + constraints.get(1).delegate.getClass()); + } + + /** + * Test that if a constraint hasn't been set that there are no problems with + * attempting to remove it. + * @throws Throwable on failure. + */ + @Test + public void testRemoveUnsetConstraint() throws Throwable { + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.remove(desc); + Constraints.remove(desc, AlsoWorks.class); + } + + // ---------- Constraints just used for testing + + /** + * Also just works + */ + public static class AlsoWorks extends BaseConstraint { + @Override + public void check(Put p) { + // NOOP + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/TestIntegerConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/TestIntegerConstraint.java new file mode 100644 index 0000000..37cb165 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/TestIntegerConstraint.java @@ -0,0 +1,56 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import static org.junit.Assert.fail; + +import org.apache.hadoop.hbase.client.Put; +import org.junit.Test; + +/** + * Test that the integer constraint will throw exceptions as expected + */ +public class TestIntegerConstraint { + + /** + * Check that we will verify values as integers, and throw an exception + * otherwise + * @throws Throwable + */ + @Test(expected = ConstraintException.class) + public void testIntegerConstraint() throws Throwable { + Put p = new Put("row".getBytes()); + p.add("family".getBytes(), "qualifier".getBytes(), Integer.toString(10) + .getBytes()); + + IntegerConstraint constraint = new IntegerConstraint(); + constraint.check(p); + + p.add("family".getBytes(), "qualifier".getBytes(), + "not a number".getBytes()); + try { + constraint.check(p); + fail("Should not be able to put a non-numeric value"); + } catch (Throwable e) { + throw e; + } + + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/WorksConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/WorksConstraint.java new file mode 100644 index 0000000..461b1f0 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/WorksConstraint.java @@ -0,0 +1,40 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.client.Put; + +/** + * It just works + */ +public class WorksConstraint extends BaseConstraint { + @Override + public void check(Put p) { + // NOOP + } + + /** + * Constraint to check that the naming of constraints doesn't mess up the + * pattern matching.(that constraint $___Constraint$NameConstraint isn't a + * problem) + */ + public static class NameConstraint extends WorksConstraint { + } +}