diff --git src/docbkx/book.xml src/docbkx/book.xml index 9617950..73dd471 100644 --- src/docbkx/book.xml +++ src/docbkx/book.xml @@ -868,6 +868,11 @@ System.out.println("md5 digest as string length: " + sbDigest.length); // ret +
Constraints + HBase currently supports 'constraints' in traditional (SQL) database parlance. The advised usage for Constraints is in enforcing business rules for attributes in the table (eg. make sure values are in the range 1-10). + Constraints could also be used to enforce referential integrity, but this is strongly discouraged as it will dramatically decrease the write throughput of the tables where integrity checking enabled. +
+ diff --git src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index 84a0d1a..133759d 100644 --- src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -477,6 +477,16 @@ public class HTableDescriptor implements WritableComparable { public void remove(final byte [] key) { values.remove(new ImmutableBytesWritable(key)); } + + /** + * Remove metadata represented by the key from the {@link #values} map + * + * @param key Key whose key and value we're to remove from HTableDescriptor + * parameters. + */ + public void remove(final String key) { + remove(Bytes.toBytes(key)); + } /** * Check if the readOnly flag of the table is set. If the readOnly flag is @@ -788,6 +798,7 @@ public class HTableDescriptor implements WritableComparable { * INTERNAL This method is a part of {@link WritableComparable} interface * and is used for de-serialization of the HTableDescriptor over RPC */ + @Override public void readFields(DataInput in) throws IOException { int version = in.readInt(); if (version < 3) @@ -822,6 +833,7 @@ public class HTableDescriptor implements WritableComparable { * INTERNAL This method is a part of {@link WritableComparable} interface * and is used for serialization of the HTableDescriptor over RPC */ + @Override public void write(DataOutput out) throws IOException { out.writeInt(TABLE_DESCRIPTOR_VERSION); Bytes.writeByteArray(out, name); @@ -850,6 +862,7 @@ public class HTableDescriptor implements WritableComparable { * @return 0 if the contents of the descriptors are exactly matching, * 1 if there is a mismatch in the contents */ + @Override public int compareTo(final HTableDescriptor other) { int result = Bytes.compareTo(this.name, other.name); if (result == 0) { @@ -1052,6 +1065,38 @@ public class HTableDescriptor implements WritableComparable { return false; } + /** + * Remove a coprocessor from those set on the table + * @param className Class name of the co-processor + */ + public void removeCoprocessor(String className) { + ImmutableBytesWritable match = null; + Matcher keyMatcher; + Matcher valueMatcher; + for (Map.Entry e : this.values + .entrySet()) { + keyMatcher = HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher(Bytes.toString(e + .getKey().get())); + if (!keyMatcher.matches()) { + continue; + } + valueMatcher = HConstants.CP_HTD_ATTR_VALUE_PATTERN.matcher(Bytes + .toString(e.getValue().get())); + if (!valueMatcher.matches()) { + continue; + } + // get className and compare + String clazz = valueMatcher.group(2).trim(); // classname is the 2nd field + // remove the CP if it is present + if (clazz.equals(className.trim())) { + match = e.getKey(); + break; + } + } + // if we found a match, remove it + if (match != null) + this.values.remove(match); + } /** * Returns the {@link Path} object representing the table directory under diff --git src/main/java/org/apache/hadoop/hbase/constraint/BaseConstraint.java src/main/java/org/apache/hadoop/hbase/constraint/BaseConstraint.java new file mode 100644 index 0000000..7ce6d45 --- /dev/null +++ src/main/java/org/apache/hadoop/hbase/constraint/BaseConstraint.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.conf.Configuration; + +/** + * Base class to use when actually implementing a {@link Constraint}. It takes + * care of getting and setting of configuration for the constraint. + */ +public abstract class BaseConstraint implements + Constraint { + + private Configuration conf; + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return this.conf; + } + +} diff --git src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java new file mode 100644 index 0000000..2d8b4d7 --- /dev/null +++ src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Put; + +/** + * Apply a {@link Constraint} (in traditional database terminology) to a HTable. + * Any number of {@link Constraint Constraints} can be added to the table, in + * any order. + *

+ * A {@link Constraint} must be added to a table before the table is loaded via + * {@link Constraints#add(HTableDescriptor, Class...)} or + * {@link Constraints#add(HTableDescriptor, org.apache.hadoop.hbase.util.Pair...)} + * (if you want to add a configuration with the {@link Constraint}). Constraints + * will be run in the order that they are added. Further, a Constraint will be + * configured before it is run (on load). + *

+ * See {@link Constraints#enableConstraint(HTableDescriptor, Class)} and + * {@link Constraints#disableConstraint(HTableDescriptor, Class)} for + * enabling/disabling of a given {@link Constraint} after it has been added. + *

+ * If a {@link Put} is not valid, the Constraint should throw some sort + * {@link ConstraintException} indicating that the {@link Put} has failed. When + * this exception is thrown, not further retries of the {@link Put} are + * attempted nor are any other {@link Constraint Constraints} attempted (the + * {@link Put} is clearly not valid). Therefore, there are performance + * implications in the order in which {@link BaseConstraint Constraints} are + * specified. + *

+ * If a {@link Constraint} fails to fail the {@link Put} via a + * {@link ConstraintException}, but instead throws a {@link RuntimeException}, + * the entire constraint processing mechanism ({@link ConstraintProcessor}) will + * be unloaded from the table. This ensures that the region server is still + * functional, but not more {@link Put Puts} will be checked via + * {@link Constraint Constraints}. + *

+ * Further, {@link Constraint Constraints} probably not be used to enforce + * cross-table references as it will cause tremendous write slowdowns, but it is + * possible. + *

+ * NOTE: Implementing classes must have a nullary (no-args) constructor + */ +public interface Constraint extends Configurable { + + /** + * Check a {@link Put} to ensure it is valid for the table. If the {@link Put} + * is valid, then just return from the method. Otherwise, throw an + * {@link Exception} specifying what happened. This {@link Exception} is + * propagated back to the client so you can see what caused the {@link Put} to + * fail. + * @param p {@link Put} to check + * @throws ConstraintException when the {@link Put} does not match the + * constraint. + */ + public void check(Put p) throws ConstraintException; + +} diff --git src/main/java/org/apache/hadoop/hbase/constraint/ConstraintException.java src/main/java/org/apache/hadoop/hbase/constraint/ConstraintException.java new file mode 100644 index 0000000..1831421 --- /dev/null +++ src/main/java/org/apache/hadoop/hbase/constraint/ConstraintException.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.client.Put; + +/** + * Exception that a user defined constraint throws on failure of a {@link Put}. + *

+ * Does NOT attempt the {@link Put} multiple times, since the constraint + * should fail every time for the same {@link Put} (it should be + * idempotent). + */ +public class ConstraintException extends DoNotRetryIOException { + private static final long serialVersionUID = 1197446454511704140L; + + public ConstraintException() { + super(); + } + + public ConstraintException(String msg) + { + super(msg); + } + + + public ConstraintException(String msg, Throwable cause) { + super(msg, cause); + } +} diff --git src/main/java/org/apache/hadoop/hbase/constraint/ConstraintProcessor.java src/main/java/org/apache/hadoop/hbase/constraint/ConstraintProcessor.java new file mode 100644 index 0000000..522369c --- /dev/null +++ src/main/java/org/apache/hadoop/hbase/constraint/ConstraintProcessor.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; + +/*** + * Processes multiple {@link Constraint Constraints} on a given table. + *

+ * This is an ease of use mechanism - all the functionality here could be + * implemented on any given system by a coprocessor. + */ +public class ConstraintProcessor extends BaseRegionObserver { + + private static final Log LOG = LogFactory.getLog(ConstraintProcessor.class); + + private final ClassLoader classloader; + + private List constraints = new ArrayList(); + + /** + * Create the constraint processor. + *

+ * Stores the current classloader. + */ + public ConstraintProcessor() { + classloader = this.getClass().getClassLoader(); + } + + @Override + public void start(CoprocessorEnvironment environment) { + // make sure we are on a region server + if (!(environment instanceof RegionCoprocessorEnvironment)) { + throw new IllegalArgumentException( + "Constraints only act on regions - started in an environment that was not a region"); + } + RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) environment; + HTableDescriptor desc = env.getRegion().getTableDesc(); + // load all the constraints from the HTD + try { + this.constraints = Constraints.getConstraints(desc, classloader); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + + if (LOG.isInfoEnabled()) { + LOG.info("Finished loading " + constraints.size() + + " user Constraints on table: " + new String(desc.getName())); + } + + } + + @Override + public void prePut(ObserverContext e, Put put, + WALEdit edit, boolean writeToWAL) throws IOException { + // check the put against the stored constraints + for (Constraint c : constraints) { + c.check(put); + } + // if we made it here, then the Put is valid + } +} diff --git src/main/java/org/apache/hadoop/hbase/constraint/Constraints.java src/main/java/org/apache/hadoop/hbase/constraint/Constraints.java new file mode 100644 index 0000000..eb3b5b9 --- /dev/null +++ src/main/java/org/apache/hadoop/hbase/constraint/Constraints.java @@ -0,0 +1,597 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +/** + * Utilities for adding/removing constraints from a table. + *

+ * Constraints can be added on table load time, via the {@link HTableDescriptor}. + *

+ * NOTE: this class is NOT thread safe. Concurrent setting/enabling/disabling of + * constraints can cause constraints to be run at incorrect times or not at all. + */ +public final class Constraints { + private static final int DEFAULT_PRIORITY = -1; + + private Constraints() { + } + + private static final Log LOG = LogFactory.getLog(Constraints.class); + private static final String CONSTRAINT_HTD_KEY_PREFIX = "constraint $"; + private static final Pattern CONSTRAINT_HTD_ATTR_KEY_PATTERN = Pattern + .compile(CONSTRAINT_HTD_KEY_PREFIX, Pattern.LITERAL); + + // configuration key for if the constraint is enabled + private static final String ENABLED_KEY = "_ENABLED"; + // configuration key for the priority + private static final String PRIORITY_KEY = "_PRIORITY"; + + // smallest priority a constraiNt can have + private static final long MIN_PRIORITY = 0L; + // ensure a priority less than the smallest we could intentionally set + private static final long UNSET_PRIORITY = MIN_PRIORITY - 1; + + private static String COUNTER_KEY = "hbase.constraint.counter"; + + /** + * Enable constraints on a table. + *

+ * Currently, if you attempt to add a constraint to the table, then + * Constraints will automatically be turned on. + * + * @param desc + * table description to add the processor + * @throws IOException + * If the {@link ConstraintProcessor} CP couldn't be added to the + * table. + */ + public static void enable(HTableDescriptor desc) throws IOException { + // if the CP has already been loaded, do nothing + String clazz = ConstraintProcessor.class.getName(); + if (desc.hasCoprocessor(clazz)) + return; + + // add the constrain processor CP to the table + desc.addCoprocessor(clazz); + } + + /** + * Turn off processing constraints for a given table, even if constraints have + * been turned on or added. + * + * @param desc + * {@link HTableDescriptor} where to disable {@link Constraint + * Constraints}. + */ + public static void disable(HTableDescriptor desc) { + desc.removeCoprocessor(ConstraintProcessor.class.getName()); + } + + /** + * Remove all {@link Constraint Constraints} that have been added to the table + * and turn off the constraint processing. + *

+ * All {@link Configuration Configurations} and their associated + * {@link Constraint} are removed. + * + * @param desc + * {@link HTableDescriptor} to remove {@link Constraint Constraints} + * from. + */ + public static void remove(HTableDescriptor desc) { + // disable constraints + disable(desc); + + // remove all the constraint settings + List keys = new ArrayList(); + // loop through all the key, values looking for constraints + for (Map.Entry e : desc + .getValues().entrySet()) { + String key = Bytes.toString((e.getKey().get())); + String[] className = CONSTRAINT_HTD_ATTR_KEY_PATTERN.split(key); + if (className.length == 2) { + keys.add(e.getKey()); + } + } + // now remove all the keys we found + for (ImmutableBytesWritable key : keys) { + desc.remove(key.get()); + } + } + + /** + * Check to see if the Constraint is currently set. + * + * @param desc + * {@link HTableDescriptor} to check + * @param clazz + * {@link Constraint} class to check for. + * @return true if the {@link Constraint} is present, even if it is + * disabled. false otherwise. + */ + public static boolean has(HTableDescriptor desc, + Class clazz) { + return getKeyValueForClass(desc, clazz) != null; + } + + /** + * Get the kv {@link Entry} in the descriptor for the specified class + * + * @param desc + * {@link HTableDescriptor} to read + * @param clazz + * to search for + * @return the {@link Pair} of in the table, if that class is + * present. null otherwise. + */ + private static Pair getKeyValueForClass( + HTableDescriptor desc, Class clazz) { + // get the serialized version of the constraint + String key = serializeConstraintClass(clazz); + String value = desc.getValue(key); + if (value == null) + return null; + return new Pair(key, value); + } + + /** + * Add configuration-less constraints to the table. + *

+ * This will overwrite any configuration associated with the previous + * constraint of the same class. + * + * @param desc + * {@link HTableDescriptor} to add a {@link Constraint} + * @param constraints + * {@link Constraint Constraints} to add. All constraints are + * considered automatically enabled on add + * @throws IOException + * If constraint could not be serialized/added to table + */ + public static void add(HTableDescriptor desc, + Class... constraints) throws IOException { + // make sure constraints are enabled + enable(desc); + long priority = getNextPriority(desc); + + // store each constraint + for (Class clazz : constraints) { + addConstraint(desc, clazz, null, priority++); + } + updateLatestPriority(desc, priority); + } + + /** + * Add constraints and their associated configurations to the table. + *

+ * Adding the same constraint class twice will overwrite the first + * constraint's configuration + * + * @param desc + * {@link HTableDescriptor} to add a {@link Constraint} + * @param constraints + * {@link Pair} of a {@link Constraint} and its associated + * {@link Configuration}. The Constraint will be configured on load + * with the specified configuration.All constraints are considered + * automatically enabled on add + * @throws IOException + * if any constraint could not be deserialized. Assumes if 1 + * constraint is not loaded properly, something has gone terribly + * wrong and that all constraints need to be enforced. + */ + public static void add(HTableDescriptor desc, + Pair, Configuration>... constraints) + throws IOException { + enable(desc); + long priority = getNextPriority(desc); + for (Pair, Configuration> pair : constraints) { + addConstraint(desc, pair.getFirst(), pair.getSecond(), priority++); + } + updateLatestPriority(desc, priority); + } + + /** + * Add a {@link Constraint} to the table with the given configuration + * + * @param desc + * table descriptor to the constraint to + * @param constraint + * to be added + * @param conf + * configuration associated with the constraint + * @throws IOException + * if any constraint could not be deserialized. Assumes if 1 + * constraint is not loaded properly, something has gone terribly + * wrong and that all constraints need to be enforced. + */ + public static void add(HTableDescriptor desc, + Class constraint, Configuration conf) + throws IOException { + enable(desc); + long priority = getNextPriority(desc); + addConstraint(desc, constraint, conf, priority++); + + updateLatestPriority(desc, priority); + } + + /** + * Write the raw constraint and configuration to the descriptor. + *

+ * This method takes care of creating a new configuration based on the passed + * in configuration and then updating that with enabled and priority of the + * constraint. + *

+ * When a constraint is added, it is automatically enabled. + */ + private static void addConstraint(HTableDescriptor desc, + Class clazz, Configuration conf, long priority) + throws IOException { + writeConstraint(desc, serializeConstraintClass(clazz), + configure(conf, true, priority)); + } + + /** + * Setup the configuration for a constraint as to whether it is enabled and + * its priority + * + * @param conf + * on which to base the new configuration + * @param enabled + * true if it should be run + * @param priority + * relative to other constraints + * @returns a new configuration, storable in the {@link HTableDescriptor} + */ + private static Configuration configure(Configuration conf, boolean enabled, + long priority) { + // create the configuration to actually be stored + // clone if possible, but otherwise just create an empty configuration + Configuration toWrite = conf == null ? new Configuration() + : new Configuration(conf); + + // update internal properties + toWrite.setBooleanIfUnset(ENABLED_KEY, enabled); + + // set if unset long + if (toWrite.getLong(PRIORITY_KEY, UNSET_PRIORITY) == UNSET_PRIORITY) { + toWrite.setLong(PRIORITY_KEY, priority); + } + + return toWrite; + } + + /** + * Just write the class to the byte [] we are expecting + * + * @param clazz + * @return key to store in the {@link HTableDescriptor} + */ + private static String serializeConstraintClass( + Class clazz) { + String constraintClazz = clazz.getName(); + return CONSTRAINT_HTD_KEY_PREFIX + constraintClazz; + } + + /** + * Write the given key and associated configuration to the + * {@link HTableDescriptor} + */ + private static void writeConstraint(HTableDescriptor desc, String key, + Configuration conf) throws IOException { + // store the key and conf in the descriptor + desc.setValue(key, serializeConfiguration(conf)); + } + + /** + * Write the configuration to a String + * + * @param conf + * to write + * @return String representation of that configuration + * @throws IOException + */ + private static String serializeConfiguration(Configuration conf) + throws IOException { + // write the configuration out to the data stream + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(bos); + conf.writeXml(dos); + dos.flush(); + byte[] data = bos.toByteArray(); + return Bytes.toString(data); + } + + /** + * Read the {@link Configuration} stored in the byte stream. + * + * @param bytes + * to read from + * @return A valid configuration + */ + private static Configuration readConfiguration(byte[] bytes) + throws IOException { + ByteArrayInputStream is = new ByteArrayInputStream(bytes); + Configuration conf = new Configuration(); + conf.addResource(is); + return conf; + } + + /** + * Read in the configuration from the String encoded-configration + * + * @param bytes + * to read from + * @return A valid configuration + * @throws IOException + * if the configuration could not be read + */ + private static Configuration readConfiguration(String bytes) + throws IOException { + return readConfiguration(Bytes.toBytes(bytes)); + } + + private static long getNextPriority(HTableDescriptor desc) { + String value = desc.getValue(COUNTER_KEY); + + long priority; + // get the current priority + if (value == null) { + priority = MIN_PRIORITY; + } else { + priority = Long.parseLong(value) + 1; + } + + return priority; + } + + private static void updateLatestPriority(HTableDescriptor desc, long priority) { + // update the max priority + desc.setValue(COUNTER_KEY, Long.toString(priority)); + } + + /** + * Update the configuration for the {@link Constraint}. Does not change the + * order in which the constraint is run. If the + * + * @param desc + * {@link HTableDescriptor} to update + * @param clazz + * {@link Constraint} to update + * @param configuration + * to update the {@link Constraint} with. + * @throws IOException + * if the Constraint was not stored correctly + * @throws IllegalArgumentException + * if the Constraint was not present on this table. + */ + public static void setConfiguration(HTableDescriptor desc, + Class clazz, Configuration configuration) + throws IOException, IllegalArgumentException { + // get the entry for this class + Pair e = getKeyValueForClass(desc, clazz); + + if (e == null) + throw new IllegalArgumentException("Constraint: " + clazz.getName() + + " is not associated with this table."); + + // clone over the configuration elements + Configuration conf = new Configuration(configuration); + + // read in the previous info about the constraint + Configuration internal = readConfiguration(e.getSecond()); + + // update the fields based on the previous settings + conf.setIfUnset(ENABLED_KEY, internal.get(ENABLED_KEY)); + conf.setIfUnset(PRIORITY_KEY, internal.get(PRIORITY_KEY)); + + // update the current value + writeConstraint(desc, e.getFirst(), conf); + } + + /** + * Remove the constraint (and associated information) for the table + * descriptor. + * + * @param desc + * {@link HTableDescriptor} to modify + * @param clazz + * {@link Constraint} class to remove + */ + public static void remove(HTableDescriptor desc, + Class clazz) { + String key = serializeConstraintClass(clazz); + desc.remove(key); + } + + /** + * Enable the given {@link Constraint}. Retains all the information (e.g. + * Configuration) for the {@link Constraint}, but makes sure that it gets + * loaded on the table. + * + * @param desc + * {@link HTableDescriptor} to modify + * @param clazz + * {@link Constraint} to enable + * @throws IOException + * If the constraint cannot be properly deserialized + */ + public static void enableConstraint(HTableDescriptor desc, + Class clazz) throws IOException { + changeConstraintEnabled(desc, clazz, true); + } + + /** + * Disable the given {@link Constraint}. Retains all the information (e.g. + * Configuration) for the {@link Constraint}, but it just doesn't load the + * {@link Constraint} on the table. + * + * @param desc + * {@link HTableDescriptor} to modify + * @param clazz + * {@link Constraint} to disable. + * @throws IOException + * if the constraint cannot be found + */ + public static void disableConstraint(HTableDescriptor desc, + Class clazz) throws IOException { + changeConstraintEnabled(desc, clazz, false); + } + + /** + * Change the whether the constraint (if it is already present) is enabled or + * disabled. + */ + private static void changeConstraintEnabled(HTableDescriptor desc, + Class clazz, boolean enabled) throws IOException { + // get the original constraint + Pair entry = getKeyValueForClass(desc, clazz); + if (entry == null) + throw new IllegalArgumentException("Constraint: " + clazz.getName() + + " is not associated with this table. You can't enable it!"); + + // create a new configuration from that conf + Configuration conf = readConfiguration(entry.getSecond()); + + // set that it is enabled + conf.setBoolean(ENABLED_KEY, enabled); + + // write it back out + writeConstraint(desc, entry.getFirst(), conf); + } + + /** + * Check to see if the given constraint is enabled. + * + * @param desc + * {@link HTableDescriptor} to check. + * @param clazz + * {@link Constraint} to check for + * @return true if the {@link Constraint} is present and enabled. + * false otherwise. + * @throws IOException + * If the constraint has improperly stored in the table + */ + public static boolean enabled(HTableDescriptor desc, + Class clazz) throws IOException { + // get the kv + Pair entry = getKeyValueForClass(desc, clazz); + // its not enabled so just return false. In fact, its not even present! + if (entry == null) + return false; + + // get the info about the constraint + Configuration conf = readConfiguration(entry.getSecond()); + + return conf.getBoolean(ENABLED_KEY, false); + } + + /** + * Get the constraints stored in the table descriptor + * + * @param desc + * To read from + * @param classloader + * To use when loading classes + * @return List of configured {@link Constraint Constraints} + * @throws IOException + * if any part of reading/arguments fails + */ + static List getConstraints(HTableDescriptor desc, + ClassLoader classloader) throws IOException { + List constraints = new ArrayList(); + // loop through all the key, values looking for constraints + for (Map.Entry e : desc + .getValues().entrySet()) { + // read out the constraint + String key = Bytes.toString(e.getKey().get()).trim(); + String[] className = CONSTRAINT_HTD_ATTR_KEY_PATTERN.split(key); + if (className.length == 2) { + key = className[1]; + if (LOG.isDebugEnabled()) { + LOG.debug("Loading constraint:" + key); + } + + // read in the rest of the constraint + Configuration conf; + try { + conf = readConfiguration(e.getValue().get()); + } catch (IOException e1) { + // long that we don't have a valid configuration stored, and move on. + LOG.warn("Corrupted configuration found for key:" + key + + ", skipping it."); + continue; + } + // if it is not enabled, skip it + if (!conf.getBoolean(ENABLED_KEY, false)) { + if (LOG.isDebugEnabled()) + LOG.debug("Constraint: " + key + " is DISABLED - skipping it"); + // go to the next constraint + continue; + } + + try { + // add the constraint, now that we expect it to be valid. + Class clazz = classloader.loadClass(key) + .asSubclass(Constraint.class); + Constraint constraint = clazz.newInstance(); + constraint.setConf(conf); + constraints.add(constraint); + } catch (ClassNotFoundException e1) { + throw new IOException(e1); + } catch (InstantiationException e1) { + throw new IOException(e1); + } catch (IllegalAccessException e1) { + throw new IOException(e1); + } + } + } + // sort them, based on the priorities + Collections.sort(constraints, constraintComparator); + return constraints; + } + + private static final Comparator constraintComparator = new Comparator() { + @Override + public int compare(Constraint c1, Constraint c2) { + // compare the priorities of the constraints stored in their configuration + return Long.valueOf(c1.getConf().getLong(PRIORITY_KEY, DEFAULT_PRIORITY)) + .compareTo(c2.getConf().getLong(PRIORITY_KEY, DEFAULT_PRIORITY)); + } + }; + +} diff --git src/main/java/org/apache/hadoop/hbase/constraint/package-info.java src/main/java/org/apache/hadoop/hbase/constraint/package-info.java new file mode 100644 index 0000000..0092bde --- /dev/null +++ src/main/java/org/apache/hadoop/hbase/constraint/package-info.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Restrict the domain of a data attribute, often times to fulfill business rules/requirements. + * +

+

Table of Contents

+ +

+ +

Overview

+ Constraints are used to enforce business rules in a database. + By checking all {@link org.apache.hadoop.hbase.client.Put Puts} on a given table, you can enforce very specific data policies. + For instance, you can ensure that a certain column family-column qualifier pair always has a value between 1 and 10. + Otherwise, the {@link org.apache.hadoop.hbase.client.Put} is rejected and the data integrity is maintained. +

+ Constraints are designed to be configurable, so a constraints can be used across different tables, but implement different behavior depending on the specific configuration given to that constraint. +

+ By adding a constraint to a table (see Example Usage), constraints will automatically enabled. + You also then have the option of disabling (just 'turn off') or remove (delete all associated information) constraints on a table. + If you remove all constraints (see {@link org.apache.hadoop.hbase.constraint.Constraints#remove(org.apache.hadoop.hbase.HTableDescriptor)}, you must re-add any {@link org.apache.hadoop.hbase.constraint.Constraint} you want on that table. + However, if they are just disabled (see {@link org.apache.hadoop.hbase.constraint.Constraints#disable(org.apache.hadoop.hbase.HTableDescriptor)}, all you need to do is enable constraints again, and everything will be turned back on. + Individual constraints can also be individually enabled, disabled or removed without affecting others. +

+ By default, constraints are disabled on a table. + This means you will not see any slow down on a table if constraints are not enabled. +

+ NOTES +

    +
  1. Constraints are run in the order that they are added to a table. This has implications for what order constraints should be added to a table.
  2. +
  3. There are certain keys that are reserved for the Configuration namespace: +
      +
    • _ENABLED - is used server-side to determine if a constraint should be run
    • +
    • _PRIORITY - used server-side to determine what order a constraint should be run
    • +
    + If these items are set, they will be respected in the constraint configuration, but they are taken care of by default in when adding constraints to an {@link org.apache.hadoop.hbase.HTableDescriptor} via the usual method. +

    + Under the hood, constraints are implemented as a Coprocessor (see {@link org.apache.hadoop.hbase.constraint.ConstraintProcessor} if you are interested). + +

    Example usage

    + First, you must define a {@link org.apache.hadoop.hbase.constraint.Constraint}. + The best way to do this is to extend {@link org.apache.hadoop.hbase.constraint.BaseConstraint}, which takes care of some of the more mundane details of using a {@link org.apache.hadoop.hbase.constraint.Constraint}. +

    + Let's look at one possible implementation of a constraint - an IntegerConstraint(there are also several simple examples in the tests). + The IntegerConstraint} hecks to make sure that the value is a String-encoded int. + It is really simple to implement this kind of constraint, the only method needs to be implemented is {@link org.apache.hadoop.hbase.constraint.Constraint#check(org.apache.hadoop.hbase.client.Put)}: + +

    +
    + public class IntegerConstraint extends BaseConstraint {
    +   public void check(Put p) throws ConstraintException {
    +  
    +     Map<byte[], List<KeyValue>> familyMap = p.getFamilyMap();
    +    
    +     for (List <KeyValue> kvs : familyMap.values()) {
    +       for (KeyValue kv : kvs) {
    +      
    +       // just make sure that we can actually pull out an int
    +       // this will automatically throw a NumberFormatException if we try to
    +       // store something that isn't an Integer.
    +      
    +       try {
    +       Integer.parseInt(new String(kv.getValue()));
    +       } catch (NumberFormatException e) {
    +       throw new ConstraintException("Value in Put (" + p
    +       + ") was not a String-encoded integer", e);
    +   } } } 
    + 
    +
    +

    + Note that all exceptions that you expect to be thrown must be caught and then rethrown as a {@link org.apache.hadoop.hbase.constraint.ConstraintException}. + This way, you can be sue that a {@link org.apache.hadoop.hbase.client.Put} fails for an expected reason, rather than for any reason. + For example, an {@link java.lang.OutOfMemoryError} is probably indicative of an inherent problem in the {@link org.apache.hadoop.hbase.constraint.Constraint}, rather than a failed {@link org.apache.hadoop.hbase.client.Put}. +

    + If an unexpected exception is thrown (for example, any kind of uncaught {@link java.lang.RuntimeException}), constraint-checking will be 'unloaded' from the regionserver where that error occurred. + This means no further {@link org.apache.hadoop.hbase.constraint.Constraint Constraints} will be checked on that server until it is reloaded. This is done to ensure the system remains as available as possible. + Therefore, be careful when writing your own Constraint. +

    + So now that we have a Constraint, we want to add it to a table. It's as easy as: + +

    +
    + HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
    + ...
    + Constraints.add(desc, IntegerConstraint.class);
    + 
    +

    + Once we added the IntegerConstraint, constraints will be enabled on the table (once it is created) and we will always check to make sure that the value is an String-encoded integer. +

    + However, suppose we also write our own constraint, MyConstraint.java. + First, you need to make sure this class-files are in the classpath (in a jar) on the regionserver where that constraint will be run. +

    + Suppose that MyConstraint also uses a Configuration (see {@link org.apache.hadoop.hbase.constraint.Constraint#getConf()}). + Then adding MyConstraint looks like this: + +

    +
    + HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
    + Configuration conf = new Configuration();
    + ...
    + (add values to the conf)
    + (modify the table descriptor)
    + ...
    + Constraints.add(desc, new Pair(MyConstraint.class, conf));
    + 
    +

    + At this point we added both the {@link org.apache.hadoop.hbase.constraint.IntegerConstraint} and MyConstraint to the table, the {@link org.apache.hadoop.hbase.constraint.IntegerConstraint} will be run first, followed by MyConstraint. +

    + Suppose we realize that the {@link org.apache.hadoop.conf.Configuration} for MyConstraint is actually wrong when it was added to the table. Note, when it is added to the table, it is not added by reference, but is instead copied into the {@link org.apache.hadoop.hbase.HTableDescriptor}. + Thus, to change the {@link org.apache.hadoop.conf.Configuration} we are using for MyConstraint, we need to do this: + +

    +
    + (add/modify the conf)
    + ...
    + Constraints.setConfiguration(desc, MyConstraint.class, conf);
    + 
    +

    + This will overwrite the previous configuration for MyConstraint, but not change the order of the constraint nor if it is enabled/disabled. +

    + Note that the same constraint class can be added multiple times to a table without repercussion. + A use case for this is the same constraint working differently based on its configuration. + +

    + Suppose then we want to disable just MyConstraint. Its as easy as: +

    +
    + Constraints.disable(desc, MyConstraint.class);
    + 
    +

    + This just turns off MyConstraint, but retains the position and the configuration associated with MyConstraint. + Now, if we want to re-enable the constraint, its just another one-liner: +

    +
    + Constraints.enable(desc, MyConstraint.class);
    + 
    +

    + Similarly, constraints on the entire table are disabled via: +

    +
    + Constraints.disable(desc);
    + 
    +

    + Or enabled via: + +

    +
    + Constraints.enable(desc);
    + 
    +

    + Lastly, suppose you want to remove MyConstraint from the table, including with position it should be run at and its configuration. + This is similarly simple: +

    +
    + Constraints.remove(desc, MyConstraint.class);
    + 
    +

    + Also, removing all constraints from a table is similarly simple: +

    +
    + Constraints.remove(desc);
    + 
    + This will remove all constraints (and associated information) from the table and turn off the constraint processing. + +

    Caveats

    + In traditional (SQL) databases, Constraints are often used to enforce referential integrity. + However, in HBase, this will likely cause significant overhead and dramatically decrease the number of {@link org.apache.hadoop.hbase.client.Put Puts}/second possible on a table. + This is because to check the referential integrity when making a {@link org.apache.hadoop.hbase.client.Put}, one must block on a scan for the 'remote' table, checking for the valid reference. + For millions of {@link org.apache.hadoop.hbase.client.Put Puts} a second, this will breakdown very quickly. + There are several options around the blocking behavior including, but not limited to: +
      +
    • Create a 'pre-join' table where the keys are already denormalized
    • +
    • Designing for 'incorrect' references
    • +
    • Using an external enforcement mechanism
    • +
    + */ +package org.apache.hadoop.hbase.constraint; \ No newline at end of file diff --git src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java new file mode 100644 index 0000000..98106ce --- /dev/null +++ src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java @@ -0,0 +1,47 @@ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.junit.Test; + +/** + * Test setting values in the descriptor + */ +public class TestHTableDescriptor { + + /** + * Test cps in the table description + * @throws Exception + */ + @Test + public void testGetSetRemoveCP() throws Exception { + HTableDescriptor desc = new HTableDescriptor("table"); + // simple CP + String className = BaseRegionObserver.class.getName(); + // add and check that it is present + desc.addCoprocessor(className); + assertTrue(desc.hasCoprocessor(className)); + // remove it and check that it is gone + desc.removeCoprocessor(className); + assertFalse(desc.hasCoprocessor(className)); + } + + /** + * Test that we add and remove strings from settings properly. + * @throws Exception + */ + @Test + public void testRemoveString() throws Exception { + HTableDescriptor desc = new HTableDescriptor("table"); + String key = "Some"; + String value = "value"; + desc.setValue(key, value); + assertEquals(value, desc.getValue(key)); + desc.remove(key); + assertEquals(null, desc.getValue(key)); + } + +} diff --git src/test/java/org/apache/hadoop/hbase/constraint/AllFailConstraint.java src/test/java/org/apache/hadoop/hbase/constraint/AllFailConstraint.java new file mode 100644 index 0000000..03fec35 --- /dev/null +++ src/test/java/org/apache/hadoop/hbase/constraint/AllFailConstraint.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.client.Put; + +/** + * Always fail the put. + */ +public class AllFailConstraint extends BaseConstraint { + + @Override + public void check(Put p) throws ConstraintException { + throw new ConstraintException("AllFailConstraint fails for all puts"); + } +} diff --git src/test/java/org/apache/hadoop/hbase/constraint/AllPassConstraint.java src/test/java/org/apache/hadoop/hbase/constraint/AllPassConstraint.java new file mode 100644 index 0000000..a33dfdb --- /dev/null +++ src/test/java/org/apache/hadoop/hbase/constraint/AllPassConstraint.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.client.Put; + +/** + * Simple test constraint that always allows the put to pass + */ +public class AllPassConstraint extends BaseConstraint { + + @Override + public void check(Put p) { + // Do nothing - it passes + } + +} diff --git src/test/java/org/apache/hadoop/hbase/constraint/CheckConfigurationConstraint.java src/test/java/org/apache/hadoop/hbase/constraint/CheckConfigurationConstraint.java new file mode 100644 index 0000000..c49098d --- /dev/null +++ src/test/java/org/apache/hadoop/hbase/constraint/CheckConfigurationConstraint.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Put; + +/** + * Test Constraint to check to make sure the configuration is set + */ +public class CheckConfigurationConstraint extends BaseConstraint { + + + private static String key = "testKey"; + private static String value = "testValue"; + + public static Configuration getConfiguration() { + Configuration conf = new Configuration(); + conf.set(key, value); + return conf; + } + + @Override + public void check(Put p) { + // NOOP + } + + @Override + public void setConf(Configuration conf) { + String val = conf.get(key); + if (val == null || !val.equals(value)) + throw new IllegalArgumentException( + "Configuration was not passed correctly"); + super.setConf(conf); + } + +} diff --git src/test/java/org/apache/hadoop/hbase/constraint/RuntimeFailConstraint.java src/test/java/org/apache/hadoop/hbase/constraint/RuntimeFailConstraint.java new file mode 100644 index 0000000..6164370 --- /dev/null +++ src/test/java/org/apache/hadoop/hbase/constraint/RuntimeFailConstraint.java @@ -0,0 +1,16 @@ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.client.Put; + +/** + * Always non-gracefully fail on attempt + */ +public class RuntimeFailConstraint extends BaseConstraint { + + @Override + public void check(Put p) throws ConstraintException { + throw new RuntimeException( + "RuntimeFailConstraint always throws a runtime exception"); + } + +} diff --git src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java new file mode 100644 index 0000000..75d4327 --- /dev/null +++ src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java @@ -0,0 +1,260 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Do the complex testing of constraints against a minicluster + */ +@Category(MediumTests.class) +public class TestConstraint { + private static final Log LOG = LogFactory + .getLog(TestConstraint.class); + + private static HBaseTestingUtility util; + private static final byte[] tableName = Bytes.toBytes("test"); + private static final byte[] dummy = Bytes.toBytes("dummy"); + private static final byte[] row1 = Bytes.toBytes("r1"); + private static final byte[] test = Bytes.toBytes("test"); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + util = new HBaseTestingUtility(); + util.startMiniCluster(); + } + + /** + * Test that we run a passing constraint + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test + public void testConstraintPasses() throws Exception { + // create the table + // it would be nice if this was also a method on the util + HTableDescriptor desc = new HTableDescriptor(tableName); + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + // add a constraint + Constraints.add(desc, CheckWasRunConstraint.class); + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we don't fail on a valid put + Put put = new Put(row1); + byte[] value = Integer.toString(10).getBytes(); + put.add(dummy, new byte[0], value); + table.put(put); + + assertTrue(CheckWasRunConstraint.wasRun); + } + + /** + * Test that constraints will fail properly + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test(timeout = 10000) + public void testConstraintFails() throws Exception { + + // create the table + // it would be nice if this was also a method on the util + HTableDescriptor desc = new HTableDescriptor(tableName); + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + + // add a constraint that is sure to fail + Constraints.add(desc, AllFailConstraint.class); + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we do fail on violation + Put put = new Put(row1); + put.add(dummy, new byte[0], "fail".getBytes()); + LOG.warn("Doing put in table"); + try { + table.put(put); + fail("This put should not have suceeded - AllFailConstraint was not run!"); + } catch (RetriesExhaustedWithDetailsException e) { + List causes = e.getCauses(); + assertEquals( + "More than one failure cause - should only be the failure constraint exception", + 1, causes.size()); + Throwable t = causes.get(0); + assertEquals(ConstraintException.class, t.getClass()); + } + } + + /** + * Check that if we just disable one constraint, then + * @throws Throwable + */ + @SuppressWarnings("unchecked") + @Test + public void testDisableConstraint() throws Throwable { + // create the table + HTableDescriptor desc = new HTableDescriptor(tableName); + // add a family to the table + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + // add a constraint to make sure it others get run + Constraints.add(desc, CheckWasRunConstraint.class); + + // Add Constraint to check + Constraints.add(desc, AllFailConstraint.class); + + // and then disable the failing constraint + Constraints.disableConstraint(desc, AllFailConstraint.class); + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we don't fail because its disabled + Put put = new Put(row1); + put.add(dummy, new byte[0], "pass".getBytes()); + table.put(put); + + assertTrue(CheckWasRunConstraint.wasRun); + } + + /** + * Test that if we disable all constraints, then nothing gets run + * @throws Throwable + */ + @SuppressWarnings("unchecked") + @Test + public void testDisableConstraints() throws Throwable { + // create the table + HTableDescriptor desc = new HTableDescriptor(tableName); + // add a family to the table + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + // add a constraint to check to see if is run + Constraints.add(desc, CheckWasRunConstraint.class); + + // then disable all the constraints + Constraints.disable(desc); + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we do fail on violation + Put put = new Put(row1); + put.add(dummy, new byte[0], "pass".getBytes()); + LOG.warn("Doing put in table"); + table.put(put); + + assertFalse(CheckWasRunConstraint.wasRun); + } + + /** + * Check to make sure a constraint is unloaded when it fails + * @throws Exception + */ + @Test + public void testIsUnloaded() throws Exception { + // create the table + HTableDescriptor desc = new HTableDescriptor(tableName); + // add a family to the table + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + // make sure that constraints are unloaded + Constraints.add(desc, RuntimeFailConstraint.class); + // add a constraint to check to see if is run + Constraints.add(desc, CheckWasRunConstraint.class); + CheckWasRunConstraint.wasRun = false; + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we do fail on violation + Put put = new Put(row1); + put.add(dummy, new byte[0], "pass".getBytes()); + + try{ + table.put(put); + fail("RuntimeFailConstraint wasn't triggered - this put shouldn't work!"); + } catch (Exception e) {// NOOP + } + + // try the put again, this time constraints are not used, so it works + table.put(put); + // and we make sure that constraints were not run... + assertFalse(CheckWasRunConstraint.wasRun); + } + + @After + public void cleanup() throws Exception { + // cleanup + CheckWasRunConstraint.wasRun = false; + util.getHBaseAdmin().disableTable(tableName); + util.getHBaseAdmin().deleteTable(tableName); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + util.shutdownMiniCluster(); + } + + /** + * Constraint to check that it was actually run (or not) + */ + public static class CheckWasRunConstraint extends BaseConstraint { + public static boolean wasRun = false; + + @Override + public void check(Put p) { + wasRun = true; + } + } + +} diff --git src/test/java/org/apache/hadoop/hbase/constraint/TestConstraints.java src/test/java/org/apache/hadoop/hbase/constraint/TestConstraints.java new file mode 100644 index 0000000..e842242 --- /dev/null +++ src/test/java/org/apache/hadoop/hbase/constraint/TestConstraints.java @@ -0,0 +1,205 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.constraint.TestConstraint.CheckWasRunConstraint; +import org.apache.hadoop.hbase.constraint.WorksConstraint.NameConstraint; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test reading/writing the constraints into the {@link HTableDescriptor} + */ +@Category(SmallTests.class) +public class TestConstraints { + + @SuppressWarnings("unchecked") + @Test + public void testSimpleReadWrite() throws Throwable { + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.add(desc, WorksConstraint.class); + + List constraints = Constraints.getConstraints(desc, + this.getClass().getClassLoader()); + assertEquals(1, constraints.size()); + + assertEquals(WorksConstraint.class, constraints.get(0).getClass()); + + // Check that we can add more than 1 constraint and that ordering is + // preserved + Constraints.add(desc, AlsoWorks.class, NameConstraint.class); + constraints = Constraints.getConstraints(desc, this.getClass() + .getClassLoader()); + assertEquals(3, constraints.size()); + + assertEquals(WorksConstraint.class, constraints.get(0).getClass()); + assertEquals(AlsoWorks.class, constraints.get(1).getClass()); + assertEquals(NameConstraint.class, constraints.get(2).getClass()); + + } + + @SuppressWarnings("unchecked") + @Test + public void testReadWriteWithConf() throws Throwable { + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.add( + desc, + new Pair, Configuration>( + CheckConfigurationConstraint.class, CheckConfigurationConstraint + .getConfiguration())); + + List c = Constraints.getConstraints(desc, this + .getClass().getClassLoader()); + assertEquals(1, c.size()); + + assertEquals(CheckConfigurationConstraint.class, c.get(0).getClass()); + + // check to make sure that we overwrite configurations + Constraints.add(desc, new Pair, Configuration>( + CheckConfigurationConstraint.class, new Configuration())); + + try { + Constraints.getConstraints(desc, this.getClass().getClassLoader()); + assertTrue("No exception thrown - configuration not overwritten", false); + } catch (IllegalArgumentException e) { + // expect to have the exception, so don't do anything + } + } + + /** + * Test that Constraints are properly enabled, disabled, and removed + * + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test + public void testEnableDisableRemove() throws Exception { + HTableDescriptor desc = new HTableDescriptor("table"); + // check general enabling/disabling of constraints + // first add a constraint + Constraints.add(desc, AllPassConstraint.class); + // make sure everything is enabled + assertTrue(Constraints.enabled(desc, AllPassConstraint.class)); + assertTrue(desc.hasCoprocessor(ConstraintProcessor.class.getName())); + + // check disabling + Constraints.disable(desc); + assertFalse(desc.hasCoprocessor(ConstraintProcessor.class.getName())); + // make sure the added constraints are still present + assertTrue(Constraints.enabled(desc, AllPassConstraint.class)); + + // check just removing the single constraint + Constraints.remove(desc, AllPassConstraint.class); + assertFalse(Constraints.has(desc, AllPassConstraint.class)); + + // Add back the single constraint + Constraints.add(desc, AllPassConstraint.class); + + // and now check that when we remove constraints, all are gone + Constraints.remove(desc); + assertFalse(desc.hasCoprocessor(ConstraintProcessor.class.getName())); + assertFalse(Constraints.has(desc, AllPassConstraint.class)); + + } + + /** + * Test that when we update a constraint the ordering is not modified. + * + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test + public void testUpdateConstraint() throws Exception { + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.add(desc, CheckConfigurationConstraint.class, + CheckWasRunConstraint.class); + Constraints.setConfiguration(desc, CheckConfigurationConstraint.class, + CheckConfigurationConstraint.getConfiguration()); + + List constraints = Constraints.getConstraints(desc, + this.getClass().getClassLoader()); + + assertEquals(2, constraints.size()); + + // check to make sure the order didn't change + assertEquals(CheckConfigurationConstraint.class, constraints.get(0) + .getClass()); + assertEquals(CheckWasRunConstraint.class, constraints.get(1).getClass()); + } + + /** + * Test that if a constraint hasn't been set that there are no problems with + * attempting to remove it. + * + * @throws Throwable + * on failure. + */ + @Test + public void testRemoveUnsetConstraint() throws Throwable { + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.remove(desc); + Constraints.remove(desc, AlsoWorks.class); + } + + @Test + public void testConfigurationPreserved() throws Throwable { + Configuration conf = new Configuration(); + conf.setBoolean("_ENABLED", false); + conf.setLong("_PRIORITY", 10); + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.add(desc, AlsoWorks.class, conf); + Constraints.add(desc, WorksConstraint.class); + assertFalse(Constraints.enabled(desc, AlsoWorks.class)); + List constraints = Constraints.getConstraints(desc, + this.getClass().getClassLoader()); + for (Constraint c : constraints) { + Configuration storedConf = c.getConf(); + if (c instanceof AlsoWorks) + assertEquals(10, storedConf.getLong("_PRIORITY", -1)); + // its just a worksconstraint + else + assertEquals(2, storedConf.getLong("_PRIORITY", -1)); + + } + + } + + // ---------- Constraints just used for testing + + /** + * Also just works + */ + public static class AlsoWorks extends BaseConstraint { + @Override + public void check(Put p) { + // NOOP + } + } + +} diff --git src/test/java/org/apache/hadoop/hbase/constraint/WorksConstraint.java src/test/java/org/apache/hadoop/hbase/constraint/WorksConstraint.java new file mode 100644 index 0000000..fa94230 --- /dev/null +++ src/test/java/org/apache/hadoop/hbase/constraint/WorksConstraint.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.client.Put; + +/** + * It just works + */ +public class WorksConstraint extends BaseConstraint { + @Override + public void check(Put p) { + // NOOP + } + + /** + * Constraint to check that the naming of constraints doesn't mess up the + * pattern matching.(that constraint $___Constraint$NameConstraint isn't a + * problem) + */ + public static class NameConstraint extends WorksConstraint { + } +}