Index: src/java/org/apache/hadoop/conf/Configuration.java =================================================================== --- src/java/org/apache/hadoop/conf/Configuration.java (revision 798782) +++ src/java/org/apache/hadoop/conf/Configuration.java (working copy) @@ -155,7 +155,7 @@ private boolean loadDefaults = true; /** - * Configurtion objects + * Configuration objects */ private static final WeakHashMap REGISTRY = new WeakHashMap(); @@ -167,7 +167,152 @@ private static final ArrayList defaultResources = new ArrayList(); - static{ + /** + * Class to keep the information about the keys which replace the deprecated + * ones. + * + * This class stores the new keys which replace the deprecated keys and also + * gives a provision to have a custom message for each of the deprecated key + * that is being replaced. It also provides method to get the appropriate + * warning message which can be logged whenever the deprecated key is used. + */ + private static class DeprecatedKeyInfo { + private String[] newKeys; + private String customMessage; + DeprecatedKeyInfo(String[] newKeys, String customMessage) { + this.newKeys = newKeys; + this.customMessage = customMessage; + } + DeprecatedKeyInfo(String[] newKeys) { + this(newKeys, null); + } + + /** + * Method to provide the warning message. It gives the custom message if + * non-null, and default message otherwise. + * @param key the associated deprecated key. + * @return message that is to be logged when a deprecated key is used. + */ + private final String getWarningMessage(String key) { + String warningMessage; + if(customMessage == null) { + StringBuilder message = new StringBuilder(key); + String deprecatedKeySuffix = " is deprecated. Instead, use "; + message.append(deprecatedKeySuffix); + for (int i = 0; i < newKeys.length; i++) { + message.append(newKeys[i]); + if(i != newKeys.length-1) { + message.append(", "); + } + } + warningMessage = message.toString(); + } + else { + warningMessage = customMessage; + } + return warningMessage; + } + } + + /** + * Stores the deprecated keys, the new keys which replace the deprecated keys + * and custom message(if any provided). + */ + private static HashMap deprecatedKeyMap = + new HashMap(); + + /** + * Stores the keys which replace the deprecated keys and are loaded with + * the values of the corresponding deprecated keys. + */ + private static ArrayList havingDeprecatedValue = + new ArrayList(); + + /** + * adds the deprecated key to the deprecation map. + * This is used only for testing and should not be used by user at runtime. + * It does not override any existing entries in the deprecation map. + * + * @param key Key that is to be deprecated + * @param newKeys list of values that take up the values of deprecated key + */ + static void addDeprecation(String key, String[] newKeys) { + if(key.equals(null) || newKeys.equals(null)) { + return; + } + if(!isDeprecated(key)) { + DeprecatedKeyInfo newKeyInfo = new DeprecatedKeyInfo(newKeys); + deprecatedKeyMap.put(key, newKeyInfo); + } + } + + /** + * checks whether the given key is deprecated. + * @param key the parameter which is to be checked for deprecation + * @return true if the key is deprecated and + * false otherwise. + */ + private static boolean isDeprecated(String key) { + return deprecatedKeyMap.containsKey(key); + } + + /** + * checks if the key is loaded with the corresponding deprecated key's value. + * @param key the attribute whose value is to be checked for corresponding + * deprecated key's value + * @return true if the key's value is that of the corresponding + * deprecated key,and false otherwise. + */ + private static boolean hasDeprecatedValue(String key) { + return havingDeprecatedValue.contains(key); + } + + /** Sets the deprecated key's value to the associated mapped keys + * @param name name of the resource + * @param attr the deprecated key + * @param value the value corresponding to the deprecated key + * @param finalParameter flag to indicate if attr is + * marked as final + */ + private void setDeprecatedValue(Object name, + String attr, String value, boolean finalParameter) { + DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(attr); + LOG.warn(keyInfo.getWarningMessage(attr)); + for (String key:keyInfo.newKeys) { + // update replacing keys with deprecated key's value in all cases, + // except when the replacing key is already set to final + // and finalParameter is false + if(finalParameters.contains(key) && !finalParameter) { + LOG.warn(name+":an attempt to override final parameter: "+key + +"; Ignoring."); + continue; + } + properties.setProperty(key, value); + havingDeprecatedValue.add(key); + if(finalParameter) { + finalParameters.add(key); + } + } + } + + /** + * Checks for the presence of the property name in the + * deprecation map. Returns the first of the list of new keys if present + * in the deprecation map or the name itself. + * @param name the property name + * @return the first property in the list of properties mapping + * the name or the name itself. + */ + private String handleDeprecation(String name) { + if (isDeprecated(name)) { + DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name); + LOG.warn(keyInfo.getWarningMessage(name)); + name = keyInfo.newKeys[0]; + } + return name; + } + + static { //print deprecation warning if hadoop-site.xml is found in classpath ClassLoader cL = Thread.currentThread().getContextClassLoader(); if (cL == null) { @@ -370,7 +515,8 @@ /** * Get the value of the name property, null if - * no such property exists. + * no such property exists. It returns the value of the new key, if + * the old key is deprecated. * * Values are processed for variable expansion * before being returned. @@ -380,30 +526,44 @@ * or null if no such property exists. */ public String get(String name) { + name = handleDeprecation(name); return substituteVars(getProps().getProperty(name)); } /** * Get the value of the name property, without doing - * variable expansion. + * variable expansion. It returns + * the value of the replacing property if name is deprecated. * * @param name the property name. - * @return the value of the name property, - * or null if no such property exists. + * @return the value of the name property or + * its replacing property and null if no such property exists. */ public String getRaw(String name) { + name = handleDeprecation(name); return getProps().getProperty(name); } /** - * Set the value of the name property. + * Set the value of the name property. Sets + * value of the replacing properties, if name is + * deprecated. * * @param name property name. * @param value property value. */ public void set(String name, String value) { - getOverlay().setProperty(name, value); - getProps().setProperty(name, value); + if (isDeprecated(name)) { + DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name); + LOG.warn(keyInfo.getWarningMessage(name)); + for(int i = 0; i < keyInfo.newKeys.length; i++) { + set(keyInfo.newKeys[i], value); + } + } + else { + getOverlay().setProperty(name, value); + getProps().setProperty(name, value); + } } /** @@ -425,8 +585,9 @@ } /** - * Get the value of the name property. If no such property - * exists, then defaultValue is returned. + * Get the value of the name or the replacing property's (if + * name is deprecated) value. If no such property exists, + * then defaultValue is returned. * * @param name property name. * @param defaultValue default value. @@ -434,6 +595,7 @@ * doesn't exist. */ public String get(String name, String defaultValue) { + name = handleDeprecation(name); return substituteVars(getProps().getProperty(name, defaultValue)); } @@ -473,7 +635,6 @@ set(name, Integer.toString(value)); } - /** * Get the value of the name property as a long. * If no such property is specified, or if the specified value is not a valid @@ -547,6 +708,7 @@ return defaultValue; } } + /** * Set the value of the name property to a float. * @@ -1241,22 +1403,34 @@ if ("value".equals(field.getTagName()) && field.hasChildNodes()) value = ((Text)field.getFirstChild()).getData(); if ("final".equals(field.getTagName()) && field.hasChildNodes()) - finalParameter = "true".equals(((Text)field.getFirstChild()).getData()); + finalParameter = + "true".equals(((Text)field.getFirstChild()).getData()); } - - // Ignore this parameter if it has already been marked as 'final' - if (attr != null && value != null) { - if (!finalParameters.contains(attr)) { + if(attr == null || value == null) { + continue; + } + // check for deprecation. + if(isDeprecated(attr)) { + setDeprecatedValue(name, attr, value, finalParameter); + continue; + } + // check for attr being loaded before by the value of a deprecated key + if(hasDeprecatedValue(attr)) { + if(!finalParameters.contains(attr) && finalParameter) { properties.setProperty(attr, value); - if (finalParameter) - finalParameters.add(attr); - } else { - LOG.warn(name+":a attempt to override final parameter: "+attr - +"; Ignoring."); + finalParameters.add(attr); } + continue; } + if (!finalParameters.contains(attr)) { + properties.setProperty(attr, value); + if (finalParameter) + finalParameters.add(attr); + } else { + LOG.warn(name+":a attempt to override final parameter: "+attr + +"; Ignoring."); + } } - } catch (IOException e) { LOG.fatal("error parsing conf file: " + e); throw new RuntimeException(e); @@ -1272,6 +1446,7 @@ } } + /** * Write out the non-default properties in this configuration to the give * {@link OutputStream}. Index: src/test/core/org/apache/hadoop/conf/TestConfiguration.java =================================================================== --- src/test/core/org/apache/hadoop/conf/TestConfiguration.java (revision 798782) +++ src/test/core/org/apache/hadoop/conf/TestConfiguration.java (working copy) @@ -21,10 +21,6 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.io.DataInputStream; -import java.io.ByteArrayOutputStream; -import java.io.ByteArrayInputStream; -import java.io.DataOutputStream; import java.util.ArrayList; import java.util.Random; @@ -38,7 +34,9 @@ private Configuration conf; final static String CONFIG = new File("./test-config.xml").getAbsolutePath(); final static String CONFIG2 = new File("./test-config2.xml").getAbsolutePath(); + final static String CONFIG3 = new File("./test-config3.xml").getAbsolutePath(); final static Random RAN = new Random(); + @Override protected void setUp() throws Exception { @@ -51,6 +49,7 @@ super.tearDown(); new File(CONFIG).delete(); new File(CONFIG2).delete(); + new File(CONFIG3).delete(); } private void startConfig() throws IOException{ @@ -384,7 +383,188 @@ assertEquals(null, conf.get("test.key2")); assertEquals("value5", conf.get("test.key4")); } + + /** + * This test is to check the precedence order between being final and + * deprecation.Based on the order of occurrence of deprecated key and + * its corresponding mapping key, various cases arise. + * The precedence order being followed is: + * 1. Final Parameter + * 2. Deprecated key's value. + * + * @throws IOException + */ + public void testDeprecation() throws IOException { + out=new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + // load keys with default values. Some of them are set to final to + // test the precedence order between deprecation and being final + appendProperty("new.key1","default.value1",true); + appendProperty("new.key2","default.value2"); + appendProperty("new.key3","default.value3",true); + appendProperty("new.key4","default.value4"); + appendProperty("new.key5","default.value5",true); + appendProperty("new.key6","default.value6"); + appendProperty("new.key7","default.value7",true); + appendProperty("new.key8","default.value8"); + appendProperty("new.key9","default.value9"); + appendProperty("new.key10","default.value10"); + appendProperty("new.key11","default.value11"); + appendProperty("new.key12","default.value12"); + appendProperty("new.key13","default.value13"); + appendProperty("new.key14","default.value14"); + appendProperty("new.key15","default.value15"); + appendProperty("new.key16","default.value16"); + endConfig(); + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + + // add deprecation mappings. + Configuration.addDeprecation("old.key1", new String[]{"new.key1"}); + Configuration.addDeprecation("old.key2", new String[]{"new.key2"}); + Configuration.addDeprecation("old.key3", new String[]{"new.key3"}); + Configuration.addDeprecation("old.key4", new String[]{"new.key4"}); + Configuration.addDeprecation("old.key5", new String[]{"new.key5"}); + Configuration.addDeprecation("old.key6", new String[]{"new.key6"}); + Configuration.addDeprecation("old.key7", new String[]{"new.key7"}); + Configuration.addDeprecation("old.key8", new String[]{"new.key8"}); + Configuration.addDeprecation("old.key9", new String[]{"new.key9"}); + Configuration.addDeprecation("old.key10", new String[]{"new.key10"}); + Configuration.addDeprecation("old.key11", new String[]{"new.key11"}); + Configuration.addDeprecation("old.key12", new String[]{"new.key12"}); + Configuration.addDeprecation("old.key13", new String[]{"new.key13"}); + Configuration.addDeprecation("old.key14", new String[]{"new.key14"}); + Configuration.addDeprecation("old.key15", new String[]{"new.key15"}); + Configuration.addDeprecation("old.key16", new String[]{"new.key16"}); + + out=new BufferedWriter(new FileWriter(CONFIG2)); + startConfig(); + + // add keys that are tested while they are loaded just after their + // corresponding default values + appendProperty("old.key1","old.value1",true); + appendProperty("old.key2","old.value2",true); + appendProperty("old.key3","old.value3"); + appendProperty("old.key4","old.value4"); + appendProperty("new.key5","new.value5",true); + appendProperty("new.key6","new.value6",true); + appendProperty("new.key7","new.value7"); + appendProperty("new.key8","new.value8"); + + // add keys that are tested while they are loaded first and are followed by + // loading of their corresponding deprecated or replacing key + appendProperty("new.key9","new.value9",true); + appendProperty("new.key10","new.value10"); + appendProperty("new.key11","new.value11",true); + appendProperty("new.key12","new.value12"); + appendProperty("old.key13","old.value13",true); + appendProperty("old.key14","old.value14"); + appendProperty("old.key15","old.value15",true); + appendProperty("old.key16","old.value16"); + endConfig(); + Path fileResource1 = new Path(CONFIG2); + conf.addResource(fileResource1); + + // get the values. Also check for consistency in get of old and new keys, + // when they are set to final or non-final + // Key - the key that is being loaded + // isFinal - true if the key is marked as final + // prev.occurrence - key that most recently loaded the current key + // with its value. + // isPrevFinal - true if key corresponding to + // prev.occurrence is marked as final. + + // Key-deprecated , isFinal-true, prev.occurrence-default.xml, + // isPrevFinal-true + assertEquals("old.value1", conf.get("new.key1")); + // check consistency in get of old and new keys + assertEquals(conf.get("old.key1"), conf.get("new.key1")); + // Key-deprecated , isFinal-true, prev.occurrence-default.xml, + // isPrevFinal-false + assertEquals("old.value2", conf.get("new.key2")); + // check consistency in get of old and new keys + assertEquals(conf.get("old.key2"), conf.get("new.key2")); + // Key-deprecated , isFinal-false, prev.occurrence-default.xml, + // isPrevFinal-true + assertEquals("default.value3", conf.get("new.key3")); + // check consistency in get of old and new keys + assertEquals(conf.get("old.key3"), conf.get("new.key3")); + // Key-deprecated , isFinal-false, prev.occurrence-default.xml, + // isPrevFinal-false + assertEquals("old.value4", conf.get("new.key4")); + // check consistency in get of old and new keys + assertEquals(conf.get("old.key4"), conf.get("new.key4")); + // Key-site.xml , isFinal-true, prev.occurrence-default.xml, + // isPrevFinal-true + assertEquals("default.value5", conf.get("new.key5")); + // Key-site.xml , isFinal-true, prev.occurrence-default.xml, + // isPrevFinal-false + assertEquals("new.value6",conf.get("new.key6")); + // Key-site.xml , isFinal-false, prev.occurrence-default.xml, + // isPrevFinal-true + assertEquals("default.value7", conf.get("new.key7")); + // Key-site.xml , isFinal-false, prev.occurrence-default.xml, + // isPrevFinal-false + assertEquals("new.value8",conf.get("new.key8")); + + + out=new BufferedWriter(new FileWriter(CONFIG3)); + startConfig(); + // add keys which are already loaded by the corresponding replacing or + // deprecated key. + appendProperty("old.key9","old.value9",true); + appendProperty("old.key10","old.value10",true); + appendProperty("old.key11","old.value11"); + appendProperty("old.key12","old.value12"); + appendProperty("new.key13","new.value13",true); + appendProperty("new.key14","new.value14",true); + appendProperty("new.key15","new.value15"); + appendProperty("new.key16","new.value16"); + endConfig(); + Path fileResource2 = new Path(CONFIG3); + conf.addResource(fileResource2); + + // get the values + // Key-deprecated , isFinal-true, prev.occurrence-site.xml, + // isPrevFinal-true + assertEquals("old.value9", conf.get("new.key9")); + // Key-deprecated , isFinal-true, prev.occurrence-site.xml, + // isPrevFinal-false + assertEquals("old.value10", conf.get("new.key10")); + // Key-deprecated , isFinal-false, prev.occurrence-site.xml, + // isPrevFinal-true + assertEquals("new.value11", conf.get("new.key11")); + // Key-deprecated , isFinal-false, prev.occurrence-site.xml, + // isPrevFinal-false + assertEquals("old.value12", conf.get("new.key12")); + // Key-site.xml , isFinal-true, prev.occurrence-deprecated, + // isPrevFinal-true + assertEquals("old.value13", conf.get("new.key13")); + // Key-site.xml , isFinal-true, prev.occurrence-deprecated, + // isPrevFinal-false + assertEquals("new.value14", conf.get("new.key14")); + // Key-site.xml , isFinal-false, prev.occurrence-deprecated, + // isPrevFinal-true + assertEquals("old.value15", conf.get("new.key15")); + // Key-site.xml , isFinal-false, prev.occurrence-deprecated, + // isPrevFinal-false + assertEquals("old.value16", conf.get("new.key16")); + + // check for consistency in get and set of deprecated and corresponding + // new keys from the user code + Configuration.addDeprecation("A", new String[]{"B"}); + + //add old and new keys + conf.reloadConfiguration(); + conf.set("A","valueA"); + conf.set("B", "valueB"); + + //get new and old keys + assertEquals("valueB", conf.get("A")); + assertEquals("valueB", conf.get("B")); + } + public void testSize() throws IOException { Configuration conf = new Configuration(false); conf.set("a", "A");