Index: src/java/org/apache/hadoop/conf/Configuration.java =================================================================== --- src/java/org/apache/hadoop/conf/Configuration.java (revision 798782) +++ src/java/org/apache/hadoop/conf/Configuration.java (working copy) @@ -167,7 +167,80 @@ private static final ArrayList defaultResources = new ArrayList(); - static{ + /** + * Class to keep the information about the keys which replace the deprecated + * ones. + * + * This class stores the new keys which replace the deprecated keys and also + * gives a provision to have a custom message for each of the deprecated key + * that is being replaced. + */ + private static class DeprecatedKeyInfo { + private String[] newKeys; + private String customMessage; + DeprecatedKeyInfo(String[] newKeys, String customMessage) { + this.newKeys = newKeys; + this.customMessage = customMessage; + } + DeprecatedKeyInfo(String[] newKeys) { + this(newKeys, null); + } + } + + /** + * logs the warning message indicating the usage of the deprecated key. + * @param key deprecated key + * @param keyInfo DeprecatedKeyInfo object which stores the new keys and + * custom message(if any). + */ + private static void logDeprecationWarning(String key, + DeprecatedKeyInfo keyInfo) { + if (keyInfo.customMessage != null) { + LOG.warn(keyInfo.customMessage); + } + else { + String deprecatedKeySuffix = " is deprecated. Instead, use "; + StringBuilder message = new StringBuilder(key); + message.append(deprecatedKeySuffix); + message.append(keyInfo.newKeys[0]); + for (int i = 1; i < keyInfo.newKeys.length; i++) { + message.append(","); + message.append(keyInfo.newKeys[i]); + } + LOG.warn(message.toString()); + } + } + + /** + * Stores the deprecated keys, the new keys which replace the deprecated keys + * and custom message(if any provided). + */ + private static HashMap deprecatedKeyMap; + + /** + * Stores the keys which replace the deprecated keys and are loaded with + * the values of the corresponding deprecated keys. + */ + private static ArrayList updatedNewKeys; + + /** + * adds the deprecated key to the deprecation map. + * This is used only for testing and should not be used by user at runtime. + * It does not override any existing entries in the deprecation map. + * + * @param key Key that is to be deprecated + * @param newKeys list of values that take up the values of deprecated key + * @param message message that is customed to be printed whenever the + * corresponding deprecated key is used + */ + protected static void addDeprecation(String key, String[] newKeys) { + if(!deprecatedKeyMap.containsKey(key)) { + DeprecatedKeyInfo newKeyInfo = new DeprecatedKeyInfo(newKeys); + deprecatedKeyMap.put(key, newKeyInfo); + } + } + + static { //print deprecation warning if hadoop-site.xml is found in classpath ClassLoader cL = Thread.currentThread().getContextClassLoader(); if (cL == null) { @@ -182,6 +255,8 @@ } addDefaultResource("core-default.xml"); addDefaultResource("core-site.xml"); + deprecatedKeyMap = new HashMap(); + updatedNewKeys = new ArrayList(); } private Properties properties; @@ -370,7 +445,8 @@ /** * Get the value of the name property, null if - * no such property exists. + * no such property exists. It returns the value of the new key, if + * the old key is deprecated. * * Values are processed for variable expansion * before being returned. @@ -380,30 +456,60 @@ * or null if no such property exists. */ public String get(String name) { - return substituteVars(getProps().getProperty(name)); + return substituteVars(getRaw(name)); } /** * Get the value of the name property, without doing - * variable expansion. + * variable expansion. It returns + * the value of the replacing property if name is deprecated. * * @param name the property name. - * @return the value of the name property, - * or null if no such property exists. + * @return the value of the name property or + * its replacing property and null if no such property exists. */ public String getRaw(String name) { + name = handleDeprecation(name); return getProps().getProperty(name); } + /** + * Checks for the presence of the property name in the + * deprecation map. Returns the first of the list of new keys if present + * in the deprecation map or the name itself. + * @param name the property name + * @return the first property in the list of properties mapping + * the name or the name itself. + */ + private String handleDeprecation(String name) { + if (deprecatedKeyMap.containsKey(name)) { + DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name); + logDeprecationWarning(name, keyInfo); + name = keyInfo.newKeys[0]; + } + return name; + } + /** - * Set the value of the name property. + * Set the value of the name property. Sets + * value of the replacing properties, if name is + * deprecated. * * @param name property name. * @param value property value. */ public void set(String name, String value) { - getOverlay().setProperty(name, value); - getProps().setProperty(name, value); + if (deprecatedKeyMap.containsKey(name)) { + DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name); + logDeprecationWarning(name,keyInfo); + for(int i = 0; i < keyInfo.newKeys.length; i++) { + set(keyInfo.newKeys[i], value); + } + } + else { + getOverlay().setProperty(name, value); + getProps().setProperty(name, value); + } } /** @@ -425,8 +531,9 @@ } /** - * Get the value of the name property. If no such property - * exists, then defaultValue is returned. + * Get the value of the name or the replacing property's (if + * name is deprecated) value. If no such property exists, + * then defaultValue is returned. * * @param name property name. * @param defaultValue default value. @@ -434,6 +541,7 @@ * doesn't exist. */ public String get(String name, String defaultValue) { + name = handleDeprecation(name); return substituteVars(getProps().getProperty(name, defaultValue)); } @@ -473,7 +581,6 @@ set(name, Integer.toString(value)); } - /** * Get the value of the name property as a long. * If no such property is specified, or if the specified value is not a valid @@ -547,6 +654,7 @@ return defaultValue; } } + /** * Set the value of the name property to a float. * @@ -1243,20 +1351,61 @@ if ("final".equals(field.getTagName()) && field.hasChildNodes()) finalParameter = "true".equals(((Text)field.getFirstChild()).getData()); } - - // Ignore this parameter if it has already been marked as 'final' - if (attr != null && value != null) { - if (!finalParameters.contains(attr)) { + if(attr == null || value == null) { + continue; + } + // check for deprecation. + if(deprecatedKeyMap.containsKey(attr)) { + DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(attr); + logDeprecationWarning(attr, keyInfo); + for (int j = 0; j < keyInfo.newKeys.length; j++) { + // override old deprecated key name with new key mapping name + attr = keyInfo.newKeys[j]; + // update replacing keys with deprecated key's value + // in all cases, except when the replacing keys are + // already set to final + if(finalParameter || !finalParameters.contains(attr)) { + properties.setProperty(attr, value); + updatedNewKeys.add(attr); + if(!finalParameters.contains(attr) && finalParameter) { + finalParameters.add(attr); + } + } + //new keys are final hence log warning. + else { + LOG.warn(name+":an attempt to override final parameter: "+attr + +"; Ignoring."); + } + } + } + else { + // set the value to key only if + // it is not set to final before and either + // 1.it is not final and is not previously set with + // value of the corresponding deprecated key, or + // 2.it is final. + if (!finalParameter) { + if(!finalParameters.contains(attr) + && !updatedNewKeys.contains(attr)) { + properties.setProperty(attr, value); + } + else if (!updatedNewKeys.contains(attr)) { + LOG.warn(name+":an attempt to override final parameter: "+attr + +"; Ignoring."); + } + } + else if (!finalParameters.contains(attr)) { properties.setProperty(attr, value); - if (finalParameter) - finalParameters.add(attr); - } else { - LOG.warn(name+":a attempt to override final parameter: "+attr - +"; Ignoring."); + finalParameters.add(attr); } + else { + if (!updatedNewKeys.contains(attr)) { + LOG.warn(name+":an attempt to override final parameter: "+attr + +"; Ignoring."); + } + } } } - } catch (IOException e) { LOG.fatal("error parsing conf file: " + e); throw new RuntimeException(e); Index: src/test/core/org/apache/hadoop/conf/TestConfiguration.java =================================================================== --- src/test/core/org/apache/hadoop/conf/TestConfiguration.java (revision 798782) +++ src/test/core/org/apache/hadoop/conf/TestConfiguration.java (working copy) @@ -21,10 +21,6 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.io.DataInputStream; -import java.io.ByteArrayOutputStream; -import java.io.ByteArrayInputStream; -import java.io.DataOutputStream; import java.util.ArrayList; import java.util.Random; @@ -38,7 +34,9 @@ private Configuration conf; final static String CONFIG = new File("./test-config.xml").getAbsolutePath(); final static String CONFIG2 = new File("./test-config2.xml").getAbsolutePath(); + final static String CONFIG3 = new File("./test-config3.xml").getAbsolutePath(); final static Random RAN = new Random(); + @Override protected void setUp() throws Exception { @@ -51,6 +49,7 @@ super.tearDown(); new File(CONFIG).delete(); new File(CONFIG2).delete(); + new File(CONFIG3).delete(); } private void startConfig() throws IOException{ @@ -384,7 +383,92 @@ assertEquals(null, conf.get("test.key2")); assertEquals("value5", conf.get("test.key4")); } + + public void testDeprecation() throws IOException { + out=new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + // add keys with default values. Some of them are set to final in order to + // test the precedence order between being deprecated and being final + appendProperty("new.key1","default.value1",true); + appendProperty("new.key2","default.value2"); + appendProperty("new.key3","default.value3",true); + appendProperty("new.key4","default.value4"); + appendProperty("new.key5","default.value5",true); + appendProperty("new.key6","default.value6"); + appendProperty("new.key7","default.value7",true); + appendProperty("new.key8","default.value8"); + endConfig(); + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + + // add few deprecation mappings.Some of them are set to final in order to + // test the precedence order between being deprecated and being final. + conf.addDeprecation("old.key1", new String[]{"new.key1"}); + conf.addDeprecation("old.key2", new String[]{"new.key2"}); + conf.addDeprecation("old.key3", new String[]{"new.key3"}); + conf.addDeprecation("old.key4", new String[]{"new.key4"}); + conf.addDeprecation("old.key9", new String[]{"new.key9"}); + + out=new BufferedWriter(new FileWriter(CONFIG2)); + startConfig(); + // add deprecated keys.Some of them are set to final in order to + // test the precedence order between being deprecated and being final. + appendProperty("old.key1","old.value1",true); + appendProperty("old.key2","old.value2",true); + appendProperty("old.key3","old.value3"); + appendProperty("old.key4","old.value4"); + appendProperty("old.key9","old.value9"); + endConfig(); + Path fileResource1 = new Path(CONFIG2); + conf.addResource(fileResource1); + + // get the values of the deprecated keys. + assertEquals("old.value1",conf.get("old.key1")); + assertEquals("old.value2",conf.get("old.key2")); + assertEquals("default.value3",conf.get("old.key3")); + assertEquals("old.value4",conf.get("old.key4")); + + // get the values of the non-deprecated keys which map to deprecated keys. + assertEquals("old.value1",conf.get("new.key1")); + assertEquals("old.value2",conf.get("new.key2")); + assertEquals("default.value3",conf.get("new.key3")); + assertEquals("old.value4",conf.get("new.key4")); + + out=new BufferedWriter(new FileWriter(CONFIG3)); + startConfig(); + // set non-deprecated keys' values.Some of them are set to final in order to + // test the precedence order between being deprecated and being final. + appendProperty("new.key1","new.value1",true); + appendProperty("new.key2","new.value2"); + appendProperty("new.key3","new.value3",true); + appendProperty("new.key4","new.value4"); + appendProperty("new.key5","new.value5",true); + appendProperty("new.key6","new.value6",true); + appendProperty("new.key7","new.value7"); + appendProperty("new.key8","new.value8"); + appendProperty("new.key9","new.value9",true); + endConfig(); + Path fileResource2 = new Path(CONFIG3); + conf.addResource(fileResource2); + // get the values of the deprecated keys + assertEquals("old.value1",conf.get("old.key1")); + assertEquals("old.value2",conf.get("old.key2")); + assertEquals("default.value3",conf.get("old.key3")); + assertEquals("old.value4",conf.get("old.key4")); + assertEquals("new.value9",conf.get("old.key9")); + + //get the values of all the non-deprecated keys. + assertEquals("old.value1",conf.get("new.key1")); + assertEquals("old.value2",conf.get("new.key2")); + assertEquals("default.value3",conf.get("new.key3")); + assertEquals("old.value4",conf.get("new.key4")); + assertEquals("new.value6",conf.get("new.key6")); + assertEquals("default.value7",conf.get("new.key7")); + assertEquals("new.value8",conf.get("new.key8")); + assertEquals("new.value9",conf.get("new.key9")); + } + public void testSize() throws IOException { Configuration conf = new Configuration(false); conf.set("a", "A");