diff --git hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java index c3fe3a39f5bf4ce831032323231f8da731b1376f..13d50d8443e5d7170ecd30bc65a4957d19afed5f 100644 --- hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java +++ hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java @@ -147,6 +147,11 @@ private Set xmlFieldsMissingInConfiguration = null; /** + * Member variable for debugging base class operation + */ + protected boolean debug = false; + + /** * Abstract method to be used by subclasses for initializing base * members. */ @@ -168,13 +173,16 @@ HashMap retVal = new HashMap(); // Setup regexp for valid properties - String propRegex = "^[A-Za-z_-]+(\\.[A-Za-z_-]+)+$"; + String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$"; Pattern p = Pattern.compile(propRegex); // Iterate through class member variables int totalFields = 0; String value; for (Field f : fields) { + if (debug) { + System.out.println("Field: " + f); + } // Filter out anything that isn't "public static final" if (!Modifier.isStatic(f.getModifiers()) || !Modifier.isPublic(f.getModifiers()) || @@ -221,8 +229,14 @@ // something like: blah.blah2(.blah3.blah4...) Matcher m = p.matcher(value); if (!m.find()) { + if (debug) { + System.out.println(" Passes Regex: false"); + } continue; } + if (debug) { + System.out.println(" Passes Regex: true"); + } // Save member variable/value as hash retVal.put(value,f.getName()); diff --git hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestHdfsConfigFields.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestHdfsConfigFields.java new file mode 100644 index 0000000000000000000000000000000000000000..27ff467902fa58d9d9e27a4b4e3f7910d4def178 --- /dev/null +++ hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestHdfsConfigFields.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.tools; + +import java.util.HashSet; + +import org.apache.hadoop.conf.TestConfigurationFieldsBase; +import org.apache.hadoop.hdfs.DFSConfigKeys; + +/** + * Unit test class to compare the following MR Configuration classes: + *

+ * {@link org.apache.hadoop.hdfs.DFSConfigKeys} + *

+ * against hdfs-default.xml for missing properties. Currently only + * throws an error if the class is missing a property. + *

+ * Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase} + * for how this class works. + */ +public class TestHdfsConfigFields extends TestConfigurationFieldsBase { + + @Override + public void initializeMemberVariables() { + xmlFilename = new String("hdfs-default.xml"); + configurationClasses = new Class[] { DFSConfigKeys.class }; + + // Set error modes + errorIfMissingConfigProps = true; + errorIfMissingXmlProps = false; + + // Allocate + xmlPropsToSkipCompare = new HashSet(); + xmlPrefixToSkipCompare = new HashSet(); + + // Used in native code fuse_connect.c + xmlPropsToSkipCompare.add("hadoop.fuse.timer.period"); + xmlPropsToSkipCompare.add("hadoop.fuse.connection.timeout"); + + // Used dynamically as part of DFSConfigKeys.DFS_NAMENODE_EDITS_PLUGIN_PREFIX + xmlPropsToSkipCompare.add("dfs.namenode.edits.journal-plugin.qjournal"); + + // Example (not real) property in hdfs-default.xml + xmlPropsToSkipCompare.add("dfs.ha.namenodes.EXAMPLENAMESERVICE"); + + // Used oddly by DataNode to create new config String + xmlPropsToSkipCompare.add("hadoop.hdfs.configuration.version"); + + // Kept in the NfsConfiguration class in the hadoop-hdfs-nfs module + xmlPrefixToSkipCompare.add("nfs"); + } + +} diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java new file mode 100644 index 0000000000000000000000000000000000000000..184037c8ce056193c70205406680f35404de2ee3 --- /dev/null +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce; + +import org.apache.hadoop.conf.TestConfigurationFieldsBase; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.ShuffleHandler; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; + +/** + * Unit test class to compare the following MR Configuration classes: + *

+ * {@link org.apache.hadoop.mapreduce.MRJobConfig} + * {@link org.apache.hadoop.mapreduce.MRConfig} + * {@link org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig} + * {@link org.apache.hadoop.mapred.ShuffleHandler} + * {@link org.apache.hadoop.mapreduce.lib.output.FileOutputFormat} + * {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat} + * {@link org.apache.hadoop.mapreduce.Job} + * {@link org.apache.hadoop.mapreduce.lib.input.NLineInputFormat} + * {@link org.apache.hadoop.mapred.JobConf} + *

+ * against mapred-default.xml for missing properties. Currently only + * throws an error if the class is missing a property. + *

+ * Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase} + * for how this class works. + */ +public class TestMapreduceConfigFields extends TestConfigurationFieldsBase { + + @Override + public void initializeMemberVariables() { + xmlFilename = new String("mapred-default.xml"); + configurationClasses = new Class[] { MRJobConfig.class, MRConfig.class, + JHAdminConfig.class, ShuffleHandler.class, FileOutputFormat.class, + FileInputFormat.class, Job.class, NLineInputFormat.class, + JobConf.class }; + + // Set error modes + errorIfMissingConfigProps = false; + errorIfMissingXmlProps = false; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 971ce0b6fc16459b2e4e8fcd091dda40e3c67075..071063bc70c8d49cee6e9ea56658c27f8f6e1b13 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -24,7 +24,8 @@ - + + Factory to create client IPC classes. yarn.ipc.client.factory.class @@ -46,7 +47,8 @@ org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC - + + The hostname of the RM. yarn.resourcemanager.hostname @@ -123,6 +125,20 @@ + + + yarn.resourcemanager.webapp.spnego-keytab-file + + + + + + + yarn.resourcemanager.webapp.spnego-principal + + + + yarn.resourcemanager.resource-tracker.address ${yarn.resourcemanager.hostname}:8031 @@ -260,6 +276,13 @@ + + + yarn.scheduler.include-port-in-node-name + + + + Enable RM to recover state after starting. If true, then yarn.resourcemanager.store.class must be specified. yarn.resourcemanager.recovery.enabled @@ -651,7 +674,99 @@ 10 - + + + + yarn.resourcemanager.configuration.file-system-based-store + + + + + + + yarn.resourcemanager.delegation-token-renewer.thread-count + + + + + + + yarn.resourcemanager.delegation.key.update-interval + + + + + + + yarn.resourcemanager.delegation.token.max-lifetime + + + + + + + yarn.resourcemanager.delegation.token.renew-interval + + + + + + + yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size + + + + + + + yarn.resourcemanager.metrics.runtime.buckets + + + + + + + yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs + + + + + + + yarn.resourcemanager.reservation-system.class + + + + + + + yarn.resourcemanager.reservation-system.enable + + + + + + + yarn.resourcemanager.reservation-system.plan.follower + + + + + + + yarn.resourcemanager.reservation-system.planfollower.time-step + + + + + + + yarn.resourcemanager.rm.container-allocation.expiry-interval-ms + + + + + The hostname of the NM. yarn.nodemanager.hostname @@ -914,6 +1029,27 @@ + + + yarn.nodemanager.webapp.https.address + + + + + + + yarn.nodemanager.webapp.spnego-keytab-file + + + + + + + yarn.nodemanager.webapp.spnego-principal + + + + How often to monitor containers. yarn.nodemanager.container-monitor.interval-ms 3000 @@ -1060,6 +1196,20 @@ + + + yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms + + + + + + + yarn.nodemanager.linux-container-executor.group + + + + T-file compression types used to compress aggregated logs. yarn.nodemanager.log-aggregation.compression-type none @@ -1154,7 +1304,63 @@ ${hadoop.tmp.dir}/yarn-nm-recovery - + + + + yarn.nodemanager.container-executor.os.sched.priority.adjustment + + + + + + + yarn.nodemanager.container-metrics.enable + + + + + + + yarn.nodemanager.container-metrics.period-ms + + + + + + + yarn.nodemanager.container-monitor.process-tree.class + + + + + + + yarn.nodemanager.disk-health-checker.enable + + + + + + + yarn.nodemanager.log.deletion-threads-count + + + + + + + yarn.nodemanager.user-home-dir + + + + + + + yarn.nodemanager.windows-secure-container-executor.group + + + + yarn.nodemanager.docker-container-executor.exec-name @@ -1164,7 +1370,15 @@ - + + + + yarn.nodemanager.docker-container-executor.image-name + + + + + yarn.nodemanager.aux-services.mapreduce_shuffle.class org.apache.hadoop.mapred.ShuffleHandler @@ -1180,7 +1394,7 @@ ${fs.defaultFS} - + The kerberos principal for the proxy, if the proxy is not @@ -1202,7 +1416,7 @@ - + @@ -1230,7 +1444,28 @@ - + + + + yarn.app.container.log.dir + + + + + + + yarn.app.container.log.backups + + + + + + + yarn.app.container.log.filesize + + + + Indicate to clients whether timeline service is enabled or not. @@ -1546,7 +1781,75 @@ 20 - + + + + security.applicationhistory.protocol.acl + + + + + + + + + yarn.is.minicluster + false + + + + + + yarn.minicluster.control-resource-monitoring + false + + + + + + yarn.minicluster.fixed.ports + + + + + + + yarn.minicluster.use-rpc + + + + + + + yarn.minicluster.yarn.nodemanager.resource.memory-mb + + + + + + + + + yarn.node-labels.enabled + + + + + + + yarn.node-labels.fs-store.retry-policy-spec + + + + + + + yarn.node-labels.fs-store.root-dir + + + + + The interval that the yarn client library uses to poll the completion status of the asynchronous API of application client protocol. @@ -1556,6 +1859,20 @@ + + + yarn.client.application-client-protocol.poll-timeout-ms + + + + + + + yarn.client.app-submission.poll-interval + + + + RSS usage of a process computed via /proc/pid/stat is not very accurate as it includes shared pages of a process. /proc/pid/smaps provides useful information like @@ -1568,6 +1885,20 @@ false + + + + yarn.log.server.url + + + + + + + yarn.tracking.url.generator + + + @@ -1703,4 +2034,5 @@ yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds -1 + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 9075d9f6dd3e589399b1c6d46ab40f77f181fa3b..ff235bbd5f5c4368ac8b2b9d9e01f39f3bfa9228 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -37,14 +37,14 @@ public void initializeMemberVariables() { xmlFilename = new String("yarn-default.xml"); configurationClasses = new Class[] { YarnConfiguration.class }; - // Allocate for usage configurationPropsToSkipCompare = new HashSet(); + configurationPrefixToSkipCompare = new HashSet(); // Set error modes errorIfMissingConfigProps = true; - errorIfMissingXmlProps = false; + errorIfMissingXmlProps = true; // Specific properties to skip configurationPropsToSkipCompare @@ -79,6 +79,16 @@ public void initializeMemberVariables() { configurationPropsToSkipCompare .add(YarnConfiguration .YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL); + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_SCM_STORE_CLASS); + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS); + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL); + + // Ignore all YARN Application Timeline Service (version 1) properties + configurationPrefixToSkipCompare + .add("yarn.timeline-service."); // Allocate for usage xmlPropsToSkipCompare = new HashSet(); @@ -94,10 +104,9 @@ public void initializeMemberVariables() { // Used in the XML file as a variable reference internal to the XML file xmlPropsToSkipCompare.add("yarn.nodemanager.hostname"); - xmlPropsToSkipCompare.add("yarn.timeline-service.hostname"); - // Currently defined in TimelineAuthenticationFilterInitializer - xmlPrefixToSkipCompare.add("yarn.timeline-service.http-authentication"); + // Ignore all YARN Application Timeline Service (version 1) properties + xmlPrefixToSkipCompare.add("yarn.timeline-service"); // Currently defined in RegistryConstants xmlPrefixToSkipCompare.add("hadoop.registry");