diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml index 5f1196c..dab5a37 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml @@ -103,6 +103,12 @@ dnsjava + + commons-validator + commons-validator + 1.4.0 + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java index 7115a4c..a399724 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java @@ -144,6 +144,21 @@ String KEY_DNS_ZONES_DIR = DNS_PREFIX + "zones-dir"; /** + * Split Reverse Zone. + * It may be necessary to spit large reverse zone subnets + * into multiple zones to handle existing hosts collocated + * with containers. + */ + String KEY_DNS_SPLIT_REVERSE_ZONE = DNS_PREFIX + "split-reverse-zone"; + + /** + * Split Reverse Zone IP Range. + * How many IPs should be part of each reverse zone split + */ + String KEY_DNS_SPLIT_REVERSE_ZONE_RANGE = DNS_PREFIX + + "split-reverse-zone-range"; + + /** * Key to set if the registry is secure: {@value}. * Turning it on changes the permissions policy from "open access" * to restrictions on kerberos with the option of diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java index 52b3c37..a5343ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.registry.server.dns; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.IOFileFilter; import org.apache.commons.net.util.Base64; @@ -95,6 +96,7 @@ import java.util.regex.Pattern; import static org.apache.hadoop.registry.client.api.RegistryConstants.*; +import org.apache.commons.validator.ValidatorException; /** * A DNS service reflecting the state of the YARN registry. Records are created @@ -188,7 +190,8 @@ protected void serviceInit(Configuration conf) throws Exception { * @return the listener port * @throws IOException */ - int initializeZones(Configuration conf) throws IOException { + int initializeZones(Configuration conf) throws IOException, + ValidatorException { int port = conf.getInt(KEY_DNS_PORT, DEFAULT_DNS_PORT); ttl = conf.getTimeDuration(KEY_DNS_TTL, 1L, TimeUnit.SECONDS); RecordCreatorFactory.setTtl(ttl); @@ -269,17 +272,83 @@ public boolean accept( } /** + * Return the number of zones in the map. + * + * @return number of zones in the map + */ + @VisibleForTesting + protected int getZoneCount() { + return zones.size(); + } + + /** * Initializes the reverse lookup zone (mapping IP to name). * * @param conf the Hadoop configuration. - * @throws IOException + * @throws IOException if the DNSSEC key can not be read. + * @throws ValidatorException if the subnet address is invalid. */ private void initializeReverseLookupZone(Configuration conf) - throws IOException { - Name reverseLookupZoneName = getReverseZoneName(conf); - Zone reverseLookupZone = - configureZone(reverseLookupZoneName, conf); - zones.put(reverseLookupZone.getOrigin(), reverseLookupZone); + throws IOException, ValidatorException { + // Determine if the subnet should be split into + // multiple reverse zones, this can be necessary in + // network configurations where the hosts and containers + // are part of the same subnet (i.e. the containers only use + // part of the subnet). + Boolean shouldSplitReverseZone = + conf.getBoolean(KEY_DNS_SPLIT_REVERSE_ZONE, false); + if(shouldSplitReverseZone) { + int subnetCount = getSubnetCountForReverseZones(conf); + addSplitReverseZones(conf, subnetCount); + // Single reverse zone + } else { + Name reverseLookupZoneName = getReverseZoneName(conf); + Zone reverseLookupZone = + configureZone(reverseLookupZoneName, conf); + zones.put(reverseLookupZone.getOrigin(), reverseLookupZone); + } + } + + /** + * When splitting the reverse zone, return the number of subnets needed, + * given the range and netmask. + * + * @param conf the Hadoop configuration. + * @return The number of subnets given the range and netmask. + */ + private int getSubnetCountForReverseZones(Configuration conf) { + String subnet = conf.get(KEY_DNS_ZONE_SUBNET); + String mask = conf.get(KEY_DNS_ZONE_MASK); + String range = conf.get(KEY_DNS_SPLIT_REVERSE_ZONE_RANGE); + SubnetUtils subnetUtils = new SubnetUtils(subnet, mask); + subnetUtils.setInclusiveHostCount(true); + int ipCount = subnetUtils.getInfo().getAddressCount(); + return ipCount / Integer.parseInt(range); + } + + /** + * Create the zones based on the zone count. + * + * @param conf the Hadoop configuration. + * @param subnetCount number of subnets to create reverse zones for. + * @throws IOException if the DNSSEC key can not be read. + * @throws ValidatorException if the subnet address is invalid. + */ + @VisibleForTesting + protected void addSplitReverseZones(Configuration conf, int subnetCount) + throws IOException, ValidatorException { + String subnet = conf.get(KEY_DNS_ZONE_SUBNET); + String range = conf.get(KEY_DNS_SPLIT_REVERSE_ZONE_RANGE); + + // Add the split reverse zones + ReverseZoneUtils reverseZoneUtils = new ReverseZoneUtils(); + for(int idx=0; idx ipPartsOut = new ArrayList<>(); + // First octet + long temp = ipNum + range * (long) index; + ipPartsOut.add(0, temp / pow3); + + // Second octet + temp = temp - ipPartsOut.get(0) * pow3; + ipPartsOut.add(1, temp / pow2); + + // Third octet + temp = temp - ipPartsOut.get(1) * pow2; + ipPartsOut.add(2, temp / pow1); + + // Fourth octet + temp = temp - ipPartsOut.get(2) * pow1; + ipPartsOut.add(3, temp); + + Iterator iter = ipPartsOut.iterator(); + StringBuilder sb = new StringBuilder(); + if(iter.hasNext()) { + sb.append(iter.next()); + while (iter.hasNext()) { + sb.append(".").append(iter.next()); + } + } + return sb.toString(); + + } + + @VisibleForTesting + public String[] splitIp(String baseIp) { + return baseIp.split("\\."); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/ReverseZoneUtilsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/ReverseZoneUtilsTest.java new file mode 100644 index 0000000..729c513 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/ReverseZoneUtilsTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.registry.server.dns; + +import org.apache.commons.validator.ValidatorException; +import static org.junit.Assert.*; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * Tests for the reverse zone utilities. + */ +public class ReverseZoneUtilsTest { + + private ReverseZoneUtils reverseZoneUtils = new ReverseZoneUtils(); + private static final String NET = "172.17.4.0"; + private static final int RANGE = 256; + private static final int INDEX = 0; + + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void testGetReverseZoneNetworkAddress() throws Exception { + assertEquals("172.17.4.0", + reverseZoneUtils.getReverseZoneNetworkAddress(NET, RANGE, INDEX)); + } + + @Test + public void testSplitIp() throws Exception { + String[] splitIp = reverseZoneUtils.splitIp(NET); + assertEquals("172", splitIp[0]); + assertEquals("17", splitIp[1]); + assertEquals("4", splitIp[2]); + assertEquals("0", splitIp[3]); + } + + @Test + public void throwIllegalArgumentExceptionIfIndexIsNegative() throws + Exception { + exception.expect(IllegalArgumentException.class); + reverseZoneUtils.getReverseZoneNetworkAddress(NET, RANGE, -1); + } + + @Test + public void throwValidatorExceptionIfIpIsInvalid() throws Exception { + exception.expect(ValidatorException.class); + reverseZoneUtils.getReverseZoneNetworkAddress( + "213124.21231.14123.13", RANGE, INDEX); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java index 37f0d23..87f9796 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java @@ -55,8 +55,7 @@ import java.util.Date; import java.util.concurrent.TimeUnit; -import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_ZONE_MASK; -import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_ZONE_SUBNET; +import static org.apache.hadoop.registry.client.api.RegistryConstants.*; /** * @@ -541,6 +540,35 @@ public void testReverseZoneNames() throws Exception { assertEquals("wrong name", "26.172.in-addr.arpa.", name.toString()); } + @Test + public void testSplitReverseZoneNames() throws Exception { + Configuration conf = new Configuration(); + registryDNS = new RegistryDNS("TestRegistry"); + conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test"); + conf.set(KEY_DNS_SPLIT_REVERSE_ZONE, "true"); + conf.set(KEY_DNS_SPLIT_REVERSE_ZONE_RANGE, "256"); + conf.set(KEY_DNS_ZONE_SUBNET, "172.26.32.0"); + conf.set(KEY_DNS_ZONE_MASK, "255.255.224.0"); + conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS); + conf.set(RegistryConstants.KEY_DNS_ZONES_DIR, + getClass().getResource("/").getFile()); + if (isSecure()) { + conf.setBoolean(RegistryConstants.KEY_DNSSEC_ENABLED, true); + conf.set(RegistryConstants.KEY_DNSSEC_PUBLIC_KEY, + "AwEAAe1Jev0Az1khlQCvf0nud1/CNHQwwPEu8BNchZthdDxKPVn29yrD " + + "CHoAWjwiGsOSw3SzIPrawSbHzyJsjn0oLBhGrH6QedFGnydoxjNsw3m/ " + + "SCmOjR/a7LGBAMDFKqFioi4gOyuN66svBeY+/5uw72+0ei9AQ20gqf6q " + + "l9Ozs5bV"); + conf.set(RegistryConstants.KEY_DNSSEC_PRIVATE_KEY_FILE, + getClass().getResource("/test.private").getFile()); + } + + registryDNS.setDomainName(conf); + registryDNS.setDNSSECEnabled(conf); + registryDNS.addSplitReverseZones(conf, 4); + assertEquals(4, registryDNS.getZoneCount()); + } + public RegistryDNS getRegistryDNS() { return registryDNS; }