diff --git a/dev-support/byteman/README.md b/dev-support/byteman/README.md new file mode 100644 index 00000000000..9a17fc55be0 --- /dev/null +++ b/dev-support/byteman/README.md @@ -0,0 +1,31 @@ + + +This folder contains example byteman scripts (http://byteman.jboss.org/) to help +Hadoop debuging. + +As the startup script of the hadoop-runner docker image supports byteman +instrumentation it's enough to set the URL of a script to a specific environment +variable to activate it with the docker runs: + + +``` +BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm +``` + +For more info see HADOOP-15656 and HDDS-342 + diff --git a/dev-support/byteman/hadooprpc.btm b/dev-support/byteman/hadooprpc.btm new file mode 100644 index 00000000000..13894fe4ab0 --- /dev/null +++ b/dev-support/byteman/hadooprpc.btm @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This script instruments hadoop rpc layer to print out all the request/response messages to the standard output. +# + +RULE Hadoop RPC request +INTERFACE ^com.google.protobuf.BlockingService +METHOD callBlockingMethod +IF true +DO traceln("--> RPC message request: " + $3.getClass().getSimpleName() + " from " + linked(Thread.currentThread(), "source")); + traceln($3.toString()) +ENDRULE + + +RULE Hadoop RPC response +INTERFACE ^com.google.protobuf.BlockingService +METHOD callBlockingMethod +AT EXIT +IF true +DO traceln("--> RPC message response: " + $3.getClass().getSimpleName() + " to " + unlink(Thread.currentThread(), "source")); + traceln($!.toString()) +ENDRULE + + +RULE Hadoop RPC source IP +CLASS org.apache.hadoop.ipc.Server$RpcCall +METHOD run +IF true +DO link(Thread.currentThread(), "source", $0.connection.toString()) +ENDRULE diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java index 9b7d7ba5d1a..8c92bd04471 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java @@ -575,7 +575,7 @@ private int help(String[] argv) { return 0; } - protected static class UsageInfo { + public static class UsageInfo { public final String args; public final String help; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java index c73e0837721..f2fa3af7d59 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java @@ -30,27 +30,30 @@ */ @InterfaceStability.Evolving public interface FailoverProxyProvider extends Closeable { - public static final class ProxyInfo { - public final T proxy; + static class ProxyInfo { + public T proxy; /* * The information (e.g., the IP address) of the current proxy object. It * provides information for debugging purposes. */ - public final String proxyInfo; + public String proxyInfo; public ProxyInfo(T proxy, String proxyInfo) { this.proxy = proxy; this.proxyInfo = proxyInfo; } + private String proxyName() { + return proxy != null ? proxy.getClass().getSimpleName() : "UnknownProxy"; + } + public String getString(String methodName) { - return proxy.getClass().getSimpleName() + "." + methodName - + " over " + proxyInfo; + return proxyName() + "." + methodName + " over " + proxyInfo; } @Override public String toString() { - return proxy.getClass().getSimpleName() + " over " + proxyInfo; + return proxyName() + " over " + proxyInfo; } } diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md b/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md index 451f9be3073..a21e28be363 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md @@ -35,10 +35,9 @@ Interface Classification ------------------------ Hadoop adopts the following interface classification, -this classification was derived from the -[OpenSolaris taxonomy](http://www.opensolaris.org/os/community/arc/policies/interface-taxonomy/#Advice) -and, to some extent, from taxonomy used inside Yahoo. -Interfaces have two main attributes: Audience and Stability +this classification was derived from the OpenSolaris taxonomy and, to some extent, +from taxonomy used inside Yahoo. +Interfaces have two main attributes: Audience and Stability. ### Audience diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-dist/src/main/compose/ozone/docker-config index 1b75c01cbe9..a1828a3cec8 100644 --- a/hadoop-dist/src/main/compose/ozone/docker-config +++ b/hadoop-dist/src/main/compose/ozone/docker-config @@ -29,3 +29,5 @@ LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. +#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index e337d2fb49c..283488367fc 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -215,9 +215,9 @@ "ozone.scm.db.cache.size.mb"; public static final int OZONE_SCM_DB_CACHE_SIZE_DEFAULT = 128; - public static final String OZONE_SCM_CONTAINER_SIZE_GB = - "ozone.scm.container.size.gb"; - public static final int OZONE_SCM_CONTAINER_SIZE_DEFAULT = 5; + public static final String OZONE_SCM_CONTAINER_SIZE = + "ozone.scm.container.size"; + public static final String OZONE_SCM_CONTAINER_SIZE_DEFAULT = "5GB"; public static final String OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY = "ozone.scm.container.placement.impl"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java index 427c08b9de1..311c1180aae 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java @@ -212,6 +212,7 @@ public void allocate(long size) { public HddsProtos.SCMContainerInfo getProtobuf() { HddsProtos.SCMContainerInfo.Builder builder = HddsProtos.SCMContainerInfo.newBuilder(); + Preconditions.checkState(containerID > 0); return builder.setAllocatedBytes(getAllocatedBytes()) .setContainerID(getContainerID()) .setUsedBytes(getUsedBytes()) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java new file mode 100644 index 00000000000..a817f4f0c81 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.utils.db; + +import org.apache.hadoop.classification.InterfaceStability; + +import java.io.IOException; +import java.util.ArrayList; + +/** + * The DBStore interface provides the ability to create Tables, which store + * a specific type of Key-Value pair. Some DB interfaces like LevelDB will not + * be able to do this. In those case a Table creation will map to a default + * store. + * + */ +@InterfaceStability.Evolving +public interface DBStore extends AutoCloseable { + + /** + * Gets an existing TableStore. + * + * @param name - Name of the TableStore to get + * @return - TableStore. + * @throws IOException on Failure + */ + Table getTable(String name) throws IOException; + + /** + * Lists the Known list of Tables in a DB. + * + * @return List of Tables, in case of Rocks DB and LevelDB we will return at + * least one entry called DEFAULT. + * @throws IOException on Failure + */ + ArrayList listTables() throws IOException; + + /** + * Compact the entire database. + * + * @throws IOException on Failure + */ + void compactDB() throws IOException; + + /** + * Moves a key from the Source Table to the destination Table. + * + * @param key - Key to move. + * @param source - Source Table. + * @param dest - Destination Table. + * @throws IOException on Failure + */ + void move(byte[] key, Table source, Table dest) throws IOException; + + /** + * Moves a key from the Source Table to the destination Table and updates the + * destination to the new value. + * + * @param key - Key to move. + * @param value - new value to write to the destination table. + * @param source - Source Table. + * @param dest - Destination Table. + * @throws IOException on Failure + */ + void move(byte[] key, byte[] value, Table source, Table dest) + throws IOException; + + /** + * Returns an estimated count of keys in this DB. + * + * @return long, estimate of keys in the DB. + */ + long getEstimatedKeyCount() throws IOException; + + +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java new file mode 100644 index 00000000000..c719d31202c --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.utils.db; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.metrics2.util.MBeans; +import org.apache.hadoop.utils.RocksDBStoreMBean; +import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.management.ObjectName; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; + +/** + * RocksDB Store that supports creating Tables in DB. + */ +public class RDBStore implements DBStore { + private static final Logger LOG = + LoggerFactory.getLogger(RDBStore.class); + private final RocksDB db; + private final File dbLocation; + private final WriteOptions writeOptions; + private final DBOptions dbOptions; + private final Hashtable handleTable; + private ObjectName statMBeanName; + + public RDBStore(File dbFile, DBOptions options, List families) + throws IOException { + Preconditions.checkNotNull(dbFile, "DB file location cannot be null"); + Preconditions.checkNotNull(families); + Preconditions.checkArgument(families.size() > 0); + handleTable = new Hashtable<>(); + + final List columnFamilyDescriptors = + new ArrayList<>(); + final List columnFamilyHandles = new ArrayList<>(); + + for (String family : families) { + columnFamilyDescriptors.add( + new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8), + new ColumnFamilyOptions())); + } + + dbOptions = options; + dbLocation = dbFile; + // TODO: Read from the next Config. + writeOptions = new WriteOptions(); + + try { + db = RocksDB.open(dbOptions, dbLocation.getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles); + + for (int x = 0; x < columnFamilyHandles.size(); x++) { + handleTable.put( + DFSUtil.bytes2String(columnFamilyHandles.get(x).getName()), + columnFamilyHandles.get(x)); + } + + if (dbOptions.statistics() != null) { + Map jmxProperties = new HashMap<>(); + jmxProperties.put("dbName", dbFile.getName()); + statMBeanName = MBeans.register("Ozone", "RocksDbStore", jmxProperties, + new RocksDBStoreMBean(dbOptions.statistics())); + if (statMBeanName == null) { + LOG.warn("jmx registration failed during RocksDB init, db path :{}", + dbFile.getAbsolutePath()); + } + } + + } catch (RocksDBException e) { + throw toIOException( + "Failed init RocksDB, db path : " + dbFile.getAbsolutePath(), e); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("RocksDB successfully opened."); + LOG.debug("[Option] dbLocation= {}", dbLocation.getAbsolutePath()); + LOG.debug("[Option] createIfMissing = {}", options.createIfMissing()); + LOG.debug("[Option] maxOpenFiles= {}", options.maxOpenFiles()); + } + } + + public static IOException toIOException(String msg, RocksDBException e) { + String statusCode = e.getStatus() == null ? "N/A" : + e.getStatus().getCodeString(); + String errMessage = e.getMessage() == null ? "Unknown error" : + e.getMessage(); + String output = msg + "; status : " + statusCode + + "; message : " + errMessage; + return new IOException(output, e); + } + + @Override + public void compactDB() throws IOException { + if (db != null) { + try { + db.compactRange(); + } catch (RocksDBException e) { + throw toIOException("Failed to compact db", e); + } + } + } + + @Override + public void close() throws IOException { + + for (final ColumnFamilyHandle handle : handleTable.values()) { + handle.close(); + } + if (dbOptions != null) { + dbOptions.close(); + } + if (writeOptions != null) { + writeOptions.close(); + } + if (statMBeanName != null) { + MBeans.unregister(statMBeanName); + statMBeanName = null; + } + if (db != null) { + db.close(); + } + } + + @Override + public void move(byte[] key, Table source, Table dest) throws IOException { + RDBTable sourceTable; + RDBTable destTable; + if (source instanceof RDBTable) { + sourceTable = (RDBTable) source; + } else { + LOG.error("Unexpected Table type. Expected RocksTable Store for Source."); + throw new IOException("Unexpected TableStore Type in source. Expected " + + "RocksDBTable."); + } + + if (dest instanceof RDBTable) { + destTable = (RDBTable) dest; + } else { + LOG.error("Unexpected Table type. Expected RocksTable Store for Dest."); + throw new IOException("Unexpected TableStore Type in dest. Expected " + + "RocksDBTable."); + } + try (WriteBatch batch = new WriteBatch()) { + byte[] value = sourceTable.get(key); + batch.put(destTable.getHandle(), key, value); + batch.delete(sourceTable.getHandle(), key); + db.write(writeOptions, batch); + } catch (RocksDBException rockdbException) { + LOG.error("Move of key failed. Key:{}", DFSUtil.bytes2String(key)); + throw toIOException("Unable to move key: " + DFSUtil.bytes2String(key), + rockdbException); + } + } + + @Override + public void move(byte[] key, byte[] value, Table source, + Table dest) throws IOException { + RDBTable sourceTable; + RDBTable destTable; + if (source instanceof RDBTable) { + sourceTable = (RDBTable) source; + } else { + LOG.error("Unexpected Table type. Expected RocksTable Store for Source."); + throw new IOException("Unexpected TableStore Type in source. Expected " + + "RocksDBTable."); + } + + if (dest instanceof RDBTable) { + destTable = (RDBTable) dest; + } else { + LOG.error("Unexpected Table type. Expected RocksTable Store for Dest."); + throw new IOException("Unexpected TableStore Type in dest. Expected " + + "RocksDBTable."); + } + try (WriteBatch batch = new WriteBatch()) { + batch.put(destTable.getHandle(), key, value); + batch.delete(sourceTable.getHandle(), key); + db.write(writeOptions, batch); + } catch (RocksDBException rockdbException) { + LOG.error("Move of key failed. Key:{}", DFSUtil.bytes2String(key)); + throw toIOException("Unable to move key: " + DFSUtil.bytes2String(key), + rockdbException); + } + } + + @Override + public long getEstimatedKeyCount() throws IOException { + try { + return Long.parseLong(db.getProperty("rocksdb.estimate-num-keys")); + } catch (RocksDBException e) { + throw toIOException("Unable to get the estimated count.", e); + } + } + + @VisibleForTesting + protected ObjectName getStatMBeanName() { + return statMBeanName; + } + + @Override + public Table getTable(String name) throws IOException { + ColumnFamilyHandle handle = handleTable.get(name); + if (handle == null) { + throw new IOException("No such table in this DB. TableName : " + name); + } + return new RDBTable(this.db, handle, this.writeOptions); + } + + @Override + public ArrayList
listTables() throws IOException { + ArrayList
returnList = new ArrayList<>(); + for (ColumnFamilyHandle handle: handleTable.values()) { + returnList.add(new RDBTable(db, handle, writeOptions)); + } + return returnList; + } +} \ No newline at end of file diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStoreIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStoreIterator.java new file mode 100644 index 00000000000..f1f2df62242 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStoreIterator.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.utils.db; + +import org.apache.hadoop.utils.db.Table.KeyValue; +import org.rocksdb.RocksIterator; + +import java.io.IOException; +import java.util.NoSuchElementException; +import java.util.function.Consumer; + +/** + * RocksDB store iterator. + */ +public class RDBStoreIterator implements TableIterator { + + private RocksIterator rocksDBIterator; + + public RDBStoreIterator(RocksIterator iterator) { + this.rocksDBIterator = iterator; + rocksDBIterator.seekToFirst(); + } + + @Override + public void forEachRemaining(Consumer action) { + while(hasNext()) { + action.accept(next()); + } + } + + @Override + public boolean hasNext() { + return rocksDBIterator.isValid(); + } + + @Override + public Table.KeyValue next() { + if (rocksDBIterator.isValid()) { + KeyValue value = KeyValue.create(rocksDBIterator.key(), rocksDBIterator + .value()); + rocksDBIterator.next(); + return value; + } + throw new NoSuchElementException("RocksDB Store has no more elements"); + } + + @Override + public void seekToFirst() { + rocksDBIterator.seekToFirst(); + } + + @Override + public void seekToLast() { + rocksDBIterator.seekToLast(); + } + + @Override + public KeyValue seek(byte[] key) { + rocksDBIterator.seek(key); + if (rocksDBIterator.isValid()) { + return KeyValue.create(rocksDBIterator.key(), + rocksDBIterator.value()); + } + return null; + } + + @Override + public void close() throws IOException { + rocksDBIterator.close(); + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBTable.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBTable.java new file mode 100644 index 00000000000..8cf6b3533fb --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBTable.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.utils.db; + +import org.apache.hadoop.hdfs.DFSUtil; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ReadOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +/** + * RocksDB implementation of ozone metadata store. + */ +public class RDBTable implements Table { + + private static final Logger LOG = + LoggerFactory.getLogger(RDBTable.class); + + private final RocksDB db; + private final ColumnFamilyHandle handle; + private final WriteOptions writeOptions; + + /** + * Constructs a TableStore. + * + * @param db - DBstore that we are using. + * @param handle - ColumnFamily Handle. + * @param writeOptions - RocksDB write Options. + */ + public RDBTable(RocksDB db, ColumnFamilyHandle handle, + WriteOptions writeOptions) { + this.db = db; + this.handle = handle; + this.writeOptions = writeOptions; + } + + /** + * Converts RocksDB exception to IOE. + * @param msg - Message to add to exception. + * @param e - Original Exception. + * @return IOE. + */ + public static IOException toIOException(String msg, RocksDBException e) { + String statusCode = e.getStatus() == null ? "N/A" : + e.getStatus().getCodeString(); + String errMessage = e.getMessage() == null ? "Unknown error" : + e.getMessage(); + String output = msg + "; status : " + statusCode + + "; message : " + errMessage; + return new IOException(output, e); + } + + /** + * Returns the Column family Handle. + * + * @return ColumnFamilyHandle. + */ + @Override + public ColumnFamilyHandle getHandle() { + return handle; + } + + @Override + public void put(byte[] key, byte[] value) throws IOException { + try { + db.put(handle, writeOptions, key, value); + } catch (RocksDBException e) { + LOG.error("Failed to write to DB. Key: {}", new String(key, + StandardCharsets.UTF_8)); + throw toIOException("Failed to put key-value to metadata " + + "store", e); + } + } + + @Override + public boolean isEmpty() throws IOException { + try (TableIterator keyIter = iterator()) { + keyIter.seekToFirst(); + return !keyIter.hasNext(); + } + } + + @Override + public byte[] get(byte[] key) throws IOException { + try { + return db.get(handle, key); + } catch (RocksDBException e) { + throw toIOException( + "Failed to get the value for the given key", e); + } + } + + @Override + public void delete(byte[] key) throws IOException { + try { + db.delete(handle, key); + } catch (RocksDBException e) { + throw toIOException("Failed to delete the given key", e); + } + } + + @Override + public void writeBatch(WriteBatch operation) throws IOException { + try { + db.write(writeOptions, operation); + } catch (RocksDBException e) { + throw toIOException("Batch write operation failed", e); + } + } + +// @Override +// public void iterate(byte[] from, EntryConsumer consumer) +// throws IOException { +// +// try (RocksIterator it = db.newIterator(handle)) { +// if (from != null) { +// it.seek(from); +// } else { +// it.seekToFirst(); +// } +// while (it.isValid()) { +// if (!consumer.consume(it.key(), it.value())) { +// break; +// } +// it.next(); +// } +// } +// } + + @Override + public TableIterator iterator() { + ReadOptions readOptions = new ReadOptions(); + return new RDBStoreIterator(db.newIterator(handle, readOptions)); + } + + @Override + public String getName() throws IOException { + try { + return DFSUtil.bytes2String(this.getHandle().getName()); + } catch (RocksDBException rdbEx) { + throw toIOException("Unable to get the table name.", rdbEx); + } + } + + @Override + public void close() throws Exception { + // Nothing do for a Column Family. + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Table.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Table.java new file mode 100644 index 00000000000..39425852926 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Table.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.utils.db; + +import org.apache.hadoop.classification.InterfaceStability; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.WriteBatch; + +import java.io.IOException; + +/** + * Interface for key-value store that stores ozone metadata. Ozone metadata is + * stored as key value pairs, both key and value are arbitrary byte arrays. Each + * Table Stores a certain kind of keys and values. This allows a DB to have + * different kind of tables. + */ +@InterfaceStability.Evolving +public interface Table extends AutoCloseable { + + /** + * Puts a key-value pair into the store. + * + * @param key metadata key + * @param value metadata value + */ + void put(byte[] key, byte[] value) throws IOException; + + /** + * @return true if the metadata store is empty. + * @throws IOException on Failure + */ + boolean isEmpty() throws IOException; + + /** + * Returns the value mapped to the given key in byte array or returns null + * if the key is not found. + * + * @param key metadata key + * @return value in byte array or null if the key is not found. + * @throws IOException on Failure + */ + byte[] get(byte[] key) throws IOException; + + /** + * Deletes a key from the metadata store. + * + * @param key metadata key + * @throws IOException on Failure + */ + void delete(byte[] key) throws IOException; + + /** + * Return the Column Family handle. TODO: This leaks an RockDB abstraction + * into Ozone code, cleanup later. + * + * @return ColumnFamilyHandle + */ + ColumnFamilyHandle getHandle(); + + /** + * A batch of PUT, DELETE operations handled as a single atomic write. + * + * @throws IOException write fails + */ + void writeBatch(WriteBatch operation) throws IOException; + + /** + * Returns the iterator for this metadata store. + * + * @return MetaStoreIterator + */ + TableIterator iterator(); + + /** + * Returns the Name of this Table. + * @return - Table Name. + * @throws IOException on failure. + */ + String getName() throws IOException; + + /** + * Class used to represent the key and value pair of a db entry. + */ + class KeyValue { + + private final byte[] key; + private final byte[] value; + + /** + * KeyValue Constructor, used to represent a key and value of a db entry. + * + * @param key - Key Bytes + * @param value - Value bytes + */ + private KeyValue(byte[] key, byte[] value) { + this.key = key; + this.value = value; + } + + /** + * Create a KeyValue pair. + * + * @param key - Key Bytes + * @param value - Value bytes + * @return KeyValue object. + */ + public static KeyValue create(byte[] key, byte[] value) { + return new KeyValue(key, value); + } + + /** + * Return key. + * + * @return byte[] + */ + public byte[] getKey() { + byte[] result = new byte[key.length]; + System.arraycopy(key, 0, result, 0, key.length); + return result; + } + + /** + * Return value. + * + * @return byte[] + */ + public byte[] getValue() { + byte[] result = new byte[value.length]; + System.arraycopy(value, 0, result, 0, value.length); + return result; + } + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TableIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TableIterator.java new file mode 100644 index 00000000000..83a8f3c8385 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TableIterator.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.utils.db; + +import java.io.Closeable; +import java.util.Iterator; + +/** + * Iterator for MetaDataStore DB. + * + * @param + */ +public interface TableIterator extends Iterator, Closeable { + + /** + * seek to first entry. + */ + void seekToFirst(); + + /** + * seek to last entry. + */ + void seekToLast(); + + /** + * Seek to the specific key. + * + * @param key - Bytes that represent the key. + * @return T. + */ + T seek(byte[] key); + +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/package-info.java new file mode 100644 index 00000000000..17d676d9283 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +/** + * Database interfaces for Ozone. + */ +package org.apache.hadoop.utils.db; \ No newline at end of file diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 568e38d040e..37a845e0aff 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -611,12 +611,11 @@ - ozone.scm.container.size.gb - 5 + ozone.scm.container.size + 5GB OZONE, PERFORMANCE, MANAGEMENT - Default container size used by Ozone. This value is specified - in GB. + Default container size used by Ozone. There are two considerations while picking this number. The speed at which a container can be replicated, determined by the network speed and the metadata that each container generates. So selecting a large number diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java new file mode 100644 index 00000000000..94a650b9597 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.utils.db; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.hdfs.DFSUtil; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.DBOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.Statistics; +import org.rocksdb.StatsLevel; + +import javax.management.MBeanServer; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * RDBStore Tests. + */ +public class TestRDBStore { + private final List families = + Arrays.asList(DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY), + "First", "Second", "Third", + "Fourth", "Fifth", + "Sixth"); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + @Rule + public ExpectedException thrown = ExpectedException.none(); + private RDBStore rdbStore = null; + private DBOptions options = null; + + @Before + public void setUp() throws Exception { + options = new DBOptions(); + options.setCreateIfMissing(true); + options.setCreateMissingColumnFamilies(true); + + Statistics statistics = new Statistics(); + statistics.setStatsLevel(StatsLevel.ALL); + options = options.setStatistics(statistics); + rdbStore = new RDBStore(folder.newFolder(), options, families); + } + + @After + public void tearDown() throws Exception { + if (rdbStore != null) { + rdbStore.close(); + } + } + + @Test + public void compactDB() throws Exception { + try (RDBStore newStore = + new RDBStore(folder.newFolder(), options, families)) { + Assert.assertNotNull("DB Store cannot be null", newStore); + try (Table firstTable = newStore.getTable(families.get(1))) { + Assert.assertNotNull("Table cannot be null", firstTable); + for (int x = 0; x < 100; x++) { + byte[] key = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + byte[] value = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + firstTable.put(key, value); + } + } + // This test does not assert anything if there is any error this test + // will throw and fail. + newStore.compactDB(); + } + } + + @Test + public void close() throws Exception { + RDBStore newStore = + new RDBStore(folder.newFolder(), options, families); + Assert.assertNotNull("DBStore cannot be null", newStore); + // This test does not assert anything if there is any error this test + // will throw and fail. + newStore.close(); + } + + @Test + public void moveKey() throws Exception { + byte[] key = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + byte[] value = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + + try (Table firstTable = rdbStore.getTable(families.get(1))) { + firstTable.put(key, value); + try (Table secondTable = rdbStore.getTable(families.get(2))) { + rdbStore.move(key, firstTable, secondTable); + byte[] newvalue = secondTable.get(key); + // Make sure we have value in the second table + Assert.assertNotNull(newvalue); + //and it is same as what we wrote to the FirstTable + Assert.assertArrayEquals(value, newvalue); + } + // After move this key must not exist in the first table. + Assert.assertNull(firstTable.get(key)); + } + } + + @Test + public void moveWithValue() throws Exception { + byte[] key = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + byte[] value = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + + byte[] nextValue = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + try (Table firstTable = rdbStore.getTable(families.get(1))) { + firstTable.put(key, value); + try (Table secondTable = rdbStore.getTable(families.get(2))) { + rdbStore.move(key, nextValue, firstTable, secondTable); + byte[] newvalue = secondTable.get(key); + // Make sure we have value in the second table + Assert.assertNotNull(newvalue); + //and it is not same as what we wrote to the FirstTable, and equals + // the new value. + Assert.assertArrayEquals(nextValue, nextValue); + } + } + + } + + @Test + public void getEstimatedKeyCount() throws Exception { + try (RDBStore newStore = + new RDBStore(folder.newFolder(), options, families)) { + Assert.assertNotNull("DB Store cannot be null", newStore); + // Write 100 keys to the first table. + try (Table firstTable = newStore.getTable(families.get(1))) { + Assert.assertNotNull("Table cannot be null", firstTable); + for (int x = 0; x < 100; x++) { + byte[] key = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + byte[] value = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + firstTable.put(key, value); + } + } + + // Write 100 keys to the secondTable table. + try (Table secondTable = newStore.getTable(families.get(2))) { + Assert.assertNotNull("Table cannot be null", secondTable); + for (int x = 0; x < 100; x++) { + byte[] key = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + byte[] value = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + secondTable.put(key, value); + } + } + // Let us make sure that our estimate is not off by 10% + Assert.assertTrue(newStore.getEstimatedKeyCount() > 180 + || newStore.getEstimatedKeyCount() < 220); + } + } + + @Test + public void getStatMBeanName() throws Exception { + + try (Table firstTable = rdbStore.getTable(families.get(1))) { + for (int y = 0; y < 100; y++) { + byte[] key = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + byte[] value = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + firstTable.put(key, value); + } + } + MBeanServer platformMBeanServer = + ManagementFactory.getPlatformMBeanServer(); + Thread.sleep(2000); + + Object keysWritten = platformMBeanServer + .getAttribute(rdbStore.getStatMBeanName(), "NUMBER_KEYS_WRITTEN"); + + Assert.assertTrue(((Long) keysWritten) >= 99L); + + Object dbWriteAverage = platformMBeanServer + .getAttribute(rdbStore.getStatMBeanName(), "DB_WRITE_AVERAGE"); + Assert.assertTrue((double) dbWriteAverage > 0); + } + + @Test + public void getTable() throws Exception { + for (String tableName : families) { + try (Table table = rdbStore.getTable(tableName)) { + Assert.assertNotNull(tableName + "is null", table); + } + } + thrown.expect(IOException.class); + rdbStore.getTable("ATableWithNoName"); + } + + @Test + public void listTables() throws Exception { + List
tableList = rdbStore.listTables(); + Assert.assertNotNull("Table list cannot be null", tableList); + Map hashTable = new HashMap<>(); + + for (Table t : tableList) { + hashTable.put(t.getName(), t); + } + + int count = families.size(); + // Assert that we have all the tables in the list and no more. + for (String name : families) { + Assert.assertTrue(hashTable.containsKey(name)); + count--; + } + Assert.assertEquals(0, count); + } +} \ No newline at end of file diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java new file mode 100644 index 00000000000..677a1f95d45 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.utils.db; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.hdfs.DFSUtil; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.DBOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.Statistics; +import org.rocksdb.StatsLevel; +import org.rocksdb.WriteBatch; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +/** + * Tests for RocksDBTable Store. + */ +public class TestRDBTableStore { + private static int count = 0; + private final List families = + Arrays.asList(DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY), + "First", "Second", "Third", + "Fourth", "Fifth", + "Sixth"); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + private RDBStore rdbStore = null; + private DBOptions options = null; + + @Before + public void setUp() throws Exception { + options = new DBOptions(); + options.setCreateIfMissing(true); + options.setCreateMissingColumnFamilies(true); + + Statistics statistics = new Statistics(); + statistics.setStatsLevel(StatsLevel.ALL); + options = options.setStatistics(statistics); + rdbStore = new RDBStore(folder.newFolder(), options, families); + } + + @After + public void tearDown() throws Exception { + if (rdbStore != null) { + rdbStore.close(); + } + } + + @Test + public void toIOException() { + } + + @Test + public void getHandle() throws Exception { + try (Table testTable = rdbStore.getTable("First")) { + Assert.assertNotNull(testTable); + Assert.assertNotNull(testTable.getHandle()); + } + } + + @Test + public void putGetAndEmpty() throws Exception { + try (Table testTable = rdbStore.getTable("First")) { + byte[] key = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + byte[] value = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + testTable.put(key, value); + Assert.assertFalse(testTable.isEmpty()); + byte[] readValue = testTable.get(key); + Assert.assertArrayEquals(value, readValue); + } + try (Table secondTable = rdbStore.getTable("Second")) { + Assert.assertTrue(secondTable.isEmpty()); + } + } + + @Test + public void delete() throws Exception { + List deletedKeys = new LinkedList<>(); + List validKeys = new LinkedList<>(); + byte[] value = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + for (int x = 0; x < 100; x++) { + deletedKeys.add( + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8)); + } + + for (int x = 0; x < 100; x++) { + validKeys.add( + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8)); + } + + // Write all the keys and delete the keys scheduled for delete. + //Assert we find only expected keys in the Table. + try (Table testTable = rdbStore.getTable("Fourth")) { + for (int x = 0; x < deletedKeys.size(); x++) { + testTable.put(deletedKeys.get(x), value); + testTable.delete(deletedKeys.get(x)); + } + + for (int x = 0; x < validKeys.size(); x++) { + testTable.put(validKeys.get(x), value); + } + + for (int x = 0; x < validKeys.size(); x++) { + Assert.assertNotNull(testTable.get(validKeys.get(0))); + } + + for (int x = 0; x < deletedKeys.size(); x++) { + Assert.assertNull(testTable.get(deletedKeys.get(0))); + } + } + } + + @Test + public void writeBatch() throws Exception { + WriteBatch batch = new WriteBatch(); + try (Table testTable = rdbStore.getTable("Fifth")) { + byte[] key = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + byte[] value = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + batch.put(testTable.getHandle(), key, value); + testTable.writeBatch(batch); + Assert.assertNotNull(testTable.get(key)); + } + batch.close(); + } + + private static boolean consume(Table.KeyValue keyValue) { + count++; + Assert.assertNotNull(keyValue.getKey()); + return true; + } + + @Test + public void forEachAndIterator() throws Exception { + final int iterCount = 100; + try (Table testTable = rdbStore.getTable("Sixth")) { + for (int x = 0; x < iterCount; x++) { + byte[] key = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + byte[] value = + RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); + testTable.put(key, value); + } + int localCount = 0; + try (TableIterator iter = testTable.iterator()) { + while (iter.hasNext()) { + Table.KeyValue keyValue = iter.next(); + localCount++; + } + + Assert.assertEquals(iterCount, localCount); + iter.seekToFirst(); + iter.forEachRemaining(TestRDBTableStore::consume); + Assert.assertEquals(iterCount, count); + + } + } + } +} \ No newline at end of file diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/package-info.java new file mode 100644 index 00000000000..f06855e038a --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +/** + * Tests for the DB Utilities. + */ +package org.apache.hadoop.utils.db; \ No newline at end of file diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java index b2427549cf6..a4c1f2f4678 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java @@ -20,7 +20,6 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerInfo; -import static java.lang.Math.max; /** * Container Report iterates the closed containers and sends a container report @@ -37,7 +36,6 @@ private long readBytes; private long writeBytes; private long containerID; - private long deleteTransactionId; public long getContainerID() { return containerID; @@ -47,9 +45,6 @@ public void setContainerID(long containerID) { this.containerID = containerID; } - - - /** * Constructs the ContainerReport. * @@ -66,7 +61,6 @@ public ContainerReport(long containerID, String finalhash) { this.readBytes = 0L; this.writeCount = 0L; this.writeBytes = 0L; - this.deleteTransactionId = 0; } /** @@ -100,9 +94,6 @@ public static ContainerReport getFromProtoBuf(ContainerInfo info) { if (info.hasWriteBytes()) { report.setWriteBytes(info.getWriteBytes()); } - if (info.hasDeleteTransactionId()) { - report.updateDeleteTransactionId(info.getDeleteTransactionId()); - } report.setContainerID(info.getContainerID()); return report; @@ -193,10 +184,6 @@ public void setBytesUsed(long bytesUsed) { this.bytesUsed = bytesUsed; } - public void updateDeleteTransactionId(long transactionId) { - this.deleteTransactionId = max(transactionId, deleteTransactionId); - } - /** * Gets a containerInfo protobuf message from ContainerReports. * @@ -213,7 +200,6 @@ public ContainerInfo getProtoBufMessage() { .setWriteBytes(this.getWriteBytes()) .setFinalhash(this.getFinalhash()) .setContainerID(this.getContainerID()) - .setDeleteTransactionId(this.deleteTransactionId) .build(); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyValueContainerReport.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyValueContainerReport.java new file mode 100644 index 00000000000..b03487b5beb --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyValueContainerReport.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.container.common.helpers; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerInfo; + +import static java.lang.Math.max; + +/** + * KeyValueContainer Report iterates the closed containers and sends a + * container report to SCM. + */ +public class KeyValueContainerReport extends ContainerReport{ + private long deleteTransactionId; + + /** + * Constructs the KeyValueContainerReport. + * + * @param containerID - Container ID. + * @param finalhash - Final Hash. + */ + public KeyValueContainerReport(long containerID, String finalhash) { + super(containerID, finalhash); + this.deleteTransactionId = 0; + } + + /** + * Sets the deleteTransactionId if it is greater than existing. + * @param transactionId - deleteTransactionId + */ + public void updateDeleteTransactionId(long transactionId) { + this.deleteTransactionId = max(transactionId, deleteTransactionId); + } + + /** + * Gets the deleteTransactionId. + * @return - deleteTransactionId. + */ + public long getDeleteTransactionId() { + return this.deleteTransactionId; + } + + /** + * Gets a containerReport from protobuf class. + * + * @param info - ContainerInfo. + * @return - ContainerReport. + */ + public static KeyValueContainerReport getFromProtoBuf(ContainerInfo info) { + Preconditions.checkNotNull(info); + KeyValueContainerReport report = new KeyValueContainerReport( + info.getContainerID(), info.getFinalhash()); + if (info.hasSize()) { + report.setSize(info.getSize()); + } + if (info.hasKeyCount()) { + report.setKeyCount(info.getKeyCount()); + } + if (info.hasUsed()) { + report.setBytesUsed(info.getUsed()); + } + if (info.hasReadCount()) { + report.setReadCount(info.getReadCount()); + } + if (info.hasReadBytes()) { + report.setReadBytes(info.getReadBytes()); + } + if (info.hasWriteCount()) { + report.setWriteCount(info.getWriteCount()); + } + if (info.hasWriteBytes()) { + report.setWriteBytes(info.getWriteBytes()); + } + if (info.hasDeleteTransactionId()) { + report.updateDeleteTransactionId(info.getDeleteTransactionId()); + } + report.setContainerID(info.getContainerID()); + return report; + } + + /** + * Gets a containerInfo protobuf message from ContainerReports. + * + * @return ContainerInfo + */ + @Override + public ContainerInfo getProtoBufMessage() { + return ContainerInfo.newBuilder() + .setKeyCount(this.getKeyCount()) + .setSize(this.getSize()) + .setUsed(this.getBytesUsed()) + .setReadCount(this.getReadCount()) + .setReadBytes(this.getReadBytes()) + .setWriteCount(this.getWriteCount()) + .setWriteBytes(this.getWriteBytes()) + .setFinalhash(this.getFinalhash()) + .setContainerID(this.getContainerID()) + .setDeleteTransactionId(this.getDeleteTransactionId()) + .build(); + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java index 26954a7029b..afd140781a6 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java @@ -33,11 +33,9 @@ import java.util.Collections; import java.util.Map; import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.yaml.snakeyaml.Yaml; -import static java.lang.Math.max; import static org.apache.hadoop.ozone.OzoneConsts.CHECKSUM; import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_ID; import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_TYPE; @@ -81,8 +79,6 @@ private HddsVolume volume; - private long deleteTransactionId; - private String checksum; public static final Charset CHARSET_ENCODING = Charset.forName("UTF-8"); private static final String DUMMY_CHECKSUM = new String(new byte[64], @@ -99,12 +95,6 @@ MAX_SIZE_GB, CHECKSUM)); - - /** - * Number of pending deletion blocks in container. - */ - private final AtomicInteger numPendingDeletionBlocks; - /** * Creates a ContainerData Object, which holds metadata of the container. * @param type - ContainerType @@ -121,7 +111,7 @@ protected ContainerData(ContainerType type, long containerId, int size) { * @param type - ContainerType * @param containerId - ContainerId * @param layOutVersion - Container layOutVersion - * @param size - Container maximum size + * @param size - Container maximum size in GB */ protected ContainerData(ContainerType type, long containerId, int layOutVersion, int size) { @@ -139,8 +129,6 @@ protected ContainerData(ContainerType type, long containerId, this.bytesUsed = new AtomicLong(0L); this.keyCount = new AtomicLong(0L); this.maxSizeGB = size; - this.numPendingDeletionBlocks = new AtomicInteger(0); - this.deleteTransactionId = 0; setChecksumTo0ByteArray(); } @@ -403,31 +391,6 @@ public void setKeyCount(long count) { this.keyCount.set(count); } - /** - * Increase the count of pending deletion blocks. - * - * @param numBlocks increment number - */ - public void incrPendingDeletionBlocks(int numBlocks) { - this.numPendingDeletionBlocks.addAndGet(numBlocks); - } - - /** - * Decrease the count of pending deletion blocks. - * - * @param numBlocks decrement number - */ - public void decrPendingDeletionBlocks(int numBlocks) { - this.numPendingDeletionBlocks.addAndGet(-1 * numBlocks); - } - - /** - * Get the number of pending deletion blocks. - */ - public int getNumPendingDeletionBlocks() { - return this.numPendingDeletionBlocks.get(); - } - public void setChecksumTo0ByteArray() { this.checksum = DUMMY_CHECKSUM; } @@ -469,20 +432,4 @@ public void computeAndSetChecksum(Yaml yaml) throws IOException { * @return Protocol Buffer Message */ public abstract ContainerProtos.ContainerData getProtoBufMessage(); - - /** - * Sets deleteTransactionId to latest delete transactionId for the container. - * - * @param transactionId latest transactionId of the container. - */ - public void updateDeleteTransactionId(long transactionId) { - deleteTransactionId = max(transactionId, deleteTransactionId); - } - - /** - * Return the latest deleteTransactionId of the container. - */ - public long getDeleteTransactionId() { - return deleteTransactionId; - } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java index aed75d3ee2e..ec6d642a095 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java @@ -18,31 +18,34 @@ package org.apache.hadoop.ozone.container.common.impl; -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerType; -import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; - -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; - import java.beans.IntrospectionException; -import java.io.IOException; -import java.io.InputStream; -import java.io.Writer; +import java.io.ByteArrayInputStream; +import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.io.OutputStreamWriter; -import java.io.File; +import java.io.Writer; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.Map; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerType; +import org.apache.hadoop.hdds.scm.container.common.helpers + .StorageContainerException; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; + +import com.google.common.base.Preconditions; +import static org.apache.hadoop.ozone.container.keyvalue + .KeyValueContainerData.KEYVALUE_YAML_TAG; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.AbstractConstruct; import org.yaml.snakeyaml.constructor.Constructor; import org.yaml.snakeyaml.introspector.BeanAccess; @@ -54,9 +57,6 @@ import org.yaml.snakeyaml.nodes.Tag; import org.yaml.snakeyaml.representer.Representer; -import static org.apache.hadoop.ozone.container.keyvalue - .KeyValueContainerData.KEYVALUE_YAML_TAG; - /** * Class for creating and reading .container files. */ @@ -106,36 +106,52 @@ public static void createContainerFile(ContainerType containerType, /** * Read the yaml file, and return containerData. * - * @param containerFile * @throws IOException */ public static ContainerData readContainerFile(File containerFile) throws IOException { Preconditions.checkNotNull(containerFile, "containerFile cannot be null"); + try (FileInputStream inputFileStream = new FileInputStream(containerFile)) { + return readContainer(inputFileStream); + } + + } + + /** + * Read the yaml file content, and return containerData. + * + * @throws IOException + */ + public static ContainerData readContainer(byte[] containerFileContent) + throws IOException { + return readContainer( + new ByteArrayInputStream(containerFileContent)); + } + + /** + * Read the yaml content, and return containerData. + * + * @throws IOException + */ + public static ContainerData readContainer(InputStream input) + throws IOException { - InputStream input = null; ContainerData containerData; - try { - PropertyUtils propertyUtils = new PropertyUtils(); - propertyUtils.setBeanAccess(BeanAccess.FIELD); - propertyUtils.setAllowReadOnlyProperties(true); + PropertyUtils propertyUtils = new PropertyUtils(); + propertyUtils.setBeanAccess(BeanAccess.FIELD); + propertyUtils.setAllowReadOnlyProperties(true); - Representer representer = new ContainerDataRepresenter(); - representer.setPropertyUtils(propertyUtils); + Representer representer = new ContainerDataRepresenter(); + representer.setPropertyUtils(propertyUtils); - Constructor containerDataConstructor = new ContainerDataConstructor(); + Constructor containerDataConstructor = new ContainerDataConstructor(); - Yaml yaml = new Yaml(containerDataConstructor, representer); - yaml.setBeanAccess(BeanAccess.FIELD); + Yaml yaml = new Yaml(containerDataConstructor, representer); + yaml.setBeanAccess(BeanAccess.FIELD); + + containerData = (ContainerData) + yaml.load(input); - input = new FileInputStream(containerFile); - containerData = (ContainerData) - yaml.load(input); - } finally { - if (input!= null) { - input.close(); - } - } return containerData; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java index 3da09f21c4d..f92ab5221f8 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java @@ -22,9 +22,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerInfo; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; import org.apache.hadoop.hdds.scm.container.common.helpers @@ -43,8 +40,6 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.stream.Collectors; -import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .Result.INVALID_CONTAINER_STATE; /** * Class that manages Containers created on the datanode. @@ -204,58 +199,19 @@ public ContainerReportsProto getContainerReport() throws IOException { ContainerReportsProto.Builder crBuilder = ContainerReportsProto.newBuilder(); - for (Container container: containers) { - long containerId = container.getContainerData().getContainerID(); - ContainerInfo.Builder ciBuilder = ContainerInfo.newBuilder(); - ContainerData containerData = container.getContainerData(); - ciBuilder.setContainerID(containerId) - .setReadCount(containerData.getReadCount()) - .setWriteCount(containerData.getWriteCount()) - .setReadBytes(containerData.getReadBytes()) - .setWriteBytes(containerData.getWriteBytes()) - .setUsed(containerData.getBytesUsed()) - .setState(getState(containerData)) - .setDeleteTransactionId(containerData.getDeleteTransactionId()); - - crBuilder.addReports(ciBuilder.build()); + crBuilder.addReports(container.getContainerReport()); } return crBuilder.build(); } - /** - * Returns LifeCycle State of the container. - * @param containerData - ContainerData - * @return LifeCycle State of the container - * @throws StorageContainerException - */ - private HddsProtos.LifeCycleState getState(ContainerData containerData) - throws StorageContainerException { - HddsProtos.LifeCycleState state; - switch (containerData.getState()) { - case OPEN: - state = HddsProtos.LifeCycleState.OPEN; - break; - case CLOSING: - state = HddsProtos.LifeCycleState.CLOSING; - break; - case CLOSED: - state = HddsProtos.LifeCycleState.CLOSED; - break; - default: - throw new StorageContainerException("Invalid Container state found: " + - containerData.getContainerID(), INVALID_CONTAINER_STATE); - } - return state; - } - public List chooseContainerForBlockDeletion(int count, ContainerDeletionChoosingPolicy deletionPolicy) throws StorageContainerException { Map containerDataMap = containerMap.entrySet().stream() - .filter(e -> e.getValue().getContainerType() - == ContainerProtos.ContainerType.KeyValueContainer) + .filter(e -> deletionPolicy.isValidContainerType( + e.getValue().getContainerType())) .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getContainerData())); return deletionPolicy diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java index 83d746bf75a..5c6c319600e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.ozone.container.common.interfaces .ContainerDeletionChoosingPolicy; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,7 +59,7 @@ LOG.debug("Select container {} for block deletion, " + "pending deletion blocks num: {}.", entry.getContainerID(), - entry.getNumPendingDeletionBlocks()); + ((KeyValueContainerData)entry).getNumPendingDeletionBlocks()); } else { break; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java index 68074fc37ca..b17680c4198 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java @@ -22,6 +22,7 @@ .StorageContainerException; import org.apache.hadoop.ozone.container.common.interfaces .ContainerDeletionChoosingPolicy; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,14 +42,11 @@ LoggerFactory.getLogger(TopNOrderedContainerDeletionChoosingPolicy.class); /** customized comparator used to compare differentiate container data. **/ - private static final Comparator CONTAINER_DATA_COMPARATOR - = new Comparator() { - @Override - public int compare(ContainerData c1, ContainerData c2) { - return Integer.compare(c2.getNumPendingDeletionBlocks(), - c1.getNumPendingDeletionBlocks()); - } - }; + private static final Comparator + KEY_VALUE_CONTAINER_DATA_COMPARATOR = (KeyValueContainerData c1, + KeyValueContainerData c2) -> + Integer.compare(c2.getNumPendingDeletionBlocks(), + c1.getNumPendingDeletionBlocks()); @Override public List chooseContainerForBlockDeletion(int count, @@ -58,13 +56,15 @@ public int compare(ContainerData c1, ContainerData c2) { "Internal assertion: candidate containers cannot be null"); List result = new LinkedList<>(); - List orderedList = new LinkedList<>(); - orderedList.addAll(candidateContainers.values()); - Collections.sort(orderedList, CONTAINER_DATA_COMPARATOR); + List orderedList = new LinkedList<>(); + for (ContainerData entry : candidateContainers.values()) { + orderedList.add((KeyValueContainerData)entry); + } + Collections.sort(orderedList, KEY_VALUE_CONTAINER_DATA_COMPARATOR); // get top N list ordered by pending deletion blocks' number int currentCount = 0; - for (ContainerData entry : orderedList) { + for (KeyValueContainerData entry : orderedList) { if (currentCount < count) { if (entry.getNumPendingDeletionBlocks() > 0) { result.add(entry); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java index a7077d99e56..9380f0cb36f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java @@ -18,25 +18,27 @@ package org.apache.hadoop.ozone.container.common.interfaces; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Map; + import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerLifeCycleState; -import org.apache.hadoop.hdds.scm.container.common.helpers. - StorageContainerException; +import org.apache.hadoop.hdds.scm.container.common.helpers + .StorageContainerException; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdfs.util.RwLock; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import java.io.File; -import java.io.IOException; -import java.util.Map; - - /** * Interface for Container Operations. */ -public interface Container extends RwLock { +public interface Container extends RwLock { /** * Creates a container. @@ -70,7 +72,7 @@ void update(Map metaData, boolean forceUpdate) * @return ContainerData - Container Data. * @throws StorageContainerException */ - ContainerData getContainerData(); + CONTAINERDATA getContainerData(); /** * Get the Container Lifecycle state. @@ -111,4 +113,23 @@ void update(Map metaData, boolean forceUpdate) */ BlockIterator blockIterator() throws IOException; + /** + * Import the container from an external archive. + */ + void importContainerData(InputStream stream, + ContainerPacker packer) throws IOException; + + /** + * Export all the data of the container to one output archive with the help + * of the packer. + * + */ + void exportContainerData(OutputStream stream, + ContainerPacker packer) throws IOException; + + /** + * Returns containerReport for the container. + */ + StorageContainerDatanodeProtocolProtos.ContainerInfo getContainerReport() + throws StorageContainerException; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java index dce86e9375e..84c4f903f37 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.container.common.interfaces; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; import org.apache.hadoop.ozone.container.common.impl.ContainerData; @@ -42,4 +43,16 @@ List chooseContainerForBlockDeletion(int count, Map candidateContainers) throws StorageContainerException; + + /** + * Determine if the container has suitable type for this policy. + * @param type type of the container + * @return whether the container type suitable for this policy. + */ + default boolean isValidContainerType(ContainerProtos.ContainerType type) { + if (type == ContainerProtos.ContainerType.KeyValueContainer) { + return true; + } + return false; + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerPacker.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerPacker.java new file mode 100644 index 00000000000..8308c23866b --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerPacker.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.container.common.interfaces; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.hadoop.ozone.container.common.impl.ContainerData; + +/** + * Service to pack/unpack ContainerData container data to/from a single byte + * stream. + */ +public interface ContainerPacker { + + /** + * Extract the container data to the path defined by the container. + *

+ * This doesn't contain the extraction of the container descriptor file. + * + * @return the byte content of the descriptor (which won't be written to a + * file but returned). + */ + byte[] unpackContainerData(Container container, + InputStream inputStream) + throws IOException; + + /** + * Compress all the container data (chunk data, metadata db AND container + * descriptor) to one single archive. + */ + void pack(Container container, OutputStream destination) + throws IOException; + + /** + * Read the descriptor from the finished archive to get the data before + * importing the container. + */ + byte[] unpackContainerDescriptor(InputStream inputStream) + throws IOException; +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index cf6f1cab41c..0a23912ff2c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -48,8 +48,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import static org.apache.hadoop.hdds.scm.HddsServerUtil.getScmHeartbeatInterval; - /** * State Machine Class. */ @@ -60,7 +58,6 @@ private final ExecutorService executorService; private final Configuration conf; private final SCMConnectionManager connectionManager; - private final long heartbeatFrequency; private StateContext context; private final OzoneContainer container; private DatanodeDetails datanodeDetails; @@ -86,7 +83,6 @@ public DatanodeStateMachine(DatanodeDetails datanodeDetails, .setNameFormat("Datanode State Machine Thread - %d").build()); connectionManager = new SCMConnectionManager(conf); context = new StateContext(this.conf, DatanodeStates.getInitState(), this); - heartbeatFrequency = getScmHeartbeatInterval(conf); container = new OzoneContainer(this.datanodeDetails, new OzoneConfiguration(conf), context); nextHB = new AtomicLong(Time.monotonicNow()); @@ -147,6 +143,7 @@ private void start() throws IOException { while (context.getState() != DatanodeStates.SHUTDOWN) { try { LOG.debug("Executing cycle Number : {}", context.getExecutionCount()); + long heartbeatFrequency = context.getHeartbeatFrequency(); nextHB.set(Time.monotonicNow() + heartbeatFrequency); context.execute(executorService, heartbeatFrequency, TimeUnit.MILLISECONDS); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java index 19c949680c0..a342294bd1d 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java @@ -33,6 +33,8 @@ import org.apache.hadoop.ozone.protocol.commands.CommandStatus .CommandStatusBuilder; import org.apache.hadoop.ozone.protocol.commands.SCMCommand; + +import static org.apache.hadoop.hdds.scm.HddsServerUtil.getScmHeartbeatInterval; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,6 +68,13 @@ private final Queue containerActions; private DatanodeStateMachine.DatanodeStates state; + /** + * Starting with a 2 sec heartbeat frequency which will be updated to the + * real HB frequency after scm registration. With this method the + * initial registration could be significant faster. + */ + private AtomicLong heartbeatFrequency = new AtomicLong(2000); + /** * Constructs a StateContext. * @@ -398,4 +407,15 @@ public boolean updateCommandStatus(Long cmdId, boolean cmdExecuted) { } return false; } + + public void configureHeartbeatFrequency(){ + heartbeatFrequency.set(getScmHeartbeatInterval(conf)); + } + + /** + * Return current heartbeat frequency in ms. + */ + public long getHeartbeatFrequency() { + return heartbeatFrequency.get(); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java index 1758c03fef6..ec2358ae18b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java @@ -101,6 +101,7 @@ public void execute(ExecutorService executor) { return RegisterEndpointTask.newBuilder() .setConfig(conf) .setEndpointStateMachine(endpoint) + .setContext(context) .setDatanodeDetails(context.getParent().getDatanodeDetails()) .setOzoneContainer(context.getParent().getContainer()) .build(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java index 25af4a1d1e9..ccab0956e72 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java @@ -29,6 +29,7 @@ .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMRegisteredResponseProto; +import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,6 +51,7 @@ private Future result; private DatanodeDetails datanodeDetails; private final OzoneContainer datanodeContainerManager; + private StateContext stateContext; /** * Creates a register endpoint task. @@ -60,10 +62,12 @@ */ @VisibleForTesting public RegisterEndpointTask(EndpointStateMachine rpcEndPoint, - Configuration conf, OzoneContainer ozoneContainer) { + Configuration conf, OzoneContainer ozoneContainer, + StateContext context) { this.rpcEndPoint = rpcEndPoint; this.conf = conf; this.datanodeContainerManager = ozoneContainer; + this.stateContext = context; } @@ -124,6 +128,7 @@ public void setDatanodeDetails( rpcEndPoint.getState().getNextState(); rpcEndPoint.setState(nextState); rpcEndPoint.zeroMissedCount(); + this.stateContext.configureHeartbeatFrequency(); } catch (IOException ex) { rpcEndPoint.logIfNeeded(ex); } finally { @@ -150,6 +155,7 @@ public static Builder newBuilder() { private Configuration conf; private DatanodeDetails datanodeDetails; private OzoneContainer container; + private StateContext context; /** * Constructs the builder class. @@ -200,6 +206,10 @@ public Builder setOzoneContainer(OzoneContainer ozoneContainer) { return this; } + public Builder setContext(StateContext stateContext) { + this.context = stateContext; + return this; + } public RegisterEndpointTask build() { if (endPointStateMachine == null) { @@ -210,8 +220,9 @@ public RegisterEndpointTask build() { if (conf == null) { LOG.error("No config specified."); - throw new IllegalArgumentException("A valid configration is needed to" + - " construct RegisterEndpoint task"); + throw new IllegalArgumentException( + "A valid configuration is needed to construct RegisterEndpoint " + + "task"); } if (datanodeDetails == null) { @@ -223,13 +234,20 @@ public RegisterEndpointTask build() { if (container == null) { LOG.error("Container is not specified"); throw new IllegalArgumentException("Container is not specified to " + - "constrict RegisterEndpoint task"); + "construct RegisterEndpoint task"); + } + + if (context == null) { + LOG.error("StateContext is not specified"); + throw new IllegalArgumentException("Container is not specified to " + + "construct RegisterEndpoint task"); } RegisterEndpointTask task = new RegisterEndpointTask(this - .endPointStateMachine, this.conf, this.container); + .endPointStateMachine, this.conf, this.container, this.context); task.setDatanodeDetails(datanodeDetails); return task; } + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java index c96f99766af..8108a119ec4 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java @@ -18,9 +18,15 @@ package org.apache.hadoop.ozone.container.keyvalue; -import com.google.common.base.Preconditions; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.StandardCopyOption; +import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileUtil; @@ -28,6 +34,8 @@ .ContainerLifeCycleState; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerType; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; import org.apache.hadoop.io.nativeio.NativeIO; @@ -35,31 +43,26 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml; -import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.common.interfaces.Container; +import org.apache.hadoop.ozone.container.common.interfaces.ContainerPacker; import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy; +import org.apache.hadoop.ozone.container.common.volume.HddsVolume; +import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils; import org.apache.hadoop.ozone.container.keyvalue.helpers .KeyValueContainerLocationUtil; import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; import org.apache.hadoop.utils.MetadataStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; - -import java.util.Map; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import com.google.common.base.Preconditions; +import org.apache.commons.io.FileUtils; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .Result.CONTAINER_ALREADY_EXISTS; -import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .Result.CONTAINER_INTERNAL_ERROR; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .Result.CONTAINER_FILES_CREATE_ERROR; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.CONTAINER_INTERNAL_ERROR; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .Result.DISK_OUT_OF_SPACE; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos @@ -68,11 +71,13 @@ .Result.INVALID_CONTAINER_STATE; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .Result.UNSUPPORTED_REQUEST; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Class to perform KeyValue Container operations. */ -public class KeyValueContainer implements Container { +public class KeyValueContainer implements Container { private static final Logger LOG = LoggerFactory.getLogger(Container.class); @@ -164,6 +169,34 @@ public void create(VolumeSet volumeSet, VolumeChoosingPolicy } } + /** + * Set all of the path realted container data fields based on the name + * conventions. + * + * @param scmId + * @param containerVolume + * @param hddsVolumeDir + */ + public void populatePathFields(String scmId, + HddsVolume containerVolume, String hddsVolumeDir) { + + long containerId = containerData.getContainerID(); + + File containerMetaDataPath = KeyValueContainerLocationUtil + .getContainerMetaDataPath(hddsVolumeDir, scmId, containerId); + + File chunksPath = KeyValueContainerLocationUtil.getChunksLocationPath( + hddsVolumeDir, scmId, containerId); + File dbFile = KeyValueContainerLocationUtil.getContainerDBFile( + containerMetaDataPath, containerId); + + //Set containerData for the KeyValueContainer. + containerData.setMetadataPath(containerMetaDataPath.getPath()); + containerData.setChunksPath(chunksPath.getPath()); + containerData.setDbFile(dbFile); + containerData.setVolume(containerVolume); + } + /** * Writes to .container file. * @@ -332,6 +365,75 @@ public KeyValueBlockIterator blockIterator() throws IOException{ containerData.getContainerPath())); } + @Override + public void importContainerData(InputStream input, + ContainerPacker packer) throws IOException { + writeLock(); + try { + if (getContainerFile().exists()) { + String errorMessage = String.format( + "Can't import container (cid=%d) data to a specific location" + + " as the container descriptor (%s) has already been exist.", + getContainerData().getContainerID(), + getContainerFile().getAbsolutePath()); + throw new IOException(errorMessage); + } + //copy the values from the input stream to the final destination + // directory. + byte[] descriptorContent = packer.unpackContainerData(this, input); + + Preconditions.checkNotNull(descriptorContent, + "Container descriptor is missing from the container archive: " + + getContainerData().getContainerID()); + + //now, we have extracted the container descriptor from the previous + //datanode. We can load it and upload it with the current data + // (original metadata + current filepath fields) + KeyValueContainerData originalContainerData = + (KeyValueContainerData) ContainerDataYaml + .readContainer(descriptorContent); + + + containerData.setState(originalContainerData.getState()); + containerData + .setContainerDBType(originalContainerData.getContainerDBType()); + containerData.setBytesUsed(originalContainerData.getBytesUsed()); + + //rewriting the yaml file with new checksum calculation. + update(originalContainerData.getMetadata(), true); + + //fill in memory stat counter (keycount, byte usage) + KeyValueContainerUtil.parseKVContainerData(containerData, config); + + } catch (Exception ex) { + //delete all the temporary data in case of any exception. + try { + FileUtils.deleteDirectory(new File(containerData.getMetadataPath())); + FileUtils.deleteDirectory(new File(containerData.getChunksPath())); + FileUtils.deleteDirectory(getContainerFile()); + } catch (Exception deleteex) { + LOG.error( + "Can not cleanup destination directories after a container import" + + " error (cid" + + containerData.getContainerID() + ")", deleteex); + } + throw ex; + } finally { + writeUnlock(); + } + } + + @Override + public void exportContainerData(OutputStream destination, + ContainerPacker packer) throws IOException { + if (getContainerData().getState() != ContainerLifeCycleState.CLOSED) { + throw new IllegalStateException( + "Only closed containers could be exported: ContainerId=" + + getContainerData().getContainerID()); + } + packer.pack(this, destination); + } + /** * Acquire read lock. */ @@ -405,6 +507,50 @@ public File getContainerFile() { .getContainerID() + OzoneConsts.CONTAINER_EXTENSION); } + /** + * Returns KeyValueContainerReport for the KeyValueContainer. + */ + @Override + public StorageContainerDatanodeProtocolProtos.ContainerInfo + getContainerReport() throws StorageContainerException{ + StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder = + StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder(); + ciBuilder.setContainerID(containerData.getContainerID()) + .setReadCount(containerData.getReadCount()) + .setWriteCount(containerData.getWriteCount()) + .setReadBytes(containerData.getReadBytes()) + .setWriteBytes(containerData.getWriteBytes()) + .setUsed(containerData.getBytesUsed()) + .setState(getHddsState()) + .setDeleteTransactionId(containerData.getDeleteTransactionId()); + return ciBuilder.build(); + } + + /** + * Returns LifeCycle State of the container. + * @return LifeCycle State of the container in HddsProtos format + * @throws StorageContainerException + */ + private HddsProtos.LifeCycleState getHddsState() + throws StorageContainerException { + HddsProtos.LifeCycleState state; + switch (containerData.getState()) { + case OPEN: + state = HddsProtos.LifeCycleState.OPEN; + break; + case CLOSING: + state = HddsProtos.LifeCycleState.CLOSING; + break; + case CLOSED: + state = HddsProtos.LifeCycleState.CLOSED; + break; + default: + throw new StorageContainerException("Invalid Container state found: " + + containerData.getContainerID(), INVALID_CONTAINER_STATE); + } + return state; + } + /** * Returns container DB file. * @return diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java index 0705cf480aa..e4cb5f35dee 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java @@ -21,6 +21,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import java.util.Collections; + +import org.apache.hadoop.conf.StorageSize; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -32,7 +34,9 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import static java.lang.Math.max; import static org.apache.hadoop.ozone.OzoneConsts.CHUNKS_PATH; import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE; import static org.apache.hadoop.ozone.OzoneConsts.METADATA_PATH; @@ -61,6 +65,13 @@ private File dbFile = null; + /** + * Number of pending deletion blocks in KeyValueContainer. + */ + private final AtomicInteger numPendingDeletionBlocks; + + private long deleteTransactionId; + static { // Initialize YAML fields KV_YAML_FIELDS = Lists.newArrayList(); @@ -73,21 +84,25 @@ /** * Constructs KeyValueContainerData object. * @param id - ContainerId - * @param size - maximum size of the container + * @param size - maximum size in GB of the container */ public KeyValueContainerData(long id, int size) { super(ContainerProtos.ContainerType.KeyValueContainer, id, size); + this.numPendingDeletionBlocks = new AtomicInteger(0); + this.deleteTransactionId = 0; } /** * Constructs KeyValueContainerData object. * @param id - ContainerId * @param layOutVersion - * @param size - maximum size of the container + * @param size - maximum size in GB of the container */ public KeyValueContainerData(long id, int layOutVersion, int size) { super(ContainerProtos.ContainerType.KeyValueContainer, id, layOutVersion, size); + this.numPendingDeletionBlocks = new AtomicInteger(0); + this.deleteTransactionId = 0; } @@ -168,6 +183,47 @@ public void setContainerDBType(String containerDBType) { this.containerDBType = containerDBType; } + /** + * Increase the count of pending deletion blocks. + * + * @param numBlocks increment number + */ + public void incrPendingDeletionBlocks(int numBlocks) { + this.numPendingDeletionBlocks.addAndGet(numBlocks); + } + + /** + * Decrease the count of pending deletion blocks. + * + * @param numBlocks decrement number + */ + public void decrPendingDeletionBlocks(int numBlocks) { + this.numPendingDeletionBlocks.addAndGet(-1 * numBlocks); + } + + /** + * Get the number of pending deletion blocks. + */ + public int getNumPendingDeletionBlocks() { + return this.numPendingDeletionBlocks.get(); + } + + /** + * Sets deleteTransactionId to latest delete transactionId for the container. + * + * @param transactionId latest transactionId of the container. + */ + public void updateDeleteTransactionId(long transactionId) { + deleteTransactionId = max(transactionId, deleteTransactionId); + } + + /** + * Return the latest deleteTransactionId of the container. + */ + public long getDeleteTransactionId() { + return deleteTransactionId; + } + /** * Returns a ProtoBuf Message from ContainerData. * @@ -212,9 +268,11 @@ public void setContainerDBType(String containerDBType) { public static KeyValueContainerData getFromProtoBuf( ContainerProtos.ContainerData protoData) throws IOException { // TODO: Add containerMaxSize to ContainerProtos.ContainerData + StorageSize storageSize = StorageSize.parse( + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT); KeyValueContainerData data = new KeyValueContainerData( protoData.getContainerID(), - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT); + (int)storageSize.getUnit().toBytes(storageSize.getValue())); for (int x = 0; x < protoData.getMetadataCount(); x++) { data.addMetadata(protoData.getMetadata(x).getKey(), protoData.getMetadata(x).getValue()); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java index 9ddb4749c8b..84095617424 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.protobuf.ByteString; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos @@ -47,6 +48,7 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.OpenContainerBlockMap; import org.apache.hadoop.ozone.container.common.interfaces.Container; +import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil; import org.apache.hadoop.ozone.container.keyvalue.helpers.SmallFileUtils; import org.apache.hadoop.ozone.container.common.helpers.KeyData; @@ -148,9 +150,9 @@ public KeyValueHandler(Configuration config, ContainerSet contSet, volumeChoosingPolicy = ReflectionUtils.newInstance(conf.getClass( HDDS_DATANODE_VOLUME_CHOOSING_POLICY, RoundRobinVolumeChoosingPolicy .class, VolumeChoosingPolicy.class), conf); - maxContainerSizeGB = config.getInt(ScmConfigKeys - .OZONE_SCM_CONTAINER_SIZE_GB, ScmConfigKeys - .OZONE_SCM_CONTAINER_SIZE_DEFAULT); + maxContainerSizeGB = (int)config.getStorageSize( + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.GB); // this handler lock is used for synchronizing createContainer Requests, // so using a fair lock here. handlerLock = new AutoCloseableLock(new ReentrantLock(true)); @@ -162,7 +164,8 @@ public VolumeChoosingPolicy getVolumeChoosingPolicyForTesting() { return volumeChoosingPolicy; } /** - * Returns OpenContainerBlockMap instance + * Returns OpenContainerBlockMap instance. + * * @return OpenContainerBlockMap */ public OpenContainerBlockMap getOpenContainerBlockMap() { @@ -269,6 +272,19 @@ ContainerCommandResponseProto handleCreateContainer( return ContainerUtils.getSuccessResponse(request); } + public void populateContainerPathFields(KeyValueContainer container, + long maxSize) throws IOException { + volumeSet.acquireLock(); + try { + HddsVolume containerVolume = volumeChoosingPolicy.chooseVolume(volumeSet + .getVolumesList(), maxSize); + String hddsVolumeDir = containerVolume.getHddsRootDir().toString(); + container.populatePathFields(scmID, containerVolume, hddsVolumeDir); + } finally { + volumeSet.releaseLock(); + } + } + /** * Handles Read Container Request. Returns the ContainerData as response. */ @@ -322,7 +338,7 @@ ContainerCommandResponseProto handleUpdateContainer( * Open containers cannot be deleted. * Holds writeLock on ContainerSet till the container is removed from * containerMap. On disk deletion of container files will happen - * asynchornously without the lock. + * asynchronously without the lock. */ ContainerCommandResponseProto handleDeleteContainer( ContainerCommandRequestProto request, KeyValueContainer kvContainer) { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/TarContainerPacker.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/TarContainerPacker.java new file mode 100644 index 00000000000..13689a705ce --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/TarContainerPacker.java @@ -0,0 +1,249 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.container.keyvalue; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Collectors; + +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.container.common.interfaces.Container; +import org.apache.hadoop.ozone.container.common.interfaces.ContainerPacker; + +import com.google.common.base.Preconditions; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.compressors.CompressorOutputStream; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.io.IOUtils; + +/** + * Compress/uncompress KeyValueContainer data to a tar.gz archive. + */ +public class TarContainerPacker + implements ContainerPacker { + + private static final String CHUNKS_DIR_NAME = OzoneConsts.STORAGE_DIR_CHUNKS; + + private static final String DB_DIR_NAME = "db"; + + private static final String CONTAINER_FILE_NAME = "container.yaml"; + + + + /** + * Given an input stream (tar file) extract the data to the specified + * directories. + * + * @param container container which defines the destination structure. + * @param inputStream the input stream. + * @throws IOException + */ + @Override + public byte[] unpackContainerData(Container container, + InputStream inputStream) + throws IOException { + byte[] descriptorFileContent = null; + try { + KeyValueContainerData containerData = container.getContainerData(); + CompressorInputStream compressorInputStream = + new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.GZIP, + inputStream); + + TarArchiveInputStream tarInput = + new TarArchiveInputStream(compressorInputStream); + + TarArchiveEntry entry = tarInput.getNextTarEntry(); + while (entry != null) { + String name = entry.getName(); + if (name.startsWith(DB_DIR_NAME + "/")) { + Path destinationPath = containerData.getDbFile().toPath() + .resolve(name.substring(DB_DIR_NAME.length() + 1)); + extractEntry(tarInput, entry.getSize(), destinationPath); + } else if (name.startsWith(CHUNKS_DIR_NAME + "/")) { + Path destinationPath = Paths.get(containerData.getChunksPath()) + .resolve(name.substring(CHUNKS_DIR_NAME.length() + 1)); + extractEntry(tarInput, entry.getSize(), destinationPath); + } else if (name.equals(CONTAINER_FILE_NAME)) { + //Don't do anything. Container file should be unpacked in a + //separated step by unpackContainerDescriptor call. + descriptorFileContent = readEntry(tarInput, entry); + } else { + throw new IllegalArgumentException( + "Unknown entry in the tar file: " + "" + name); + } + entry = tarInput.getNextTarEntry(); + } + return descriptorFileContent; + + } catch (CompressorException e) { + throw new IOException( + "Can't uncompress the given container: " + container + .getContainerData().getContainerID(), + e); + } + } + + private void extractEntry(TarArchiveInputStream tarInput, long size, + Path path) throws IOException { + Preconditions.checkNotNull(path, "Path element should not be null"); + Path parent = Preconditions.checkNotNull(path.getParent(), + "Path element should have a parent directory"); + Files.createDirectories(parent); + try (BufferedOutputStream bos = new BufferedOutputStream( + new FileOutputStream(path.toAbsolutePath().toString()))) { + int bufferSize = 1024; + byte[] buffer = new byte[bufferSize + 1]; + long remaining = size; + while (remaining > 0) { + int read = + tarInput.read(buffer, 0, (int) Math.min(remaining, bufferSize)); + if (read >= 0) { + remaining -= read; + bos.write(buffer, 0, read); + } else { + remaining = 0; + } + } + } + + } + + /** + * Given a containerData include all the required container data/metadata + * in a tar file. + * + * @param container Container to archive (data + metadata). + * @param destination Destination tar file/stream. + * @throws IOException + */ + @Override + public void pack(Container container, + OutputStream destination) + throws IOException { + + KeyValueContainerData containerData = container.getContainerData(); + + try (CompressorOutputStream gzippedOut = new CompressorStreamFactory() + .createCompressorOutputStream(CompressorStreamFactory.GZIP, + destination)) { + + try (ArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream( + gzippedOut)) { + + includePath(containerData.getDbFile().toString(), DB_DIR_NAME, + archiveOutputStream); + + includePath(containerData.getChunksPath(), CHUNKS_DIR_NAME, + archiveOutputStream); + + includeFile(container.getContainerFile(), + CONTAINER_FILE_NAME, + archiveOutputStream); + } + } catch (CompressorException e) { + throw new IOException( + "Can't compress the container: " + containerData.getContainerID(), + e); + } + + } + + @Override + public byte[] unpackContainerDescriptor(InputStream inputStream) + throws IOException { + try { + CompressorInputStream compressorInputStream = + new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.GZIP, + inputStream); + + TarArchiveInputStream tarInput = + new TarArchiveInputStream(compressorInputStream); + + TarArchiveEntry entry = tarInput.getNextTarEntry(); + while (entry != null) { + String name = entry.getName(); + if (name.equals(CONTAINER_FILE_NAME)) { + return readEntry(tarInput, entry); + } + entry = tarInput.getNextTarEntry(); + } + + } catch (CompressorException e) { + throw new IOException( + "Can't read the container descriptor from the container archive", + e); + } + throw new IOException( + "Container descriptor is missing from the container archive."); + } + + private byte[] readEntry(TarArchiveInputStream tarInput, + TarArchiveEntry entry) throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + int bufferSize = 1024; + byte[] buffer = new byte[bufferSize + 1]; + long remaining = entry.getSize(); + while (remaining > 0) { + int read = + tarInput.read(buffer, 0, (int) Math.min(remaining, bufferSize)); + remaining -= read; + bos.write(buffer, 0, read); + } + return bos.toByteArray(); + } + + private void includePath(String containerPath, String subdir, + ArchiveOutputStream archiveOutputStream) throws IOException { + + for (Path path : Files.list(Paths.get(containerPath)) + .collect(Collectors.toList())) { + + includeFile(path.toFile(), subdir + "/" + path.getFileName(), + archiveOutputStream); + } + } + + private void includeFile(File file, String entryName, + ArchiveOutputStream archiveOutputStream) throws IOException { + ArchiveEntry archiveEntry = + archiveOutputStream.createArchiveEntry(file, entryName); + archiveOutputStream.putArchiveEntry(archiveEntry); + try (FileInputStream fis = new FileInputStream(file)) { + IOUtils.copy(fis, archiveOutputStream); + } + archiveOutputStream.closeArchiveEntry(); + } + +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java index 2352cf64465..ed4536f6248 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java @@ -17,10 +17,14 @@ */ package org.apache.hadoop.ozone.container.keyvalue.helpers; -import com.google.common.base.Preconditions; -import org.apache.commons.io.FileUtils; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; + import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; @@ -32,16 +36,12 @@ import org.apache.hadoop.utils.MetadataKeyFilters; import org.apache.hadoop.utils.MetadataStore; import org.apache.hadoop.utils.MetadataStoreBuilder; + +import com.google.common.base.Preconditions; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; -import java.util.Map; - /** * Class which defines utility methods for KeyValueContainer. */ @@ -157,7 +157,7 @@ public static ContainerCommandResponseProto getReadContainerResponse( * @throws IOException */ public static void parseKVContainerData(KeyValueContainerData kvContainerData, - OzoneConfiguration config) throws IOException { + Configuration config) throws IOException { long containerID = kvContainerData.getContainerID(); File metadataPath = new File(kvContainerData.getMetadataPath()); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java index 42db66d6426..12ce163ba19 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java @@ -57,6 +57,7 @@ public void testKeyValueData() { assertEquals(val.get(), kvData.getReadCount()); assertEquals(val.get(), kvData.getWriteCount()); assertEquals(val.get(), kvData.getKeyCount()); + assertEquals(val.get(), kvData.getNumPendingDeletionBlocks()); assertEquals(MAXSIZE, kvData.getMaxSizeGB()); kvData.setState(state); @@ -68,6 +69,7 @@ public void testKeyValueData() { kvData.incrReadCount(); kvData.incrWriteCount(); kvData.incrKeyCount(); + kvData.incrPendingDeletionBlocks(1); assertEquals(state, kvData.getState()); assertEquals(containerDBType, kvData.getContainerDBType()); @@ -79,7 +81,7 @@ public void testKeyValueData() { assertEquals(1, kvData.getReadCount()); assertEquals(1, kvData.getWriteCount()); assertEquals(1, kvData.getKeyCount()); - + assertEquals(1, kvData.getNumPendingDeletionBlocks()); } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java index 6ff2eca5569..7359868ceeb 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java @@ -23,7 +23,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; - +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerLifeCycleState; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; @@ -37,6 +38,8 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.DiskChecker; import org.apache.hadoop.utils.MetadataStore; + +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -46,6 +49,8 @@ import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -74,7 +79,6 @@ private String scmId = UUID.randomUUID().toString(); private VolumeSet volumeSet; private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy; - private long containerID = 1L; private KeyValueContainerData keyValueContainerData; private KeyValueContainer keyValueContainer; @@ -141,13 +145,14 @@ private void addBlocks(int count) throws Exception { } + @SuppressWarnings("RedundantCast") @Test public void testCreateContainer() throws Exception { // Create Container. keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - keyValueContainerData = (KeyValueContainerData) keyValueContainer + keyValueContainerData = keyValueContainer .getContainerData(); String containerMetaDataPath = keyValueContainerData @@ -166,6 +171,86 @@ public void testCreateContainer() throws Exception { "DB does not exist"); } + @Test + public void testContainerImportExport() throws Exception { + + long containerId = keyValueContainer.getContainerData().getContainerID(); + // Create Container. + keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); + + + keyValueContainerData = keyValueContainer + .getContainerData(); + + keyValueContainerData.setState(ContainerLifeCycleState.CLOSED); + + int numberOfKeysToWrite = 12; + //write one few keys to check the key count after import + MetadataStore metadataStore = KeyUtils.getDB(keyValueContainerData, conf); + for (int i = 0; i < numberOfKeysToWrite; i++) { + metadataStore.put(("test" + i).getBytes(), "test".getBytes()); + } + metadataStore.close(); + + Map metadata = new HashMap<>(); + metadata.put("key1", "value1"); + keyValueContainer.update(metadata, true); + + //destination path + File folderToExport = folder.newFile("exported.tar.gz"); + + TarContainerPacker packer = new TarContainerPacker(); + + //export the container + try (FileOutputStream fos = new FileOutputStream(folderToExport)) { + keyValueContainer + .exportContainerData(fos, packer); + } + + //delete the original one + keyValueContainer.delete(true); + + //create a new one + KeyValueContainerData containerData = + new KeyValueContainerData(containerId, 1, + keyValueContainerData.getMaxSizeGB()); + KeyValueContainer container = new KeyValueContainer(containerData, conf); + + HddsVolume containerVolume = volumeChoosingPolicy.chooseVolume(volumeSet + .getVolumesList(), 1); + String hddsVolumeDir = containerVolume.getHddsRootDir().toString(); + + container.populatePathFields(scmId, containerVolume, hddsVolumeDir); + try (FileInputStream fis = new FileInputStream(folderToExport)) { + container.importContainerData(fis, packer); + } + + Assert.assertEquals("value1", containerData.getMetadata().get("key1")); + Assert.assertEquals(keyValueContainerData.getContainerDBType(), + containerData.getContainerDBType()); + Assert.assertEquals(keyValueContainerData.getState(), + containerData.getState()); + Assert.assertEquals(numberOfKeysToWrite, + containerData.getKeyCount()); + Assert.assertEquals(keyValueContainerData.getLayOutVersion(), + containerData.getLayOutVersion()); + Assert.assertEquals(keyValueContainerData.getMaxSizeGB(), + containerData.getMaxSizeGB()); + Assert.assertEquals(keyValueContainerData.getBytesUsed(), + containerData.getBytesUsed()); + + //Can't overwrite existing container + try { + try (FileInputStream fis = new FileInputStream(folderToExport)) { + container.importContainerData(fis, packer); + } + fail("Container is imported twice. Previous files are overwritten"); + } catch (Exception ex) { + //all good + } + + } + @Test public void testDuplicateContainer() throws Exception { try { @@ -224,7 +309,7 @@ public void testCloseContainer() throws Exception { keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); keyValueContainer.close(); - keyValueContainerData = (KeyValueContainerData) keyValueContainer + keyValueContainerData = keyValueContainer .getContainerData(); assertEquals(ContainerProtos.ContainerLifeCycleState.CLOSED, @@ -249,7 +334,7 @@ public void testUpdateContainer() throws IOException { metadata.put("OWNER", "hdfs"); keyValueContainer.update(metadata, true); - keyValueContainerData = (KeyValueContainerData) keyValueContainer + keyValueContainerData = keyValueContainer .getContainerData(); assertEquals(2, keyValueContainerData.getMetadata().size()); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java new file mode 100644 index 00000000000..a599f721696 --- /dev/null +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.container.keyvalue; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.container.common.interfaces.ContainerPacker; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Test the tar/untar for a given container. + */ +public class TestTarContainerPacker { + + private static final String TEST_DB_FILE_NAME = "test1"; + + private static final String TEST_DB_FILE_CONTENT = "test1"; + + private static final String TEST_CHUNK_FILE_NAME = "chunk1"; + + private static final String TEST_CHUNK_FILE_CONTENT = "This is a chunk"; + + private static final String TEST_DESCRIPTOR_FILE_CONTENT = "descriptor"; + + private ContainerPacker packer = new TarContainerPacker(); + + private static final Path SOURCE_CONTAINER_ROOT = + Paths.get("target/test/data/packer-source-dir"); + + private static final Path DEST_CONTAINER_ROOT = + Paths.get("target/test/data/packer-dest-dir"); + + @BeforeClass + public static void init() throws IOException { + initDir(SOURCE_CONTAINER_ROOT); + initDir(DEST_CONTAINER_ROOT); + } + + private static void initDir(Path path) throws IOException { + if (path.toFile().exists()) { + FileUtils.deleteDirectory(path.toFile()); + } + path.toFile().mkdirs(); + } + + private KeyValueContainerData createContainer(long id, Path dir, + OzoneConfiguration conf) throws IOException { + + Path containerDir = dir.resolve("container" + id); + Path dbDir = containerDir.resolve("db"); + Path dataDir = containerDir.resolve("data"); + Files.createDirectories(dbDir); + Files.createDirectories(dataDir); + + KeyValueContainerData containerData = new KeyValueContainerData(id, -1); + containerData.setChunksPath(dataDir.toString()); + containerData.setMetadataPath(dbDir.getParent().toString()); + containerData.setDbFile(dbDir.toFile()); + + + return containerData; + } + + @Test + public void pack() throws IOException, CompressorException { + + //GIVEN + OzoneConfiguration conf = new OzoneConfiguration(); + + KeyValueContainerData sourceContainerData = + createContainer(1L, SOURCE_CONTAINER_ROOT, conf); + + KeyValueContainer sourceContainer = + new KeyValueContainer(sourceContainerData, conf); + + //sample db file in the metadata directory + try (FileWriter writer = new FileWriter( + sourceContainerData.getDbFile().toPath() + .resolve(TEST_DB_FILE_NAME) + .toFile())) { + IOUtils.write(TEST_DB_FILE_CONTENT, writer); + } + + //sample chunk file in the chunk directory + try (FileWriter writer = new FileWriter( + Paths.get(sourceContainerData.getChunksPath()) + .resolve(TEST_CHUNK_FILE_NAME) + .toFile())) { + IOUtils.write(TEST_CHUNK_FILE_CONTENT, writer); + } + + //sample container descriptor file + try (FileWriter writer = new FileWriter( + sourceContainer.getContainerFile())) { + IOUtils.write(TEST_DESCRIPTOR_FILE_CONTENT, writer); + } + + Path targetFile = + SOURCE_CONTAINER_ROOT.getParent().resolve("container.tar.gz"); + + //WHEN: pack it + try (FileOutputStream output = new FileOutputStream(targetFile.toFile())) { + packer.pack(sourceContainer, output); + } + + //THEN: check the result + try (FileInputStream input = new FileInputStream(targetFile.toFile())) { + CompressorInputStream uncompressed = new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.GZIP, input); + TarArchiveInputStream tarStream = new TarArchiveInputStream(uncompressed); + + TarArchiveEntry entry; + Map entries = new HashMap<>(); + while ((entry = tarStream.getNextTarEntry()) != null) { + entries.put(entry.getName(), entry); + } + + Assert.assertTrue( + entries.containsKey("container.yaml")); + + } + + //read the container descriptor only + try (FileInputStream input = new FileInputStream(targetFile.toFile())) { + String containerYaml = new String(packer.unpackContainerDescriptor(input), + Charset.forName(StandardCharsets.UTF_8.name())); + Assert.assertEquals(TEST_DESCRIPTOR_FILE_CONTENT, containerYaml); + } + + KeyValueContainerData destinationContainerData = + createContainer(2L, DEST_CONTAINER_ROOT, conf); + + KeyValueContainer destinationContainer = + new KeyValueContainer(destinationContainerData, conf); + + String descriptor = ""; + + //unpackContainerData + try (FileInputStream input = new FileInputStream(targetFile.toFile())) { + descriptor = + new String(packer.unpackContainerData(destinationContainer, input), + Charset.forName(StandardCharsets.UTF_8.name())); + } + + assertExampleMetadataDbIsGood( + destinationContainerData.getDbFile().toPath()); + assertExampleChunkFileIsGood( + Paths.get(destinationContainerData.getChunksPath())); + Assert.assertFalse( + "Descriptor file should not been exctarcted by the " + + "unpackContainerData Call", + destinationContainer.getContainerFile().exists()); + Assert.assertEquals(TEST_DESCRIPTOR_FILE_CONTENT, descriptor); + + } + + + private void assertExampleMetadataDbIsGood(Path dbPath) + throws IOException { + + Path dbFile = dbPath.resolve(TEST_DB_FILE_NAME); + + Assert.assertTrue( + "example DB file is missing after pack/unpackContainerData: " + dbFile, + Files.exists(dbFile)); + + try (FileInputStream testFile = new FileInputStream(dbFile.toFile())) { + List strings = IOUtils + .readLines(testFile, Charset.forName(StandardCharsets.UTF_8.name())); + Assert.assertEquals(1, strings.size()); + Assert.assertEquals(TEST_DB_FILE_CONTENT, strings.get(0)); + } + } + + private void assertExampleChunkFileIsGood(Path chunkDirPath) + throws IOException { + + Path chunkFile = chunkDirPath.resolve(TEST_CHUNK_FILE_NAME); + + Assert.assertTrue( + "example chunk file is missing after pack/unpackContainerData: " + + chunkFile, + Files.exists(chunkFile)); + + try (FileInputStream testFile = new FileInputStream(chunkFile.toFile())) { + List strings = IOUtils + .readLines(testFile, Charset.forName(StandardCharsets.UTF_8.name())); + Assert.assertEquals(1, strings.size()); + Assert.assertEquals(TEST_CHUNK_FILE_CONTENT, strings.get(0)); + } + } + +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java index f3a111fa574..ca2a6a0f746 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java @@ -18,6 +18,7 @@ import java.util.UUID; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.Mapping; import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; @@ -30,7 +31,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.metrics2.util.MBeans; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; @@ -98,9 +98,10 @@ public BlockManagerImpl(final Configuration conf, this.nodeManager = nodeManager; this.containerManager = containerManager; - this.containerSize = OzoneConsts.GB * conf.getInt( - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB, - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT); + this.containerSize = (long)conf.getStorageSize( + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, + StorageUnit.BYTES); this.containerProvisionBatchSize = conf.getInt( diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java index 11863f29bd5..b000bfd0b22 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java @@ -20,6 +20,7 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.SCMContainerInfo; @@ -66,7 +67,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_CONTAINER_SIZE_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_CONTAINER_SIZE_GB; + .OZONE_SCM_CONTAINER_SIZE; import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes .FAILED_TO_CHANGE_CONTAINER_STATE; import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath; @@ -129,9 +130,8 @@ public ContainerMapping( this.lock = new ReentrantLock(); - // To be replaced with code getStorageSize once it is committed. - size = conf.getLong(OZONE_SCM_CONTAINER_SIZE_GB, - OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024; + size = (long)conf.getStorageSize(OZONE_SCM_CONTAINER_SIZE, + OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); this.containerStateManager = new ContainerStateManager(conf, this); LOG.trace("Container State Manager created."); @@ -505,15 +505,26 @@ public ContainerWithPipeline getMatchingContainerWithPipeline(final long size, */ @Override public void processContainerReports(DatanodeDetails datanodeDetails, - ContainerReportsProto reports) + ContainerReportsProto reports, boolean isRegisterCall) throws IOException { List containerInfos = reports.getReportsList(); PendingDeleteStatusList pendingDeleteStatusList = new PendingDeleteStatusList(datanodeDetails); - for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState : + for (StorageContainerDatanodeProtocolProtos.ContainerInfo contInfo : containerInfos) { - byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID()); + // Update replica info during registration process. + if (isRegisterCall) { + try { + getStateManager().addContainerReplica(ContainerID. + valueof(contInfo.getContainerID()), datanodeDetails); + } catch (Exception ex) { + // Continue to next one after logging the error. + LOG.error("Error while adding replica for containerId {}.", + contInfo.getContainerID(), ex); + } + } + byte[] dbKey = Longs.toByteArray(contInfo.getContainerID()); lock.lock(); try { byte[] containerBytes = containerStore.get(dbKey); @@ -522,12 +533,12 @@ public void processContainerReports(DatanodeDetails datanodeDetails, HddsProtos.SCMContainerInfo.PARSER.parseFrom(containerBytes); HddsProtos.SCMContainerInfo newState = - reconcileState(datanodeState, knownState, datanodeDetails); + reconcileState(contInfo, knownState, datanodeDetails); - if (knownState.getDeleteTransactionId() > datanodeState + if (knownState.getDeleteTransactionId() > contInfo .getDeleteTransactionId()) { pendingDeleteStatusList - .addPendingDeleteStatus(datanodeState.getDeleteTransactionId(), + .addPendingDeleteStatus(contInfo.getDeleteTransactionId(), knownState.getDeleteTransactionId(), knownState.getContainerID()); } @@ -558,7 +569,7 @@ public void processContainerReports(DatanodeDetails datanodeDetails, LOG.error("Error while processing container report from datanode :" + " {}, for container: {}, reason: container doesn't exist in" + "container database.", datanodeDetails, - datanodeState.getContainerID()); + contInfo.getContainerID()); } } finally { lock.unlock(); @@ -733,21 +744,7 @@ public void flushContainerInfo() throws IOException { // return info of a deleted container. may revisit this in the future, // for now, just skip a not-found container if (containerBytes != null) { - HddsProtos.SCMContainerInfo oldInfoProto = - HddsProtos.SCMContainerInfo.PARSER.parseFrom(containerBytes); - ContainerInfo oldInfo = ContainerInfo.fromProtobuf(oldInfoProto); - ContainerInfo newInfo = new ContainerInfo.Builder() - .setAllocatedBytes(info.getAllocatedBytes()) - .setNumberOfKeys(oldInfo.getNumberOfKeys()) - .setOwner(oldInfo.getOwner()) - .setPipelineID(oldInfo.getPipelineID()) - .setState(oldInfo.getState()) - .setUsedBytes(oldInfo.getUsedBytes()) - .setDeleteTransactionId(oldInfo.getDeleteTransactionId()) - .setReplicationFactor(oldInfo.getReplicationFactor()) - .setReplicationType(oldInfo.getReplicationType()) - .build(); - containerStore.put(dbKey, newInfo.getProtobuf().toByteArray()); + containerStore.put(dbKey, info.getProtobuf().toByteArray()); } else { LOG.debug("Container state manager has container {} but not found " + "in container store, a deleted container?", diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java index b26eed2c75d..5a9e726f888 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java @@ -84,7 +84,8 @@ public void onMessage(ContainerReportFromDatanode containerReportFromDatanode, try { //update state in container db and trigger close container events - containerMapping.processContainerReports(datanodeOrigin, containerReport); + containerMapping + .processContainerReports(datanodeOrigin, containerReport, false); Set containerIds = containerReport.getReportsList().stream() .map(containerProto -> containerProto.getContainerID()) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java index 5df7dc7e687..5eb81951f5d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; @@ -35,7 +36,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.statemachine .InvalidStateTransitionException; import org.apache.hadoop.ozone.common.statemachine.StateMachine; @@ -148,9 +148,10 @@ public ContainerStateManager(Configuration configuration, finalStates); initializeStateMachine(); - this.containerSize = OzoneConsts.GB * configuration.getInt( - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB, - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT); + this.containerSize =(long)configuration.getStorageSize( + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, + StorageUnit.BYTES); lastUsedMap = new ConcurrentHashMap<>(); containerCount = new AtomicLong(0); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java index ac84be44a38..f4b5bb22f6f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java @@ -115,7 +115,7 @@ ContainerWithPipeline allocateContainer(HddsProtos.ReplicationType type, * @param reports Container report */ void processContainerReports(DatanodeDetails datanodeDetails, - ContainerReportsProto reports) + ContainerReportsProto reports, boolean isRegisterCall) throws IOException; /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java index 028d14bd6d8..5343bce10e7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java @@ -18,6 +18,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerStateManager; @@ -38,7 +39,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.server.events.EventPublisher; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.statemachine .InvalidStateTransitionException; import org.apache.hadoop.ozone.common.statemachine.StateMachine; @@ -94,9 +94,10 @@ public PipelineSelector(NodeManager nodeManager, this.conf = conf; this.eventPublisher = eventPublisher; this.placementPolicy = createContainerPlacementPolicy(nodeManager, conf); - this.containerSize = OzoneConsts.GB * this.conf.getInt( - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB, - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT); + this.containerSize = (long)this.conf.getStorageSize( + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, + StorageUnit.BYTES); node2PipelineMap = new Node2PipelineMap(); this.standaloneManager = new StandaloneManagerImpl(this.nodeManager, placementPolicy, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java index 0d34787cc6d..92158039cd8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java @@ -196,7 +196,7 @@ public SCMRegisteredResponseProto register( if (registeredCommand.getError() == SCMRegisteredResponseProto.ErrorCode.success) { scm.getScmContainerManager().processContainerReports(datanodeDetails, - containerReportsProto); + containerReportsProto, true); } return getRegisteredResponse(registeredCommand); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java index 543cad3c970..4790c829060 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java @@ -19,6 +19,7 @@ import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -39,7 +40,7 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.CREATE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.CREATED; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE; import static org.apache.hadoop.hdds.scm.events.SCMEvents.CLOSE_CONTAINER; import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND; @@ -58,9 +59,8 @@ @BeforeClass public static void setUp() throws Exception { configuration = SCMTestUtils.getConf(); - size = configuration - .getLong(OZONE_SCM_CONTAINER_SIZE_GB, OZONE_SCM_CONTAINER_SIZE_DEFAULT) - * 1024 * 1024 * 1024; + size = (long)configuration.getStorageSize(OZONE_SCM_CONTAINER_SIZE, + OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); testDir = GenericTestUtils .getTestDir(TestCloseContainerEventHandler.class.getSimpleName()); configuration diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java index 2dc7e9960a4..b0b39f17689 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java @@ -242,7 +242,7 @@ public void testContainerCreationLeaseTimeout() throws IOException, } @Test - public void testFullContainerReport() throws IOException { + public void testFullContainerReport() throws Exception { ContainerInfo info = createContainer(); DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails(); List reports = @@ -266,13 +266,26 @@ public void testFullContainerReport() throws IOException { .newBuilder(); crBuilder.addAllReports(reports); - mapping.processContainerReports(datanodeDetails, crBuilder.build()); + mapping.processContainerReports(datanodeDetails, crBuilder.build(), false); ContainerInfo updatedContainer = mapping.getContainer(info.getContainerID()); Assert.assertEquals(100000000L, updatedContainer.getNumberOfKeys()); Assert.assertEquals(2000000000L, updatedContainer.getUsedBytes()); + + for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) { + LambdaTestUtils.intercept(SCMException.class, "No entry " + + "exist for containerId:", () -> mapping.getStateManager() + .getContainerReplicas(ContainerID.valueof(c.getContainerID()))); + } + + mapping.processContainerReports(TestUtils.randomDatanodeDetails(), + crBuilder.build(), true); + for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) { + Assert.assertTrue(mapping.getStateManager().getContainerReplicas( + ContainerID.valueof(c.getContainerID())).size() > 0); + } } @Test @@ -301,7 +314,7 @@ public void testContainerCloseWithContainerReport() throws IOException { ContainerReportsProto.newBuilder(); crBuilder.addAllReports(reports); - mapping.processContainerReports(datanodeDetails, crBuilder.build()); + mapping.processContainerReports(datanodeDetails, crBuilder.build(), false); ContainerInfo updatedContainer = mapping.getContainer(info.getContainerID()); @@ -360,4 +373,12 @@ private ContainerInfo createContainer() return containerInfo; } + @Test + public void testFlushAllContainers() throws IOException { + ContainerInfo info = createContainer(); + List containers = mapping.getStateManager().getAllContainers(); + Assert.assertTrue(containers.size() > 0); + mapping.flushContainerInfo(); + } + } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java index 0c0f25d0c14..210df088f84 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.container.closer; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.ContainerMapping; @@ -50,7 +51,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_CONTAINER_SIZE_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_CONTAINER_SIZE_GB; + .OZONE_SCM_CONTAINER_SIZE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent .CREATE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent @@ -71,8 +72,8 @@ @BeforeClass public static void setUp() throws Exception { configuration = SCMTestUtils.getConf(); - size = configuration.getLong(OZONE_SCM_CONTAINER_SIZE_GB, - OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024; + size = (long)configuration.getStorageSize(OZONE_SCM_CONTAINER_SIZE, + OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); configuration.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS); testDir = GenericTestUtils @@ -222,6 +223,6 @@ private void sendContainerReport(ContainerInfo info, long used) throws .setDeleteTransactionId(0); reports.addReports(ciBuilder); mapping.processContainerReports(TestUtils.randomDatanodeDetails(), - reports.build()); + reports.build(), false); } } \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java index 5071d8d7311..5efcdd113ff 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java @@ -309,7 +309,8 @@ private EndpointStateMachine registerTaskHelper(InetSocketAddress scmAddress, when(ozoneContainer.getContainerReport()).thenReturn( TestUtils.getRandomContainerReports(10)); RegisterEndpointTask endpointTask = - new RegisterEndpointTask(rpcEndPoint, conf, ozoneContainer); + new RegisterEndpointTask(rpcEndPoint, conf, ozoneContainer, + mock(StateContext.class)); if (!clearDatanodeDetails) { DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails(); endpointTask.setDatanodeDetails(datanodeDetails); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java index 8d71d00551e..f54322c523b 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java @@ -23,6 +23,7 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.cli.container.ContainerCommandHandler; import org.apache.hadoop.hdds.scm.cli.container.CreateContainerHandler; @@ -49,7 +50,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_CONTAINER_SIZE_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_CONTAINER_SIZE_GB; + .OZONE_SCM_CONTAINER_SIZE; import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients; import static org.apache.hadoop.hdds.scm.cli.ResultCode.EXECUTION_ERROR; import static org.apache.hadoop.hdds.scm.cli.ResultCode.SUCCESS; @@ -107,8 +108,9 @@ private static ScmClient getScmClient(OzoneConfiguration ozoneConf) StorageContainerLocationProtocolPB.class); InetSocketAddress scmAddress = getScmAddressForClients(ozoneConf); - int containerSizeGB = ozoneConf.getInt(OZONE_SCM_CONTAINER_SIZE_GB, - OZONE_SCM_CONTAINER_SIZE_DEFAULT); + int containerSizeGB = (int)ozoneConf.getStorageSize( + OZONE_SCM_CONTAINER_SIZE, OZONE_SCM_CONTAINER_SIZE_DEFAULT, + StorageUnit.GB); ContainerOperationClient.setContainerSizeB(containerSizeGB*OzoneConsts.GB); RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class, diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java index e0fdb3242d8..252b70dde44 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java @@ -18,14 +18,68 @@ package org.apache.hadoop.hdfs.server.namenode.ha; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.hdfs.HAUtilClient; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.io.retry.FailoverProxyProvider; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class AbstractNNFailoverProxyProvider implements FailoverProxyProvider { + protected static final Logger LOG = + LoggerFactory.getLogger(AbstractNNFailoverProxyProvider.class); - private AtomicBoolean fallbackToSimpleAuth; + protected Configuration conf; + protected Class xface; + protected HAProxyFactory factory; + protected UserGroupInformation ugi; + protected AtomicBoolean fallbackToSimpleAuth; + + protected AbstractNNFailoverProxyProvider() { + } + + protected AbstractNNFailoverProxyProvider(Configuration conf, URI uri, + Class xface, HAProxyFactory factory) { + this.conf = new Configuration(conf); + this.xface = xface; + this.factory = factory; + try { + this.ugi = UserGroupInformation.getCurrentUser(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + int maxRetries = this.conf.getInt( + HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_KEY, + HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_DEFAULT); + this.conf.setInt( + CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, + maxRetries); + + int maxRetriesOnSocketTimeouts = this.conf.getInt( + HdfsClientConfigKeys + .Failover.CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_KEY, + HdfsClientConfigKeys + .Failover.CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT); + this.conf.setInt( + CommonConfigurationKeysPublic + .IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, + maxRetriesOnSocketTimeouts); + } /** * Inquire whether logical HA URI is used for the implementation. If it is @@ -51,4 +105,100 @@ public synchronized void setFallbackToSimpleAuth( public synchronized AtomicBoolean getFallbackToSimpleAuth() { return fallbackToSimpleAuth; } + + /** + * ProxyInfo to a NameNode. Includes its address. + */ + public static class NNProxyInfo extends ProxyInfo { + private InetSocketAddress address; + + public NNProxyInfo(InetSocketAddress address) { + super(null, address.toString()); + this.address = address; + } + + public InetSocketAddress getAddress() { + return address; + } + } + + @Override + public Class getInterface() { + return xface; + } + + /** + * Create a proxy if it has not been created yet. + */ + protected NNProxyInfo createProxyIfNeeded(NNProxyInfo pi) { + if (pi.proxy == null) { + assert pi.getAddress() != null : "Proxy address is null"; + try { + pi.proxy = factory.createProxy(conf, + pi.getAddress(), xface, ugi, false, getFallbackToSimpleAuth()); + } catch (IOException ioe) { + LOG.error("{} Failed to create RPC proxy to NameNode", + this.getClass().getSimpleName(), ioe); + throw new RuntimeException(ioe); + } + } + return pi; + } + + /** + * Get list of configured NameNode proxy addresses. + * Randomize the list if requested. + */ + protected List> getProxyAddresses(URI uri, String addressKey) { + final List> proxies = new ArrayList>(); + Map> map = + DFSUtilClient.getAddresses(conf, null, addressKey); + Map addressesInNN = map.get(uri.getHost()); + + if (addressesInNN == null || addressesInNN.size() == 0) { + throw new RuntimeException("Could not find any configured addresses " + + "for URI " + uri); + } + + Collection addressesOfNns = addressesInNN.values(); + for (InetSocketAddress address : addressesOfNns) { + proxies.add(new NNProxyInfo(address)); + } + // Randomize the list to prevent all clients pointing to the same one + boolean randomized = getRandomOrder(conf, uri); + if (randomized) { + Collections.shuffle(proxies); + } + + // The client may have a delegation token set for the logical + // URI of the cluster. Clone this token to apply to each of the + // underlying IPC addresses so that the IPC code can find it. + HAUtilClient.cloneDelegationTokenForLogicalUri(ugi, uri, addressesOfNns); + return proxies; + } + + /** + * Check whether random order is configured for failover proxy provider + * for the namenode/nameservice. + * + * @param conf Configuration + * @param nameNodeUri The URI of namenode/nameservice + * @return random order configuration + */ + public static boolean getRandomOrder( + Configuration conf, URI nameNodeUri) { + String host = nameNodeUri.getHost(); + String configKeyWithHost = HdfsClientConfigKeys.Failover.RANDOM_ORDER + + "." + host; + + if (conf.get(configKeyWithHost) != null) { + return conf.getBoolean( + configKeyWithHost, + HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT); + } + + return conf.getBoolean( + HdfsClientConfigKeys.Failover.RANDOM_ORDER, + HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java index f46532ad972..92e75cee364 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java @@ -19,23 +19,11 @@ import java.io.Closeable; import java.io.IOException; -import java.net.InetSocketAddress; import java.net.URI; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.List; -import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.hdfs.DFSUtilClient; -import org.apache.hadoop.hdfs.HAUtilClient; -import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.security.UserGroupInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; @@ -48,17 +36,9 @@ public class ConfiguredFailoverProxyProvider extends AbstractNNFailoverProxyProvider { - private static final Logger LOG = - LoggerFactory.getLogger(ConfiguredFailoverProxyProvider.class); - - protected final Configuration conf; - protected final List> proxies = - new ArrayList>(); - protected final UserGroupInformation ugi; - protected final Class xface; + protected final List> proxies; private int currentProxyIndex = 0; - protected final HAProxyFactory factory; public ConfiguredFailoverProxyProvider(Configuration conf, URI uri, Class xface, HAProxyFactory factory) { @@ -67,83 +47,8 @@ public ConfiguredFailoverProxyProvider(Configuration conf, URI uri, public ConfiguredFailoverProxyProvider(Configuration conf, URI uri, Class xface, HAProxyFactory factory, String addressKey) { - this.xface = xface; - this.conf = new Configuration(conf); - int maxRetries = this.conf.getInt( - HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_KEY, - HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_DEFAULT); - this.conf.setInt( - CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, - maxRetries); - - int maxRetriesOnSocketTimeouts = this.conf.getInt( - HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_KEY, - HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT); - this.conf.setInt( - CommonConfigurationKeysPublic - .IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, - maxRetriesOnSocketTimeouts); - - try { - ugi = UserGroupInformation.getCurrentUser(); - - Map> map = - DFSUtilClient.getAddresses(conf, null, addressKey); - Map addressesInNN = map.get(uri.getHost()); - - if (addressesInNN == null || addressesInNN.size() == 0) { - throw new RuntimeException("Could not find any configured addresses " + - "for URI " + uri); - } - - Collection addressesOfNns = addressesInNN.values(); - for (InetSocketAddress address : addressesOfNns) { - proxies.add(new AddressRpcProxyPair(address)); - } - // Randomize the list to prevent all clients pointing to the same one - boolean randomized = getRandomOrder(conf, uri); - if (randomized) { - Collections.shuffle(proxies); - } - - // The client may have a delegation token set for the logical - // URI of the cluster. Clone this token to apply to each of the - // underlying IPC addresses so that the IPC code can find it. - HAUtilClient.cloneDelegationTokenForLogicalUri(ugi, uri, addressesOfNns); - this.factory = factory; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Check whether random order is configured for failover proxy provider - * for the namenode/nameservice. - * - * @param conf Configuration - * @param nameNodeUri The URI of namenode/nameservice - * @return random order configuration - */ - private static boolean getRandomOrder( - Configuration conf, URI nameNodeUri) { - String host = nameNodeUri.getHost(); - String configKeyWithHost = HdfsClientConfigKeys.Failover.RANDOM_ORDER - + "." + host; - - if (conf.get(configKeyWithHost) != null) { - return conf.getBoolean( - configKeyWithHost, - HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT); - } - - return conf.getBoolean( - HdfsClientConfigKeys.Failover.RANDOM_ORDER, - HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT); - } - - @Override - public Class getInterface() { - return xface; + super(conf, uri, xface, factory); + this.proxies = getProxyAddresses(uri, addressKey); } /** @@ -151,21 +56,8 @@ private static boolean getRandomOrder( */ @Override public synchronized ProxyInfo getProxy() { - AddressRpcProxyPair current = proxies.get(currentProxyIndex); - return getProxy(current); - } - - protected ProxyInfo getProxy(AddressRpcProxyPair current) { - if (current.namenode == null) { - try { - current.namenode = factory.createProxy(conf, - current.address, xface, ugi, false, getFallbackToSimpleAuth()); - } catch (IOException e) { - LOG.error("Failed to create RPC proxy to NameNode", e); - throw new RuntimeException(e); - } - } - return new ProxyInfo(current.namenode, current.address.toString()); + NNProxyInfo current = proxies.get(currentProxyIndex); + return createProxyIfNeeded(current); } @Override @@ -177,31 +69,18 @@ synchronized void incrementProxyIndex() { currentProxyIndex = (currentProxyIndex + 1) % proxies.size(); } - /** - * A little pair object to store the address and connected RPC proxy object to - * an NN. Note that {@link AddressRpcProxyPair#namenode} may be null. - */ - protected static class AddressRpcProxyPair { - public final InetSocketAddress address; - public T namenode; - - public AddressRpcProxyPair(InetSocketAddress address) { - this.address = address; - } - } - /** * Close all the proxy objects which have been opened over the lifetime of * this proxy provider. */ @Override public synchronized void close() throws IOException { - for (AddressRpcProxyPair proxy : proxies) { - if (proxy.namenode != null) { - if (proxy.namenode instanceof Closeable) { - ((Closeable)proxy.namenode).close(); + for (ProxyInfo proxy : proxies) { + if (proxy.proxy != null) { + if (proxy.proxy instanceof Closeable) { + ((Closeable)proxy.proxy).close(); } else { - RPC.stopProxy(proxy.namenode); + RPC.stopProxy(proxy.proxy); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java index ed250a0f42e..e70374047a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java @@ -19,15 +19,11 @@ import java.io.Closeable; import java.io.IOException; -import java.net.InetSocketAddress; import java.net.URI; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdfs.DFSUtilClient; -import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.security.UserGroupInformation; /** * A NNFailoverProxyProvider implementation which works on IP failover setup. @@ -47,53 +43,18 @@ */ public class IPFailoverProxyProvider extends AbstractNNFailoverProxyProvider { - private final Configuration conf; - private final Class xface; - private final URI nameNodeUri; - private final HAProxyFactory factory; - private ProxyInfo nnProxyInfo = null; + private final NNProxyInfo nnProxyInfo; public IPFailoverProxyProvider(Configuration conf, URI uri, Class xface, HAProxyFactory factory) { - this.xface = xface; - this.nameNodeUri = uri; - this.factory = factory; - - this.conf = new Configuration(conf); - int maxRetries = this.conf.getInt( - HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_KEY, - HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_DEFAULT); - this.conf.setInt( - CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, - maxRetries); - - int maxRetriesOnSocketTimeouts = this.conf.getInt( - HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_KEY, - HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT); - this.conf.setInt( - CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, - maxRetriesOnSocketTimeouts); - } - - @Override - public Class getInterface() { - return xface; + super(conf, uri, xface, factory); + this.nnProxyInfo = new NNProxyInfo(DFSUtilClient.getNNAddress(uri)); } @Override - public synchronized ProxyInfo getProxy() { + public synchronized NNProxyInfo getProxy() { // Create a non-ha proxy if not already created. - if (nnProxyInfo == null) { - try { - // Create a proxy that is not wrapped in RetryProxy - InetSocketAddress nnAddr = DFSUtilClient.getNNAddress(nameNodeUri); - nnProxyInfo = new ProxyInfo(factory.createProxy(conf, nnAddr, xface, - UserGroupInformation.getCurrentUser(), false), nnAddr.toString()); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - return nnProxyInfo; + return createProxyIfNeeded(nnProxyInfo); } /** Nothing to do for IP failover */ @@ -106,7 +67,7 @@ public void performFailover(T currentProxy) { */ @Override public synchronized void close() throws IOException { - if (nnProxyInfo == null) { + if (nnProxyInfo.proxy == null) { return; } if (nnProxyInfo.proxy instanceof Closeable) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java index c264de32d2e..d45441fe4a1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java @@ -22,6 +22,8 @@ import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT; import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry; import java.io.IOException; @@ -124,12 +126,19 @@ public MountTableResolver(Configuration conf, Router routerService, this.stateStore = null; } - int maxCacheSize = conf.getInt( - FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, - FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT); - this.locationCache = CacheBuilder.newBuilder() - .maximumSize(maxCacheSize) - .build(); + boolean mountTableCacheEnable = conf.getBoolean( + FEDERATION_MOUNT_TABLE_CACHE_ENABLE, + FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT); + if (mountTableCacheEnable) { + int maxCacheSize = conf.getInt( + FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, + FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT); + this.locationCache = CacheBuilder.newBuilder() + .maximumSize(maxCacheSize) + .build(); + } else { + this.locationCache = null; + } registerCacheExternal(); initDefaultNameService(conf); @@ -239,7 +248,7 @@ public void removeEntry(final String srcPath) { */ private void invalidateLocationCache(final String path) { LOG.debug("Invalidating {} from {}", path, locationCache); - if (locationCache.size() == 0) { + if (locationCache == null || locationCache.size() == 0) { return; } @@ -359,7 +368,9 @@ public void clear() { LOG.info("Clearing all mount location caches"); writeLock.lock(); try { - this.locationCache.invalidateAll(); + if (this.locationCache != null) { + this.locationCache.invalidateAll(); + } this.tree.clear(); } finally { writeLock.unlock(); @@ -372,6 +383,9 @@ public PathLocation getDestinationForPath(final String path) verifyMountTable(); readLock.lock(); try { + if (this.locationCache == null) { + return lookupLocation(path); + } Callable meh = new Callable() { @Override public PathLocation call() throws Exception { @@ -603,7 +617,10 @@ private MountTable findDeepest(final String path) { * Get the size of the cache. * @return Size of the cache. */ - protected long getCacheSize() { - return this.locationCache.size(); + protected long getCacheSize() throws IOException{ + if (this.locationCache != null) { + return this.locationCache.size(); + } + throw new IOException("localCache is null"); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java index 363db208056..87df5d2cd5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java @@ -186,6 +186,10 @@ FEDERATION_ROUTER_PREFIX + "mount-table.max-cache-size"; /** Remove cache entries if we have more than 10k. */ public static final int FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT = 10000; + public static final String FEDERATION_MOUNT_TABLE_CACHE_ENABLE = + FEDERATION_ROUTER_PREFIX + "mount-table.cache.enable"; + public static final boolean FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT = + true; // HDFS Router-based federation admin public static final String DFS_ROUTER_ADMIN_HANDLER_COUNT_KEY = diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml index 8806cb27de9..8be5b8ab77c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml @@ -393,6 +393,15 @@ + + dfs.federation.router.mount-table.cache.enable + true + + Set to true to enable mount table cache (Path to Remote Location cache). + Disabling the cache is recommended when a large amount of unique paths are queried. + + + dfs.federation.router.quota.enable false diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java index cb3b472ced0..b19a9732940 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.federation.resolver; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE; import static org.junit.Assert.assertEquals; @@ -37,6 +38,7 @@ import org.apache.hadoop.hdfs.server.federation.router.Router; import org.apache.hadoop.hdfs.server.federation.store.MountTableStore; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -472,6 +474,35 @@ public void testUpdate() throws IOException { assertNull(entry2); } + @Test + public void testDisableLocalCache() throws IOException { + Configuration conf = new Configuration(); + // Disable mount table cache + conf.setBoolean(FEDERATION_MOUNT_TABLE_CACHE_ENABLE, false); + conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0"); + MountTableResolver tmpMountTable = new MountTableResolver(conf); + + // Root mount point + Map map = getMountTableEntry("1", "/"); + tmpMountTable.addEntry(MountTable.newInstance("/", map)); + + // /tmp + map = getMountTableEntry("2", "/tmp"); + tmpMountTable.addEntry(MountTable.newInstance("/tmp", map)); + + // Check localCache is null + try { + tmpMountTable.getCacheSize(); + fail("getCacheSize call should fail."); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("localCache is null", e); + } + + // Check resolve path without cache + assertEquals("2->/tmp/tesfile1.txt", + tmpMountTable.getDestinationForPath("/tmp/tesfile1.txt").toString()); + } + @Test public void testCacheCleaning() throws Exception { for (int i = 0; i < 1000; i++) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java index d2b48ccec53..26337237119 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java @@ -89,8 +89,9 @@ AsyncLogger createLogger(Configuration conf, NamespaceInfo nsInfo, /** * Format the log directory. * @param nsInfo the namespace info to format with + * @param force the force option to format */ - public ListenableFuture format(NamespaceInfo nsInfo); + public ListenableFuture format(NamespaceInfo nsInfo, boolean force); /** * @return whether or not the remote node has any valid data. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java index d46c2cf790f..b52e312943a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java @@ -299,12 +299,12 @@ void appendReport(StringBuilder sb) { return QuorumCall.create(calls); } - QuorumCall format(NamespaceInfo nsInfo) { + QuorumCall format(NamespaceInfo nsInfo, boolean force) { Map> calls = Maps.newHashMap(); for (AsyncLogger logger : loggers) { ListenableFuture future = - logger.format(nsInfo); + logger.format(nsInfo, force); calls.put(logger, future); } return QuorumCall.create(calls); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java index 30367357b82..4fca1bb0ad0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java @@ -502,11 +502,12 @@ private synchronized void unreserveQueueSpace(int size) { } @Override - public ListenableFuture format(final NamespaceInfo nsInfo) { + public ListenableFuture format(final NamespaceInfo nsInfo, + final boolean force) { return singleThreadExecutor.submit(new Callable() { @Override public Void call() throws Exception { - getProxy().format(journalId, nameServiceId, nsInfo); + getProxy().format(journalId, nameServiceId, nsInfo, force); return null; } }); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java index 4faaa983bee..bd452923eae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java @@ -220,8 +220,8 @@ public static void checkJournalId(String jid) { } @Override - public void format(NamespaceInfo nsInfo) throws IOException { - QuorumCall call = loggers.format(nsInfo); + public void format(NamespaceInfo nsInfo, boolean force) throws IOException { + QuorumCall call = loggers.format(nsInfo, force); try { call.waitFor(loggers.size(), loggers.size(), 0, timeoutMs, "format"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java index 5558bd54721..8dad26104c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java @@ -68,7 +68,7 @@ GetJournalStateResponseProto getJournalState(String journalId, * Format the underlying storage for the given namespace. */ void format(String journalId, String nameServiceId, - NamespaceInfo nsInfo) throws IOException; + NamespaceInfo nsInfo, boolean force) throws IOException; /** * Begin a new epoch. See the HDFS-3077 design doc for details. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java index 865d2969220..2ad19da0960 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java @@ -147,7 +147,7 @@ public FormatResponseProto format(RpcController controller, try { impl.format(request.getJid().getIdentifier(), request.hasNameServiceId() ? request.getNameServiceId() : null, - PBHelper.convert(request.getNsInfo())); + PBHelper.convert(request.getNsInfo()), request.getForce()); return FormatResponseProto.getDefaultInstance(); } catch (IOException ioe) { throw new ServiceException(ioe); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java index d7cd7b55811..42d35f57fb5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java @@ -136,11 +136,13 @@ private JournalIdProto convertJournalId(String jid) { @Override public void format(String jid, String nameServiceId, - NamespaceInfo nsInfo) throws IOException { + NamespaceInfo nsInfo, + boolean force) throws IOException { try { FormatRequestProto.Builder req = FormatRequestProto.newBuilder() .setJid(convertJournalId(jid)) - .setNsInfo(PBHelper.convert(nsInfo)); + .setNsInfo(PBHelper.convert(nsInfo)) + .setForce(force); if(nameServiceId != null) { req.setNameServiceId(nameServiceId); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java index 6bf4903b1ec..612fd3d19f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java @@ -204,10 +204,10 @@ private static void purgeMatching(File dir, List patterns, } } - void format(NamespaceInfo nsInfo) throws IOException { + void format(NamespaceInfo nsInfo, boolean force) throws IOException { unlockAll(); try { - sd.analyzeStorage(StartupOption.FORMAT, this, true); + sd.analyzeStorage(StartupOption.FORMAT, this, !force); } finally { sd.unlock(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java index 8f25d260b64..7e88afa78dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java @@ -227,13 +227,13 @@ private synchronized EditLogFile scanStorageForLatestEdits() throws IOException /** * Format the local storage with the given namespace. */ - void format(NamespaceInfo nsInfo) throws IOException { + void format(NamespaceInfo nsInfo, boolean force) throws IOException { Preconditions.checkState(nsInfo.getNamespaceID() != 0, "can't format with uninitialized namespace info: %s", nsInfo); LOG.info("Formatting journal id : " + journalId + " with namespace info: " + - nsInfo); - storage.format(nsInfo); + nsInfo + " and force: " + force); + storage.format(nsInfo, force); refreshCachedData(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java index b1a3c9665d7..0f11026b1d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java @@ -176,9 +176,10 @@ public NewEpochResponseProto newEpoch(String journalId, @Override public void format(String journalId, String nameServiceId, - NamespaceInfo nsInfo) + NamespaceInfo nsInfo, + boolean force) throws IOException { - jn.getOrCreateJournal(journalId, nameServiceId).format(nsInfo); + jn.getOrCreateJournal(journalId, nameServiceId).format(nsInfo, force); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java index e1ddfb909cf..eac91bf4832 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java @@ -42,7 +42,7 @@ } @Override - public void format(NamespaceInfo nsInfo) { + public void format(NamespaceInfo nsInfo, boolean force) { // format() should only get called at startup, before any BNs // can register with the NN. throw new UnsupportedOperationException( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index d6fb212c7d9..547ad577c38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -417,13 +417,14 @@ synchronized void close() { * File-based journals are skipped, since they are formatted by the * Storage format code. */ - synchronized void formatNonFileJournals(NamespaceInfo nsInfo) throws IOException { + synchronized void formatNonFileJournals(NamespaceInfo nsInfo, boolean force) + throws IOException { Preconditions.checkState(state == State.BETWEEN_LOG_SEGMENTS, "Bad state: %s", state); for (JournalManager jm : journalSet.getJournalManagers()) { if (!(jm instanceof FileJournalManager)) { - jm.format(nsInfo); + jm.format(nsInfo, force); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 5cfc0176f1d..6d107be7891 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -160,7 +160,8 @@ protected FSImage(Configuration conf, archivalManager = new NNStorageRetentionManager(conf, storage, editLog); } - void format(FSNamesystem fsn, String clusterId) throws IOException { + void format(FSNamesystem fsn, String clusterId, boolean force) + throws IOException { long fileCount = fsn.getFilesTotal(); // Expect 1 file, which is the root inode Preconditions.checkState(fileCount == 1, @@ -171,7 +172,7 @@ void format(FSNamesystem fsn, String clusterId) throws IOException { ns.clusterID = clusterId; storage.format(ns); - editLog.formatNonFileJournals(ns); + editLog.formatNonFileJournals(ns, force); saveFSImageInAllDirs(fsn, 0); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index b0fb26ccb66..06bf0081370 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1078,8 +1078,8 @@ private void loadFSImage(StartupOption startOpt) throws IOException { // format before starting up if requested if (startOpt == StartupOption.FORMAT) { - - fsImage.format(this, fsImage.getStorage().determineClusterId());// reuse current id + // reuse current id + fsImage.format(this, fsImage.getStorage().determineClusterId(), false); startOpt = StartupOption.REGULAR; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java index c71c09ad4d8..185ad731b26 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java @@ -100,7 +100,7 @@ public FileJournalManager(Configuration conf, StorageDirectory sd, public void close() throws IOException {} @Override - public void format(NamespaceInfo ns) throws IOException { + public void format(NamespaceInfo ns, boolean force) throws IOException { // Formatting file journals is done by the StorageDirectory // format code, since they may share their directory with // checkpoints, etc. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java index ae1bc3b7db7..d6d20945bc1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java @@ -43,7 +43,7 @@ * Format the underlying storage, removing any previously * stored data. */ - void format(NamespaceInfo ns) throws IOException; + void format(NamespaceInfo ns, boolean force) throws IOException; /** * Begin writing to a new segment of the log stream, which starts at diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java index e7f2adb5bd6..868df017ca6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java @@ -188,7 +188,7 @@ public boolean isShared() { } @Override - public void format(NamespaceInfo nsInfo) throws IOException { + public void format(NamespaceInfo nsInfo, boolean force) throws IOException { // The operation is done by FSEditLog itself throw new UnsupportedOperationException(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 7f78d2fcfd9..a8034da85e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -1159,7 +1159,7 @@ private static boolean format(Configuration conf, boolean force, return true; // aborted } - fsImage.format(fsn, clusterId); + fsImage.format(fsn, clusterId, force); } catch (IOException ioe) { LOG.warn("Encountered exception during format: ", ioe); fsImage.close(); @@ -1262,7 +1262,7 @@ private static boolean initializeSharedEdits(Configuration conf, // actually want to save a checkpoint - just prime the dirs with // the existing namespace info newSharedStorage.format(nsInfo); - sharedEditsImage.getEditLog().formatNonFileJournals(nsInfo); + sharedEditsImage.getEditLog().formatNonFileJournals(nsInfo, force); // Need to make sure the edit log segments are in good shape to initialize // the shared edits dir. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto index a37c7236a65..625966fd33c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto @@ -242,6 +242,7 @@ message FormatRequestProto { required JournalIdProto jid = 1; required NamespaceInfoProto nsInfo = 2; optional string nameServiceId = 3; + optional bool force = 4 [ default = false ]; } message FormatResponseProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java index 5101a41f0e5..0fc142929f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java @@ -56,7 +56,7 @@ public void testSingleThreaded() throws IOException { QuorumJournalManager qjm = new QuorumJournalManager( conf, uri, FAKE_NSINFO); try { - qjm.format(FAKE_NSINFO); + qjm.format(FAKE_NSINFO, false); } finally { qjm.close(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java index 6ad43f5835e..40f213e7071 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java @@ -105,7 +105,7 @@ private static long determineMaxIpcNumber() throws Exception { long ret; try { qjm = createInjectableQJM(cluster); - qjm.format(FAKE_NSINFO); + qjm.format(FAKE_NSINFO, false); doWorkload(cluster, qjm); SortedSet ipcCounts = Sets.newTreeSet(); @@ -156,7 +156,7 @@ public void testRecoverAfterDoubleFailures() throws Exception { QuorumJournalManager qjm = null; try { qjm = createInjectableQJM(cluster); - qjm.format(FAKE_NSINFO); + qjm.format(FAKE_NSINFO, false); List loggers = qjm.getLoggerSetForTests().getLoggersForTests(); failIpcNumber(loggers.get(0), failA); failIpcNumber(loggers.get(1), failB); @@ -240,7 +240,7 @@ public void testRandomized() throws Exception { // Format the cluster using a non-faulty QJM. QuorumJournalManager qjmForInitialFormat = createInjectableQJM(cluster); - qjmForInitialFormat.format(FAKE_NSINFO); + qjmForInitialFormat.format(FAKE_NSINFO, false); qjmForInitialFormat.close(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java index 69856ae3fa9..00bec22564d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java @@ -100,7 +100,7 @@ public void setup() throws Exception { qjm = createSpyingQJM(); spies = qjm.getLoggerSetForTests().getLoggersForTests(); - qjm.format(QJMTestUtil.FAKE_NSINFO); + qjm.format(QJMTestUtil.FAKE_NSINFO, false); qjm.recoverUnfinalizedSegments(); assertEquals(1, qjm.getLoggerSetForTests().getEpoch()); } @@ -149,7 +149,7 @@ public void testFormat() throws Exception { QuorumJournalManager qjm = closeLater(new QuorumJournalManager( conf, cluster.getQuorumJournalURI("testFormat-jid"), FAKE_NSINFO)); assertFalse(qjm.hasSomeData()); - qjm.format(FAKE_NSINFO); + qjm.format(FAKE_NSINFO, false); assertTrue(qjm.hasSomeData()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java index 75dcf2fbda8..9e1e3bb9dc8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java @@ -19,6 +19,7 @@ import static org.junit.Assert.fail; import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.eq; import java.io.IOException; @@ -89,7 +90,8 @@ public void setup() throws Exception { NewEpochResponseProto.newBuilder().build() ).when(logger).newEpoch(Mockito.anyLong()); - futureReturns(null).when(logger).format(Mockito.any()); + futureReturns(null).when(logger).format(Mockito.any(), + anyBoolean()); } qjm.recoverUnfinalizedSegments(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java index b71d69445c7..b8d2652ef46 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java @@ -73,7 +73,7 @@ public void setup() throws Exception { conf = new Configuration(); journal = new Journal(conf, TEST_LOG_DIR, JID, StartupOption.REGULAR, mockErrorReporter); - journal.format(FAKE_NSINFO); + journal.format(FAKE_NSINFO, false); } @After @@ -207,7 +207,7 @@ public void testFormatResetsCachedValues() throws Exception { // Clear the storage directory before reformatting it journal.getStorage().getJournalManager() .getStorageDirectory().clearDirectory(); - journal.format(FAKE_NSINFO_2); + journal.format(FAKE_NSINFO_2, false); assertEquals(0, journal.getLastPromisedEpoch()); assertEquals(0, journal.getLastWriterEpoch()); @@ -425,7 +425,7 @@ public void testFormatNonEmptyStorageDirectories() throws Exception { try { // Format again here and to format the non-empty directories in // journal node. - journal.format(FAKE_NSINFO); + journal.format(FAKE_NSINFO, false); fail("Did not fail to format non-empty directories in journal node."); } catch (IOException ioe) { GenericTestUtils.assertExceptionContains( @@ -434,4 +434,15 @@ public void testFormatNonEmptyStorageDirectories() throws Exception { } } + @Test + public void testFormatNonEmptyStorageDirectoriesWhenforceOptionIsTrue() + throws Exception { + try { + // Format again here and to format the non-empty directories in + // journal node. + journal.format(FAKE_NSINFO, true); + } catch (IOException ioe) { + fail("Format should be success with force option."); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java index 8d587927fba..4cc5968e6b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java @@ -159,11 +159,11 @@ public void setup() throws Exception { HdfsServerConstants.StartupOption.REGULAR); NamespaceInfo fakeNameSpaceInfo = new NamespaceInfo( 12345, "mycluster", "my-bp"+nsId, 0L); - journal.format(fakeNameSpaceInfo); + journal.format(fakeNameSpaceInfo, false); } } else { journal = jn.getOrCreateJournal(journalId); - journal.format(FAKE_NSINFO); + journal.format(FAKE_NSINFO, false); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java index 1de37a4d245..7550c4e9e79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java @@ -82,7 +82,7 @@ public void testJournalNodeMXBean() throws Exception { // format the journal ns1 final NamespaceInfo FAKE_NSINFO = new NamespaceInfo(12345, "mycluster", "my-bp", 0L); - jn.getOrCreateJournal(NAMESERVICE).format(FAKE_NSINFO); + jn.getOrCreateJournal(NAMESERVICE).format(FAKE_NSINFO, false); // check again after format // getJournalsStatus diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java index 8de96417536..c23604b9988 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java @@ -341,7 +341,7 @@ public void testSyncAfterJNformat() throws Exception{ } // Format the JN - journal1.format(nsInfo); + journal1.format(nsInfo, false); // Roll some more edits for (int i = 4; i < 10; i++) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java index 020ecb56aa5..edcf9e15a91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java @@ -155,7 +155,7 @@ public DummyJournalManager(Configuration conf, URI u, } @Override - public void format(NamespaceInfo nsInfo) throws IOException { + public void format(NamespaceInfo nsInfo, boolean force) throws IOException { formatCalled = true; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index ac4b73b2417..2cb37166c9b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -58,10 +58,14 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; @@ -538,4 +542,22 @@ public Resource getResourceProfile(String profile) throws YarnException, IOException { return client.getResourceTypeInfo(); } + + @Override + public Set getClusterAttributes() + throws YarnException, IOException { + return client.getClusterAttributes(); + } + + @Override + public Map> getAttributesToNodes( + Set attributes) throws YarnException, IOException { + return client.getAttributesToNodes(attributes); + } + + @Override + public Map> getNodeToAttributes( + Set hostNames) throws YarnException, IOException { + return client.getNodeToAttributes(hostNames); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index f97d0a48f72..5972f65e961 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -82,8 +82,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -100,6 +104,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; @@ -521,6 +527,25 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo( throws YarnException, IOException { return null; } + + @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + return null; + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + return null; + } + + @Override + public GetNodesToAttributesResponse getNodesToAttributes( + GetNodesToAttributesRequest request) throws YarnException, IOException { + return null; + } } class HistoryService extends AMService implements HSClientProtocol { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java index 4ca4124086f..dcf40221d57 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java @@ -203,8 +203,12 @@ public void testBlockDeletion() throws Exception { MetadataStore meta = KeyUtils.getDB( (KeyValueContainerData) containerData.get(0), conf); Map containerMap = containerSet.getContainerMap(); - long transactionId = containerMap.get(containerData.get(0).getContainerID()) - .getContainerData().getDeleteTransactionId(); + // NOTE: this test assumes that all the container is KetValueContainer and + // have DeleteTransactionId in KetValueContainerData. If other + // types is going to be added, this test should be checked. + long transactionId = ((KeyValueContainerData)containerMap + .get(containerData.get(0).getContainerID()).getContainerData()) + .getDeleteTransactionId(); // Number of deleted blocks in container should be equal to 0 before diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java index 2524de6ba6d..57941515e9c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java @@ -139,7 +139,9 @@ public void testBlockDeletion() Assert.assertTrue(verifyBlocksCreated(omKeyLocationInfoGroupList)); // No containers with deleted blocks Assert.assertTrue(containerIdsWithDeletedBlocks.isEmpty()); - // Delete transactionIds for the containers should be 0 + // Delete transactionIds for the containers should be 0. + // NOTE: this test assumes that all the container is KetValueContainer. If + // other container types is going to be added, this test should be checked. matchContainerTransactionIds(); om.deleteKey(keyArgs); Thread.sleep(5000); @@ -185,7 +187,8 @@ private void verifyBlockDeletionEvent() logCapturer.clearOutput(); scm.getScmContainerManager().processContainerReports( - cluster.getHddsDatanodes().get(0).getDatanodeDetails(), dummyReport); + cluster.getHddsDatanodes().get(0).getDatanodeDetails(), dummyReport, + false); // wait for event to be handled by event handler Thread.sleep(1000); String output = logCapturer.getOutput(); @@ -215,8 +218,9 @@ private void matchContainerTransactionIds() throws IOException { Assert.assertEquals( scm.getContainerInfo(containerId).getDeleteTransactionId(), 0); } - Assert.assertEquals(dnContainerSet.getContainer(containerId) - .getContainerData().getDeleteTransactionId(), + Assert.assertEquals(((KeyValueContainerData)dnContainerSet + .getContainer(containerId).getContainerData()) + .getDeleteTransactionId(), scm.getContainerInfo(containerId).getDeleteTransactionId()); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java index 3d39dbb6dd4..84b7b764b9e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java @@ -33,7 +33,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Test; @@ -52,7 +52,7 @@ public void test() throws IOException, TimeoutException, InterruptedException, //setup a cluster (1G free space is enough for a unit test) OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OZONE_SCM_CONTAINER_SIZE_GB, "1"); + conf.set(OZONE_SCM_CONTAINER_SIZE, "1GB"); MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(1).build(); cluster.waitForClusterToBeReady(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java index a5b101fa70b..9e082126296 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java @@ -23,14 +23,13 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.client.rest.OzoneException; import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand; import org.apache.hadoop.test.GenericTestUtils; import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_CONTAINER_SIZE_GB; + .OZONE_SCM_CONTAINER_SIZE; import org.junit.Test; /** @@ -47,7 +46,7 @@ public void test() throws IOException, TimeoutException, InterruptedException, .captureLogs(ReplicateContainerCommandHandler.LOG); OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OZONE_SCM_CONTAINER_SIZE_GB, "1"); + conf.set(OZONE_SCM_CONTAINER_SIZE, "1GB"); MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build(); cluster.waitForClusterToBeReady(); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java index 5c744305c5b..0757a66223e 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java @@ -249,7 +249,7 @@ void assertCounterInRange(Job job, Enum counter, long min, long max) Counter c = job.getCounters().findCounter(counter); long value = c.getValue(); String description = - String.format("%s value %s", c.getDisplayName(), value); + String.format("%s value %s", c.getDisplayName(), value, false); if (min >= 0) { assertTrue(description + " too below minimum " + min, diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java index 69946c88bef..2eee3517170 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.sls.nodemanager; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -32,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; @@ -219,6 +221,11 @@ public Integer getDecommissioningTimeout() { return null; } + @Override + public Set getAllNodeAttributes() { + return Collections.emptySet(); + } + @Override public RMContext getRMContext() { return null; diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index a96b7901bfc..248b634b170 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; @@ -206,6 +207,11 @@ public Integer getDecommissioningTimeout() { public Map getAllocationTagsWithCount() { return node.getAllocationTagsWithCount(); } + + @Override + public Set getAllNodeAttributes() { + return node.getAllNodeAttributes(); + } @Override public RMContext getRMContext() { diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java index b8bc8be4315..2957d239f13 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java @@ -32,7 +32,6 @@ import java.util.SortedMap; import java.util.Locale; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.Lock; @@ -48,6 +47,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.concurrent.HadoopScheduledThreadPoolExecutor; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; @@ -169,7 +169,7 @@ void init(ResourceScheduler resourceScheduler, Configuration config) web.start(); // a thread to update histogram timer - pool = new ScheduledThreadPoolExecutor(2); + pool = new HadoopScheduledThreadPoolExecutor(2); pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000, TimeUnit.MILLISECONDS); @@ -518,7 +518,8 @@ public void run() { @Override public void run() { - if(running) { + SchedulerWrapper wrapper = (SchedulerWrapper) scheduler; + if(running && wrapper.getTracker().getQueueSet() != null) { // all WebApp to get real tracking json String trackingMetrics = web.generateRealTimeTrackingMetrics(); // output diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 69afe6f88a7..8290fcda8d9 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -55,6 +55,7 @@ function hadoop_usage hadoop_add_subcommand "timelinereader" client "run the timeline reader server" hadoop_add_subcommand "timelineserver" daemon "run the timeline server" hadoop_add_subcommand "top" client "view cluster information" + hadoop_add_subcommand "nodeattributes" client "node attributes cli client" hadoop_add_subcommand "version" client "print the version" hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true } @@ -186,6 +187,10 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' ;; + nodeattributes) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="false" + HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.NodeAttributesCLI' + ;; timelineserver) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index 3c4e4d01002..941a688134f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -27,8 +27,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -39,6 +43,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; @@ -642,4 +648,53 @@ GetResourceProfileResponse getResourceProfile( @Unstable GetAllResourceTypeInfoResponse getResourceTypeInfo( GetAllResourceTypeInfoRequest request) throws YarnException, IOException; + + /** + *

+ * The interface used by client to get attributes to nodes mappings + * available in ResourceManager. + *

+ * + * @param request request to get details of attributes to nodes mapping. + * @return Response containing the details of attributes to nodes mappings. + * @throws YarnException if any error happens inside YARN + * @throws IOException incase of other errors + */ + @Public + @Unstable + GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException; + + /** + *

+ * The interface used by client to get node attributes available in + * ResourceManager. + *

+ * + * @param request request to get node attributes collection of this cluster. + * @return Response containing node attributes collection. + * @throws YarnException if any error happens inside YARN. + * @throws IOException incase of other errors. + */ + @Public + @Unstable + GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException; + + /** + *

+ * The interface used by client to get node to attributes mappings. + * in existing cluster. + *

+ * + * @param request request to get nodes to attributes mapping. + * @return nodes to attributes mappings. + * @throws YarnException if any error happens inside YARN. + * @throws IOException + */ + @Public + @Unstable + GetNodesToAttributesResponse getNodesToAttributes( + GetNodesToAttributesRequest request) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java new file mode 100644 index 00000000000..94814e9053e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The request from clients to get node to attribute value mapping for all or + * given set of Node AttributeKey's in the cluster from the + * ResourceManager. + *

+ * + * @see ApplicationClientProtocol#getAttributesToNodes + * (GetAttributesToNodesRequest) + */ +@Public +@Evolving +public abstract class GetAttributesToNodesRequest { + + public static GetAttributesToNodesRequest newInstance() { + return Records.newRecord(GetAttributesToNodesRequest.class); + } + + public static GetAttributesToNodesRequest newInstance( + Set attributes) { + GetAttributesToNodesRequest request = + Records.newRecord(GetAttributesToNodesRequest.class); + request.setNodeAttributes(attributes); + return request; + } + + /** + * Set node attributeKeys for which the mapping of hostname to attribute value + * is required. + * + * @param attributes Set provided. + */ + @Public + @Unstable + public abstract void setNodeAttributes(Set attributes); + + /** + * Get node attributeKeys for which mapping of hostname to attribute value is + * required. + * + * @return Set + */ + @Public + @Unstable + public abstract Set getNodeAttributes(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java new file mode 100644 index 00000000000..9bd529f3c4c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The response sent by the ResourceManager to a client requesting + * node to attribute value mapping for all or given set of Node AttributeKey's. + *

+ * + * @see ApplicationClientProtocol#getAttributesToNodes + * (GetAttributesToNodesRequest) + */ +@Public +@Evolving +public abstract class GetAttributesToNodesResponse { + public static GetAttributesToNodesResponse newInstance( + Map> map) { + GetAttributesToNodesResponse response = + Records.newRecord(GetAttributesToNodesResponse.class); + response.setAttributeToNodes(map); + return response; + } + + @Public + @Evolving + public abstract void setAttributeToNodes( + Map> map); + + /** + * Get mapping of NodeAttributeKey to its associated mapping of list of + * NodeToAttributeValuenode to attribute value. + * + * @return Map> node attributes + * to list of NodeToAttributeValuenode. + */ + @Public + @Evolving + public abstract Map> getAttributesToNodes(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesRequest.java new file mode 100644 index 00000000000..ca81f9a0841 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesRequest.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The request from clients to get node attributes in the cluster from the + * ResourceManager. + *

+ * + * @see ApplicationClientProtocol#getClusterNodeAttributes + * (GetClusterNodeAttributesRequest) + */ +@Public +@Evolving +public abstract class GetClusterNodeAttributesRequest { + + /** + * Create new instance of GetClusterNodeAttributesRequest. + * + * @return GetClusterNodeAttributesRequest is returned. + */ + public static GetClusterNodeAttributesRequest newInstance() { + return Records.newRecord(GetClusterNodeAttributesRequest.class); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java new file mode 100644 index 00000000000..b0ccd906a32 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The response sent by the ResourceManager to a client requesting + * a node attributes in cluster. + *

+ * + * @see ApplicationClientProtocol#getClusterNodeAttributes + * (GetClusterNodeAttributesRequest) + */ +@Public +@Evolving +public abstract class GetClusterNodeAttributesResponse { + + /** + * Create instance of GetClusterNodeAttributesResponse. + * + * @param attributes + * @return GetClusterNodeAttributesResponse. + */ + public static GetClusterNodeAttributesResponse newInstance( + Set attributes) { + GetClusterNodeAttributesResponse response = + Records.newRecord(GetClusterNodeAttributesResponse.class); + response.setNodeAttributes(attributes); + return response; + } + + /** + * Set node attributes to the response. + * + * @param attributes Map of Node attributeKey to Type. + */ + @Public + @Unstable + public abstract void setNodeAttributes(Set attributes); + + /** + * Get node attributes from the response. + * + * @return Node attributes. + */ + @Public + @Unstable + public abstract Set getNodeAttributes(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToAttributesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToAttributesRequest.java new file mode 100644 index 00000000000..8e91bcafed0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToAttributesRequest.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.util.Records; + +import java.util.Set; + +/** + *

+ * The request from clients to get nodes to attributes mapping + * in the cluster from the ResourceManager. + *

+ * + * @see ApplicationClientProtocol#getNodesToAttributes + * (GetNodesToAttributesRequest) + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class GetNodesToAttributesRequest { + + public static GetNodesToAttributesRequest newInstance(Set hostNames) { + GetNodesToAttributesRequest request = + Records.newRecord(GetNodesToAttributesRequest.class); + request.setHostNames(hostNames); + return request; + } + + /** + * Set hostnames for which mapping is required. + * + * @param hostnames + */ + @InterfaceAudience.Public + @InterfaceStability.Evolving + public abstract void setHostNames(Set hostnames); + + /** + * Get hostnames for which mapping is required. + * + * @return Set of hostnames. + */ + @InterfaceAudience.Public + @InterfaceStability.Evolving + public abstract Set getHostNames(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToAttributesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToAttributesResponse.java new file mode 100644 index 00000000000..acc07bb1847 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToAttributesResponse.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.util.Records; + +import java.util.Map; +import java.util.Set; + +/** + *

+ * The response sent by the ResourceManager to a client requesting + * nodes to attributes mapping. + *

+ * + * @see ApplicationClientProtocol#getNodesToAttributes + * (GetNodesToAttributesRequest) + */ +@Public +@Evolving +public abstract class GetNodesToAttributesResponse { + + public static GetNodesToAttributesResponse newInstance( + Map> map) { + GetNodesToAttributesResponse response = + Records.newRecord(GetNodesToAttributesResponse.class); + response.setNodeToAttributes(map); + return response; + } + + @Public + @Evolving + public abstract void setNodeToAttributes(Map> map); + + /** + * Get hostnames to NodeAttributes mapping. + * + * @return Map> host to attributes. + */ + @Public + @Evolving + public abstract Map> getNodeToAttributes(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java new file mode 100644 index 00000000000..70649390821 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * Node Attribute is a kind of a label which represents one of the + * attribute/feature of a Node. Its different from node partition label as + * resource guarantees across the queues will not be maintained for these type + * of labels. + *

+ *

+ * A given Node can be mapped with any kind of attribute, few examples are + * HAS_SSD=true, JAVA_VERSION=JDK1.8, OS_TYPE=WINDOWS. + *

+ *

+ * Its not compulsory for all the attributes to have value, empty string is the + * default value of the NodeAttributeType.STRING + *

+ *

+ * Node Attribute Prefix is used as namespace to segregate the attributes. + *

+ */ +@Public +@Unstable +public abstract class NodeAttribute { + + public static final String PREFIX_DISTRIBUTED = "nm.yarn.io"; + public static final String PREFIX_CENTRALIZED = "rm.yarn.io"; + + public static NodeAttribute newInstance(String attributeName, + NodeAttributeType attributeType, String attributeValue) { + return newInstance(PREFIX_CENTRALIZED, attributeName, attributeType, + attributeValue); + } + + public static NodeAttribute newInstance(String attributePrefix, + String attributeName, NodeAttributeType attributeType, + String attributeValue) { + NodeAttribute nodeAttribute = Records.newRecord(NodeAttribute.class); + NodeAttributeKey nodeAttributeKey = + NodeAttributeKey.newInstance(attributePrefix, attributeName); + nodeAttribute.setAttributeKey(nodeAttributeKey); + nodeAttribute.setAttributeType(attributeType); + nodeAttribute.setAttributeValue(attributeValue); + return nodeAttribute; + } + + @Public + @Unstable + public abstract NodeAttributeKey getAttributeKey(); + + @Public + @Unstable + public abstract void setAttributeKey(NodeAttributeKey attributeKey); + + @Public + @Unstable + public abstract String getAttributeValue(); + + @Public + @Unstable + public abstract void setAttributeValue(String attributeValue); + + @Public + @Unstable + public abstract NodeAttributeType getAttributeType(); + + @Public + @Unstable + public abstract void setAttributeType(NodeAttributeType attributeType); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeInfo.java new file mode 100644 index 00000000000..d294333ed1e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeInfo.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * Node Attribute Info describes a NodeAttribute. + *

+ */ +@Public +@Unstable +public abstract class NodeAttributeInfo { + + public static NodeAttributeInfo newInstance(NodeAttribute nodeAttribute) { + return newInstance(nodeAttribute.getAttributeKey(), + nodeAttribute.getAttributeType()); + } + + public static NodeAttributeInfo newInstance(NodeAttributeKey nodeAttributeKey, + NodeAttributeType attributeType) { + NodeAttributeInfo nodeAttribute = + Records.newRecord(NodeAttributeInfo.class); + nodeAttribute.setAttributeKey(nodeAttributeKey); + nodeAttribute.setAttributeType(attributeType); + return nodeAttribute; + } + + @Public + @Unstable + public abstract NodeAttributeKey getAttributeKey(); + + @Public + @Unstable + public abstract void setAttributeKey(NodeAttributeKey attributeKey); + + @Public + @Unstable + public abstract NodeAttributeType getAttributeType(); + + @Public + @Unstable + public abstract void setAttributeType(NodeAttributeType attributeType); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeKey.java new file mode 100644 index 00000000000..35ff26f07f1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeKey.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * Node AttributeKey uniquely identifies a given Node Attribute. Node Attribute + * is identified based on attribute prefix and name. + *

+ *

+ * Node Attribute Prefix is used as namespace to segregate the attributes. + *

+ */ +@Public +@Unstable +public abstract class NodeAttributeKey { + + public static NodeAttributeKey newInstance(String attributeName) { + return newInstance(NodeAttribute.PREFIX_CENTRALIZED, attributeName); + } + + public static NodeAttributeKey newInstance(String attributePrefix, + String attributeName) { + NodeAttributeKey nodeAttributeKey = + Records.newRecord(NodeAttributeKey.class); + nodeAttributeKey.setAttributePrefix(attributePrefix); + nodeAttributeKey.setAttributeName(attributeName); + return nodeAttributeKey; + } + + @Public + @Unstable + public abstract String getAttributePrefix(); + + @Public + @Unstable + public abstract void setAttributePrefix(String attributePrefix); + + @Public + @Unstable + public abstract String getAttributeName(); + + @Public + @Unstable + public abstract void setAttributeName(String attributeName); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeOpCode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeOpCode.java new file mode 100644 index 00000000000..76db063eed5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeOpCode.java @@ -0,0 +1,43 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; + +/** + * Enumeration of various node attribute op codes. + */ +@Public +@Evolving +public enum NodeAttributeOpCode { + /** + * Default as No OP. + */ + NO_OP, + /** + * EQUALS op code for Attribute. + */ + EQ, + + /** + * NOT EQUALS op code for Attribute. + */ + NE +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeType.java new file mode 100644 index 00000000000..3f281c81b19 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeType.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + *

+ * Type of a node Attribute. + *

+ * Based on this attribute expressions and values will be evaluated. + */ +@Public +@Unstable +public enum NodeAttributeType { + /** string type node attribute. */ + STRING +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java index 3a80641bb6d..625ad234081 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java @@ -258,4 +258,17 @@ public NodeUpdateType getNodeUpdateType() { * Set the node update type (null indicates absent node update type). * */ public void setNodeUpdateType(NodeUpdateType nodeUpdateType) {} + + /** + * Set the node attributes of node. + * + * @param nodeAttributes set of node attributes. + */ + public abstract void setNodeAttributes(Set nodeAttributes); + + /** + * Get node attributes of node. + * @return the set of node attributes. + */ + public abstract Set getNodeAttributes(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeToAttributeValue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeToAttributeValue.java new file mode 100644 index 00000000000..0bcb8b68b41 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeToAttributeValue.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * Mapping of Attribute Value to a Node. + *

+ */ +@Public +@Unstable +public abstract class NodeToAttributeValue { + public static NodeToAttributeValue newInstance(String hostname, + String attributeValue) { + NodeToAttributeValue nodeToAttributeValue = + Records.newRecord(NodeToAttributeValue.class); + nodeToAttributeValue.setAttributeValue(attributeValue); + nodeToAttributeValue.setHostname(hostname); + return nodeToAttributeValue; + } + + @Public + @Unstable + public abstract String getAttributeValue(); + + @Public + @Unstable + public abstract void setAttributeValue(String attributeValue); + + @Public + @Unstable + public abstract String getHostname(); + + @Public + @Unstable + public abstract void setHostname(String hostname); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java index 0fe8273e6d7..79196fbf851 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java @@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.NodeAttributeOpCode; /** * {@code PlacementConstraint} represents a placement constraint for a resource @@ -155,13 +156,22 @@ public String toString() { private int minCardinality; private int maxCardinality; private Set targetExpressions; + private NodeAttributeOpCode attributeOpCode; public SingleConstraint(String scope, int minCardinality, - int maxCardinality, Set targetExpressions) { + int maxCardinality, NodeAttributeOpCode opCode, + Set targetExpressions) { this.scope = scope; this.minCardinality = minCardinality; this.maxCardinality = maxCardinality; this.targetExpressions = targetExpressions; + this.attributeOpCode = opCode; + } + + public SingleConstraint(String scope, int minCardinality, + int maxCardinality, Set targetExpressions) { + this(scope, minCardinality, maxCardinality, NodeAttributeOpCode.NO_OP, + targetExpressions); } public SingleConstraint(String scope, int minC, int maxC, @@ -169,6 +179,13 @@ public SingleConstraint(String scope, int minC, int maxC, this(scope, minC, maxC, new HashSet<>(Arrays.asList(targetExpressions))); } + public SingleConstraint(String scope, int minC, int maxC, + NodeAttributeOpCode opCode, + TargetExpression... targetExpressions) { + this(scope, minC, maxC, opCode, + new HashSet<>(Arrays.asList(targetExpressions))); + } + /** * Get the scope of the constraint. * @@ -205,6 +222,15 @@ public int getMaxCardinality() { return targetExpressions; } + /** + * Get the NodeAttributeOpCode of the constraint. + * + * @return nodeAttribute Op Code + */ + public NodeAttributeOpCode getNodeAttributeOpCode() { + return attributeOpCode; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -225,6 +251,10 @@ public boolean equals(Object o) { if (!getScope().equals(that.getScope())) { return false; } + if (getNodeAttributeOpCode() != null && !getNodeAttributeOpCode() + .equals(that.getNodeAttributeOpCode())) { + return false; + } return getTargetExpressions().equals(that.getTargetExpressions()); } @@ -233,6 +263,7 @@ public int hashCode() { int result = getScope().hashCode(); result = 31 * result + getMinCardinality(); result = 31 * result + getMaxCardinality(); + result = 31 * result + getNodeAttributeOpCode().hashCode(); result = 31 * result + getTargetExpressions().hashCode(); return result; } @@ -259,6 +290,13 @@ public String toString() { .append(getScope()).append(",") .append(targetExpr) .toString()); + } else if (min == -1 && max == -1) { + // node attribute + targetConstraints.add(new StringBuilder() + .append(getScope()).append(",") + .append(getNodeAttributeOpCode()).append(",") + .append(targetExpr) + .toString()); } else { // cardinality targetConstraints.add(new StringBuilder() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java index d22a6bd90c0..73fa328833f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType; +import org.apache.hadoop.yarn.api.records.NodeAttributeOpCode; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr; @@ -85,6 +86,24 @@ public static AbstractConstraint targetNotIn(String scope, return new SingleConstraint(scope, 0, 0, targetExpressions); } + /** + * Creates a constraint that requires allocations to be placed on nodes that + * belong to a scope (e.g., node or rack) that satisfy any of the + * target expressions based on node attribute op code. + * + * @param scope the scope within which the target expressions should not be + * true + * @param opCode Node Attribute code which could be equals, not equals. + * @param targetExpressions the expressions that need to not be true within + * the scope + * @return the resulting placement constraint + */ + public static AbstractConstraint targetNodeAttribute(String scope, + NodeAttributeOpCode opCode, + TargetExpression... targetExpressions) { + return new SingleConstraint(scope, -1, -1, opCode, targetExpressions); + } + /** * Creates a constraint that restricts the number of allocations within a * given scope (e.g., node or rack). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 148edb9f26c..dd804a30937 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3463,6 +3463,22 @@ public static boolean isAclEnabled(Configuration conf) { public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX + "fs-store.root-dir"; + /** + * Node-attribute configurations. + */ + public static final String NODE_ATTRIBUTE_PREFIX = + YARN_PREFIX + "node-attribute."; + /** + * Node attribute store implementation class. + */ + public static final String FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS = + NODE_ATTRIBUTE_PREFIX + "fs-store.impl.class"; + /** + * File system node attribute store directory. + */ + public static final String FS_NODE_ATTRIBUTE_STORE_ROOT_DIR = + NODE_ATTRIBUTE_PREFIX + "fs-store.root-dir"; + /** * Flag to indicate if the node labels feature enabled, by default it's * disabled @@ -3525,16 +3541,25 @@ public static boolean areNodeLabelsEnabled( private static final String NM_NODE_LABELS_PREFIX = NM_PREFIX + "node-labels."; + private static final String NM_NODE_ATTRIBUTES_PREFIX = NM_PREFIX + + "node-attributes."; + public static final String NM_NODE_LABELS_PROVIDER_CONFIG = NM_NODE_LABELS_PREFIX + "provider"; + public static final String NM_NODE_ATTRIBUTES_PROVIDER_CONFIG = + NM_NODE_ATTRIBUTES_PREFIX + "provider"; + // whitelist names for the yarn.nodemanager.node-labels.provider - public static final String CONFIG_NODE_LABELS_PROVIDER = "config"; - public static final String SCRIPT_NODE_LABELS_PROVIDER = "script"; + public static final String CONFIG_NODE_DESCRIPTOR_PROVIDER = "config"; + public static final String SCRIPT_NODE_DESCRIPTOR_PROVIDER = "script"; private static final String NM_NODE_LABELS_PROVIDER_PREFIX = NM_NODE_LABELS_PREFIX + "provider."; + private static final String NM_NODE_ATTRIBUTES_PROVIDER_PREFIX = + NM_NODE_ATTRIBUTES_PREFIX + "provider."; + public static final String NM_NODE_LABELS_RESYNC_INTERVAL = NM_NODE_LABELS_PREFIX + "resync-interval-ms"; @@ -3559,6 +3584,9 @@ public static boolean areNodeLabelsEnabled( public static final String NM_PROVIDER_CONFIGURED_NODE_PARTITION = NM_NODE_LABELS_PROVIDER_PREFIX + "configured-node-partition"; + public static final String NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES = + NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "configured-node-attributes"; + private static final String RM_NODE_LABELS_PREFIX = RM_PREFIX + "node-labels."; @@ -3606,6 +3634,33 @@ public static boolean areNodeLabelsEnabled( NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX + "opts"; /** + * Node attribute provider fetch attributes interval and timeout. + */ + public static final String NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS = + NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "fetch-interval-ms"; + + public static final long + DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS = 10 * 60 * 1000; + + public static final String NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS = + NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "fetch-timeout-ms"; + + public static final long DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS + = DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS * 2; + + /** + * Script to collect node attributes. + */ + private static final String NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PREFIX = + NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "script."; + + public static final String NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PATH = + NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PREFIX + "path"; + + public static final String NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_OPTS = + NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PREFIX + "opts"; + + /* * Support to view apps for given user in secure cluster. * @deprecated This field is deprecated for {@link #FILTER_ENTITY_LIST_BY_USER} */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java index 852334245ce..58bb270f9cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java @@ -30,6 +30,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; @@ -37,6 +39,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesResponse; @@ -52,8 +56,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse; @Private public interface ResourceManagerAdministrationProtocol extends GetUserMappingsProtocol { @@ -144,4 +146,11 @@ public CheckForDecommissioningNodesResponse checkForDecommissioningNodes( public RefreshClusterMaxPriorityResponse refreshClusterMaxPriority( RefreshClusterMaxPriorityRequest request) throws YarnException, IOException; + + + @Private + @Idempotent + public NodesToAttributesMappingResponse mapAttributesToNodes( + NodesToAttributesMappingRequest request) throws YarnException, + IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AttributeMappingOperationType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AttributeMappingOperationType.java new file mode 100644 index 00000000000..5de15040503 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AttributeMappingOperationType.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + *

+ * Type of node to attribute mapping operation. + *

+ * + */ +@Public +@Unstable +public enum AttributeMappingOperationType { + /** Replaces the existing node to attribute mapping with new mapping.*/ + REPLACE, + + /** Add attribute(s) to a node and if it already exists will update the + * value.*/ + ADD, + + /** Removes attribute(s) mapped to a node. */ + REMOVE +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeToAttributes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeToAttributes.java new file mode 100644 index 00000000000..b2e38b4490f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeToAttributes.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.util.Records; + +/** + * Represents a mapping of Node id to list of attributes. + */ +@Public +@Unstable +public abstract class NodeToAttributes { + + public static NodeToAttributes newInstance(String node, + List attributes) { + NodeToAttributes nodeIdToAttributes = + Records.newRecord(NodeToAttributes.class); + nodeIdToAttributes.setNode(node); + nodeIdToAttributes.setNodeAttributes(attributes); + return nodeIdToAttributes; + } + + @Public + @Unstable + public abstract String getNode(); + + @Public + @Unstable + public abstract void setNode(String node); + + @Public + @Unstable + public abstract List getNodeAttributes(); + + @Public + @Unstable + public abstract void setNodeAttributes(List attributes); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingRequest.java new file mode 100644 index 00000000000..71421ed6665 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingRequest.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * list of node-attribute mapping request info. + */ +@Public +@Unstable +public abstract class NodesToAttributesMappingRequest { + + public static NodesToAttributesMappingRequest newInstance( + AttributeMappingOperationType operation, + List nodesToAttributes, boolean failOnUnknownNodes) { + NodesToAttributesMappingRequest request = + Records.newRecord(NodesToAttributesMappingRequest.class); + request.setNodesToAttributes(nodesToAttributes); + request.setFailOnUnknownNodes(failOnUnknownNodes); + request.setOperation(operation); + return request; + } + + @Public + @Unstable + public abstract void setNodesToAttributes( + List nodesToAttributes); + + @Public + @Unstable + public abstract List getNodesToAttributes(); + + @Public + @Unstable + public abstract void setFailOnUnknownNodes(boolean failOnUnknownNodes); + + @Public + @Unstable + public abstract boolean getFailOnUnknownNodes(); + + @Public + @Unstable + public abstract void setOperation(AttributeMappingOperationType operation); + + @Public + @Unstable + public abstract AttributeMappingOperationType getOperation(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingResponse.java new file mode 100644 index 00000000000..8e44adfc6a3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingResponse.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.yarn.util.Records; + +public class NodesToAttributesMappingResponse { + public static NodesToAttributesMappingResponse newInstance() { + return Records.newRecord(NodesToAttributesMappingResponse.class); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java index 2926c9d1de8..93fd706b0c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java @@ -19,6 +19,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.NodeAttributeOpCode; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraints; @@ -44,11 +45,12 @@ @InterfaceStability.Unstable public final class PlacementConstraintParser { + public static final char EXPRESSION_VAL_DELIM = ','; private static final char EXPRESSION_DELIM = ':'; private static final char KV_SPLIT_DELIM = '='; - private static final char EXPRESSION_VAL_DELIM = ','; private static final char BRACKET_START = '('; private static final char BRACKET_END = ')'; + private static final String KV_NE_DELIM = "!="; private static final String IN = "in"; private static final String NOT_IN = "notin"; private static final String AND = "and"; @@ -349,6 +351,91 @@ public String nextElement() { } } + /** + * Constraint parser used to parse a given target expression. + */ + public static class NodeConstraintParser extends ConstraintParser { + + public NodeConstraintParser(String expression) { + super(new BaseStringTokenizer(expression, + String.valueOf(EXPRESSION_VAL_DELIM))); + } + + @Override + public AbstractConstraint parse() + throws PlacementConstraintParseException { + PlacementConstraint.AbstractConstraint placementConstraints = null; + String attributeName = ""; + NodeAttributeOpCode opCode = NodeAttributeOpCode.EQ; + String scope = SCOPE_NODE; + + Set constraintEntities = new TreeSet<>(); + while (hasMoreTokens()) { + String currentTag = nextToken(); + StringTokenizer attributeKV = getAttributeOpCodeTokenizer(currentTag); + + // Usually there will be only one k=v pair. However in case when + // multiple values are present for same attribute, it will also be + // coming as next token. for example, java=1.8,1.9 or python!=2. + if (attributeKV.countTokens() > 1) { + opCode = getAttributeOpCode(currentTag); + attributeName = attributeKV.nextToken(); + currentTag = attributeKV.nextToken(); + } + constraintEntities.add(currentTag); + } + + if(attributeName.isEmpty()) { + throw new PlacementConstraintParseException( + "expecting valid expression like k=v or k!=v, but get " + + constraintEntities); + } + + PlacementConstraint.TargetExpression target = null; + if (!constraintEntities.isEmpty()) { + target = PlacementConstraints.PlacementTargets + .nodeAttribute(attributeName, + constraintEntities + .toArray(new String[constraintEntities.size()])); + } + + placementConstraints = PlacementConstraints + .targetNodeAttribute(scope, opCode, target); + return placementConstraints; + } + + private StringTokenizer getAttributeOpCodeTokenizer(String currentTag) { + StringTokenizer attributeKV = new StringTokenizer(currentTag, + KV_NE_DELIM); + + // Try with '!=' delim as well. + if (attributeKV.countTokens() < 2) { + attributeKV = new StringTokenizer(currentTag, + String.valueOf(KV_SPLIT_DELIM)); + } + return attributeKV; + } + + /** + * Below conditions are validated. + * java=8 : OpCode = EQUALS + * java!=8 : OpCode = NEQUALS + * @param currentTag tag + * @return Attribute op code. + */ + private NodeAttributeOpCode getAttributeOpCode(String currentTag) + throws PlacementConstraintParseException { + if (currentTag.contains(KV_NE_DELIM)) { + return NodeAttributeOpCode.NE; + } else if (currentTag.contains(String.valueOf(KV_SPLIT_DELIM))) { + return NodeAttributeOpCode.EQ; + } + throw new PlacementConstraintParseException( + "expecting valid expression like k=v or k!=v, but get " + + currentTag); + } + } + /** * Constraint parser used to parse a given target expression, such as * "NOTIN, NODE, foo, bar". @@ -363,20 +450,23 @@ public TargetConstraintParser(String expression) { @Override public AbstractConstraint parse() throws PlacementConstraintParseException { - PlacementConstraint.AbstractConstraint placementConstraints; + PlacementConstraint.AbstractConstraint placementConstraints = null; String op = nextToken(); if (op.equalsIgnoreCase(IN) || op.equalsIgnoreCase(NOT_IN)) { String scope = nextToken(); scope = parseScope(scope); - Set allocationTags = new TreeSet<>(); + Set constraintEntities = new TreeSet<>(); while(hasMoreTokens()) { String tag = nextToken(); - allocationTags.add(tag); + constraintEntities.add(tag); + } + PlacementConstraint.TargetExpression target = null; + if(!constraintEntities.isEmpty()) { + target = PlacementConstraints.PlacementTargets.allocationTag( + constraintEntities + .toArray(new String[constraintEntities.size()])); } - PlacementConstraint.TargetExpression target = - PlacementConstraints.PlacementTargets.allocationTag( - allocationTags.toArray(new String[allocationTags.size()])); if (op.equalsIgnoreCase(IN)) { placementConstraints = PlacementConstraints .targetIn(scope, target); @@ -550,6 +640,11 @@ public static AbstractConstraint parseExpression(String constraintStr) new ConjunctionConstraintParser(constraintStr); constraintOptional = Optional.ofNullable(jp.tryParse()); } + if (!constraintOptional.isPresent()) { + NodeConstraintParser np = + new NodeConstraintParser(constraintStr); + constraintOptional = Optional.ofNullable(np.tryParse()); + } if (!constraintOptional.isPresent()) { throw new PlacementConstraintParseException( "Invalid constraint expression " + constraintStr); @@ -584,12 +679,13 @@ public static AbstractConstraint parseExpression(String constraintStr) */ public static Map parsePlacementSpec( String expression) throws PlacementConstraintParseException { + // Continue handling for application tag based constraint otherwise. // Respect insertion order. Map result = new LinkedHashMap<>(); PlacementConstraintParser.ConstraintTokenizer tokenizer = new PlacementConstraintParser.MultipleConstraintsTokenizer(expression); tokenizer.validate(); - while(tokenizer.hasMoreElements()) { + while (tokenizer.hasMoreElements()) { String specStr = tokenizer.nextElement(); // each spec starts with sourceAllocationTag=numOfContainers and // followed by a constraint expression. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto index 81adef19335..fdd4bc5aca8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto @@ -64,4 +64,7 @@ service ApplicationClientProtocolService { rpc getResourceProfiles(GetAllResourceProfilesRequestProto) returns (GetAllResourceProfilesResponseProto); rpc getResourceProfile(GetResourceProfileRequestProto) returns (GetResourceProfileResponseProto); rpc getResourceTypeInfo(GetAllResourceTypeInfoRequestProto) returns (GetAllResourceTypeInfoResponseProto); + rpc getClusterNodeAttributes (GetClusterNodeAttributesRequestProto) returns (GetClusterNodeAttributesResponseProto); + rpc getAttributesToNodes (GetAttributesToNodesRequestProto) returns (GetAttributesToNodesResponseProto); + rpc getNodesToAttributes (GetNodesToAttributesRequestProto) returns (GetNodesToAttributesResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto index 113462305cd..032aa8e67b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto @@ -45,4 +45,5 @@ service ResourceManagerAdministrationProtocolService { rpc replaceLabelsOnNodes(ReplaceLabelsOnNodeRequestProto) returns (ReplaceLabelsOnNodeResponseProto); rpc checkForDecommissioningNodes(CheckForDecommissioningNodesRequestProto) returns (CheckForDecommissioningNodesResponseProto); rpc refreshClusterMaxPriority(RefreshClusterMaxPriorityRequestProto) returns (RefreshClusterMaxPriorityResponseProto); + rpc mapAttributesToNodes(NodesToAttributesMappingRequestProto) returns (NodesToAttributesMappingResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto index e8c92d962f3..d37e36a1878 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto @@ -130,6 +130,22 @@ enum DecommissionTypeProto { GRACEFUL = 2; FORCEFUL = 3; } + + +enum AttributeMappingOperationTypeProto { + REPLACE = 1; + ADD = 2; + REMOVE = 3; +} + +message NodesToAttributesMappingRequestProto { + optional AttributeMappingOperationTypeProto operation = 1 [default = REPLACE]; + repeated NodeToAttributesProto nodeToAttributes = 2; + optional bool failOnUnknownNodes = 3; +} + +message NodesToAttributesMappingResponseProto { +} ////////////////////////////////////////////////////////////////// ///////////// RM Failover related records //////////////////////// ////////////////////////////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index d6138e865ff..5fe2cc94550 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -355,6 +355,7 @@ message NodeReportProto { optional ResourceUtilizationProto node_utilization = 12; optional uint32 decommissioning_timeout = 13; optional NodeUpdateTypeProto node_update_type = 14; + repeated NodeAttributeProto node_attributes = 15; } message NodeIdToLabelsProto { @@ -372,6 +373,42 @@ message NodeLabelProto { optional bool isExclusive = 2 [default = true]; } +enum NodeAttributeTypeProto { + STRING = 1; +} + +message NodeAttributeKeyProto { + optional string attributePrefix = 1 [default="rm.yarn.io"]; + required string attributeName = 2; +} + +message NodeAttributeProto { + required NodeAttributeKeyProto attributeKey = 1; + optional NodeAttributeTypeProto attributeType = 2 [default = STRING]; + optional string attributeValue = 3 [default=""]; +} + + +message NodeAttributeInfoProto { + required NodeAttributeKeyProto attributeKey = 1; + required NodeAttributeTypeProto attributeType = 2; +} + +message NodeToAttributeValueProto { + required string hostname = 1; + required string attributeValue = 2; +} + +message AttributeToNodesProto { + required NodeAttributeKeyProto nodeAttribute = 1; + repeated NodeToAttributeValueProto nodeValueMap = 2; +} + +message NodeToAttributesProto { + optional string node = 1; + repeated NodeAttributeProto nodeAttributes = 2; +} + enum ContainerTypeProto { APPLICATION_MASTER = 1; TASK = 2; @@ -609,11 +646,18 @@ message PlacementConstraintProto { optional CompositePlacementConstraintProto compositeConstraint = 2; } +enum NodeAttributeOpCodeProto { + NO_OP = 1; + EQ = 2; + NE = 3; +} + message SimplePlacementConstraintProto { required string scope = 1; repeated PlacementConstraintTargetProto targetExpressions = 2; optional int32 minCardinality = 3; optional int32 maxCardinality = 4; + optional NodeAttributeOpCodeProto attributeOpCode = 5; } message PlacementConstraintTargetProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index acd452dc79f..248f775bdeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -260,6 +260,29 @@ message GetClusterNodeLabelsResponseProto { repeated NodeLabelProto nodeLabels = 2; } +message GetClusterNodeAttributesRequestProto { +} + +message GetClusterNodeAttributesResponseProto { + repeated NodeAttributeInfoProto nodeAttributes = 1; +} + +message GetAttributesToNodesRequestProto { + repeated NodeAttributeKeyProto nodeAttributes = 1; +} + +message GetAttributesToNodesResponseProto { + repeated AttributeToNodesProto attributesToNodes = 1; +} + +message GetNodesToAttributesRequestProto { + repeated string hostnames = 1; +} + +message GetNodesToAttributesResponseProto { + repeated NodeToAttributesProto nodesToAttributes = 1; +} + message UpdateApplicationPriorityRequestProto { required ApplicationIdProto applicationId = 1; required PriorityProto applicationPriority = 2; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java index a69571c5c80..9806ba4ac96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java @@ -22,6 +22,8 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; + +import org.apache.hadoop.yarn.api.records.NodeAttributeOpCode; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; @@ -38,8 +40,14 @@ import org.apache.hadoop.yarn.util.constraint.PlacementConstraintParser.SourceTagsTokenizer; import org.apache.hadoop.yarn.util.constraint.PlacementConstraintParser.ConstraintTokenizer; -import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.*; import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.and; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.cardinality; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.or; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetNodeAttribute; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetNotIn; import org.junit.Assert; import org.junit.Test; @@ -443,4 +451,55 @@ private void verifyConstraintToString(String inputExpr, + constrainExpr + ", caused by: " + e.getMessage()); } } + + @Test + public void testParseNodeAttributeSpec() + throws PlacementConstraintParseException { + Map result; + PlacementConstraint.AbstractConstraint expectedPc1, expectedPc2; + PlacementConstraint actualPc1, actualPc2; + + // A single node attribute constraint + result = PlacementConstraintParser + .parsePlacementSpec("xyz=4,rm.yarn.io/foo=true"); + Assert.assertEquals(1, result.size()); + TargetExpression target = PlacementTargets + .nodeAttribute("rm.yarn.io/foo", "true"); + expectedPc1 = targetNodeAttribute("node", NodeAttributeOpCode.EQ, target); + + actualPc1 = result.values().iterator().next(); + Assert.assertEquals(expectedPc1, actualPc1.getConstraintExpr()); + + // A single node attribute constraint + result = PlacementConstraintParser + .parsePlacementSpec("xyz=3,rm.yarn.io/foo!=abc"); + Assert.assertEquals(1, result.size()); + target = PlacementTargets + .nodeAttribute("rm.yarn.io/foo", "abc"); + expectedPc1 = targetNodeAttribute("node", NodeAttributeOpCode.NE, target); + + actualPc1 = result.values().iterator().next(); + Assert.assertEquals(expectedPc1, actualPc1.getConstraintExpr()); + + actualPc1 = result.values().iterator().next(); + Assert.assertEquals(expectedPc1, actualPc1.getConstraintExpr()); + + // A single node attribute constraint + result = PlacementConstraintParser + .parsePlacementSpec( + "xyz=1,rm.yarn.io/foo!=abc:zxy=1,rm.yarn.io/bar=true"); + Assert.assertEquals(2, result.size()); + target = PlacementTargets + .nodeAttribute("rm.yarn.io/foo", "abc"); + expectedPc1 = targetNodeAttribute("node", NodeAttributeOpCode.NE, target); + target = PlacementTargets + .nodeAttribute("rm.yarn.io/bar", "true"); + expectedPc2 = targetNodeAttribute("node", NodeAttributeOpCode.EQ, target); + + Iterator valueIt = result.values().iterator(); + actualPc1 = valueIt.next(); + actualPc2 = valueIt.next(); + Assert.assertEquals(expectedPc1, actualPc1.getConstraintExpr()); + Assert.assertEquals(expectedPc2, actualPc2.getConstraintExpr()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 76fa38f922a..f3693097515 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -523,9 +523,13 @@ public boolean init(String[] args) throws ParseException, IOException { if (cliParser.hasOption("placement_spec")) { String placementSpec = cliParser.getOptionValue("placement_spec"); - LOG.info("Placement Spec received [{}]", placementSpec); - parsePlacementSpecs(placementSpec); + String decodedSpec = getDecodedPlacementSpec(placementSpec); + LOG.info("Placement Spec received [{}]", decodedSpec); + + this.numTotalContainers = 0; + parsePlacementSpecs(decodedSpec); LOG.info("Total num containers requested [{}]", numTotalContainers); + if (numTotalContainers == 0) { throw new IllegalArgumentException( "Cannot run distributed shell with no containers"); @@ -694,23 +698,25 @@ public boolean init(String[] args) throws ParseException, IOException { return true; } - private void parsePlacementSpecs(String placementSpecifications) { - // Client sends placement spec in encoded format - Base64.Decoder decoder = Base64.getDecoder(); - byte[] decodedBytes = decoder.decode( - placementSpecifications.getBytes(StandardCharsets.UTF_8)); - String decodedSpec = new String(decodedBytes, StandardCharsets.UTF_8); - LOG.info("Decode placement spec: " + decodedSpec); + private void parsePlacementSpecs(String decodedSpec) { Map pSpecs = PlacementSpec.parse(decodedSpec); this.placementSpecs = new HashMap<>(); - this.numTotalContainers = 0; for (PlacementSpec pSpec : pSpecs.values()) { - this.numTotalContainers += pSpec.numContainers; + this.numTotalContainers += pSpec.getNumContainers(); this.placementSpecs.put(pSpec.sourceTag, pSpec); } } + private String getDecodedPlacementSpec(String placementSpecifications) { + Base64.Decoder decoder = Base64.getDecoder(); + byte[] decodedBytes = decoder.decode( + placementSpecifications.getBytes(StandardCharsets.UTF_8)); + String decodedSpec = new String(decodedBytes, StandardCharsets.UTF_8); + LOG.info("Decode placement spec: " + decodedSpec); + return decodedSpec; + } + /** * Helper function to print usage * @@ -798,6 +804,7 @@ public void run() throws YarnException, IOException, InterruptedException { } } } + RegisterApplicationMasterResponse response = amRMClient .registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl, placementConstraintMap); @@ -845,14 +852,18 @@ public void run() throws YarnException, IOException, InterruptedException { // Keep looping until all the containers are launched and shell script // executed on them ( regardless of success/failure). if (this.placementSpecs == null) { + LOG.info("placementSpecs null"); for (int i = 0; i < numTotalContainersToRequest; ++i) { ContainerRequest containerAsk = setupContainerAskForRM(); amRMClient.addContainerRequest(containerAsk); } } else { + LOG.info("placementSpecs to create req:" + placementSpecs); List schedReqs = new ArrayList<>(); for (PlacementSpec pSpec : this.placementSpecs.values()) { - for (int i = 0; i < pSpec.numContainers; i++) { + LOG.info("placementSpec :" + pSpec + ", container:" + pSpec + .getNumContainers()); + for (int i = 0; i < pSpec.getNumContainers(); i++) { SchedulingRequest sr = setupSchedulingRequest(pSpec); schedReqs.add(sr); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index c8a71b320c0..446b6088b0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -103,7 +103,7 @@ * the provided shell command on a set of containers.

* *

This client is meant to act as an example on how to write yarn-based applications.

- * + * *

To submit an application, a client first needs to connect to the ResourceManager * aka ApplicationsManager or ASM via the {@link ApplicationClientProtocol}. The {@link ApplicationClientProtocol} * provides a way for the client to get access to cluster information and to request for a @@ -192,6 +192,8 @@ // Placement specification private String placementSpec = ""; + // Node Attribute specification + private String nodeAttributeSpec = ""; // log4j.properties file // if available, add to local resources and set into classpath private String log4jPropFile = ""; @@ -448,6 +450,7 @@ public boolean init(String[] args) throws ParseException { // Check if it is parsable PlacementSpec.parse(this.placementSpec); } + appName = cliParser.getOptionValue("appname", "DistributedShell"); amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); amQueue = cliParser.getOptionValue("queue", "default"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/PlacementSpec.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/PlacementSpec.java index 290925980a5..ceaa37d5879 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/PlacementSpec.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/PlacementSpec.java @@ -37,8 +37,8 @@ LoggerFactory.getLogger(PlacementSpec.class); public final String sourceTag; - public final int numContainers; public final PlacementConstraint constraint; + private int numContainers; public PlacementSpec(String sourceTag, int numContainers, PlacementConstraint constraint) { @@ -47,6 +47,22 @@ public PlacementSpec(String sourceTag, int numContainers, this.constraint = constraint; } + /** + * Get the number of container for this spec. + * @return container count + */ + public int getNumContainers() { + return numContainers; + } + + /** + * Set number of containers for this spec. + * @param numContainers number of containers. + */ + public void setNumContainers(int numContainers) { + this.numContainers = numContainers; + } + // Placement specification should be of the form: // PlacementSpec => ""|KeyVal;PlacementSpec // KeyVal => SourceTag=Constraint @@ -71,6 +87,7 @@ public PlacementSpec(String sourceTag, int numContainers, public static Map parse(String specs) throws IllegalArgumentException { LOG.info("Parsing Placement Specs: [{}]", specs); + Map pSpecs = new HashMap<>(); Map parsed; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java index 92294468105..ca6cc508b27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java @@ -600,6 +600,26 @@ public String getStatusString(String appIdOrName) throws IOException, return output; } + @Override + public int actionUpgradeExpress(String appName, File path) + throws IOException, YarnException { + int result; + try { + Service service = + loadAppJsonFromLocalFS(path.getAbsolutePath(), appName, null, null); + service.setState(ServiceState.EXPRESS_UPGRADING); + String buffer = jsonSerDeser.toJson(service); + LOG.info("Upgrade in progress. Please wait.."); + ClientResponse response = getApiClient(getServicePath(appName)) + .put(ClientResponse.class, buffer); + result = processResponse(response); + } catch (Exception e) { + LOG.error("Failed to upgrade application: ", e); + result = EXIT_EXCEPTION_THROWN; + } + return result; + } + @Override public int initiateUpgrade(String appName, String fileName, boolean autoFinalize) throws IOException, YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index 4db0ac8f409..cd6f0d79e2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -440,7 +440,8 @@ public Response updateService(@Context HttpServletRequest request, if (updateServiceData.getState() != null && ( updateServiceData.getState() == ServiceState.UPGRADING || updateServiceData.getState() == - ServiceState.UPGRADING_AUTO_FINALIZE)) { + ServiceState.UPGRADING_AUTO_FINALIZE) || + updateServiceData.getState() == ServiceState.EXPRESS_UPGRADING) { return upgradeService(updateServiceData, ugi); } @@ -690,7 +691,11 @@ private Response upgradeService(Service service, ServiceClient sc = getServiceClient(); sc.init(YARN_CONFIG); sc.start(); - sc.initiateUpgrade(service); + if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) { + sc.actionUpgradeExpress(service); + } else { + sc.initiateUpgrade(service); + } sc.close(); return null; }); @@ -706,7 +711,8 @@ private Response processComponentsUpgrade(UserGroupInformation ugi, String serviceName, Set compNames) throws YarnException, IOException, InterruptedException { Service service = getServiceFromClient(ugi, serviceName); - if (service.getState() != ServiceState.UPGRADING) { + if (!service.getState().equals(ServiceState.UPGRADING) && + !service.getState().equals(ServiceState.UPGRADING_AUTO_FINALIZE)) { throw new YarnException( String.format("The upgrade of service %s has not been initiated.", service.getName())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java index 5bf183319fd..2ef8f7ee7b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java @@ -166,7 +166,7 @@ public UpgradeServiceResponseProto upgrade( LOG.info("Upgrading service to version {} by {}", request.getVersion(), UserGroupInformation.getCurrentUser()); context.getServiceManager().processUpgradeRequest(request.getVersion(), - request.getAutoFinalize()); + request.getAutoFinalize(), request.getExpressUpgrade()); return UpgradeServiceResponseProto.newBuilder().build(); } catch (Exception ex) { return UpgradeServiceResponseProto.newBuilder().setError(ex.getMessage()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java index 0196be2a989..3a55472c0c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java @@ -19,6 +19,9 @@ package org.apache.hadoop.yarn.service; import org.apache.hadoop.yarn.event.AbstractEvent; +import org.apache.hadoop.yarn.service.api.records.Component; + +import java.util.Queue; /** * Events are handled by {@link ServiceManager} to manage the service @@ -29,6 +32,8 @@ private final ServiceEventType type; private String version; private boolean autoFinalize; + private boolean expressUpgrade; + private Queue compsToUpgradeInOrder; public ServiceEvent(ServiceEventType serviceEventType) { super(serviceEventType); @@ -56,4 +61,24 @@ public ServiceEvent setAutoFinalize(boolean autoFinalize) { this.autoFinalize = autoFinalize; return this; } + + public boolean isExpressUpgrade() { + return expressUpgrade; + } + + public ServiceEvent setExpressUpgrade(boolean expressUpgrade) { + this.expressUpgrade = expressUpgrade; + return this; + } + + public Queue getCompsToUpgradeInOrder() { + return compsToUpgradeInOrder; + } + + public ServiceEvent setCompsToUpgradeInOrder( + Queue compsToUpgradeInOrder) { + this.compsToUpgradeInOrder = compsToUpgradeInOrder; + return this; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java index 05ecb3fc9be..04454b1d290 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.service.api.records.ComponentState; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.component.Component; @@ -40,8 +41,11 @@ import java.io.IOException; import java.text.MessageFormat; import java.util.EnumSet; +import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.concurrent.locks.ReentrantReadWriteLock; import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser; @@ -67,6 +71,8 @@ private final SliderFileSystem fs; private String upgradeVersion; + private Queue compsToUpgradeInOrder; private static final StateMachineFactory STATE_MACHINE_FACTORY = @@ -141,14 +147,20 @@ private State getState() { @Override public State transition(ServiceManager serviceManager, ServiceEvent event) { + serviceManager.upgradeVersion = event.getVersion(); try { - if (!event.isAutoFinalize()) { - serviceManager.serviceSpec.setState(ServiceState.UPGRADING); + if (event.isExpressUpgrade()) { + serviceManager.serviceSpec.setState(ServiceState.EXPRESS_UPGRADING); + serviceManager.compsToUpgradeInOrder = event + .getCompsToUpgradeInOrder(); + serviceManager.upgradeNextCompIfAny(); + } else if (event.isAutoFinalize()) { + serviceManager.serviceSpec.setState(ServiceState + .UPGRADING_AUTO_FINALIZE); } else { serviceManager.serviceSpec.setState( - ServiceState.UPGRADING_AUTO_FINALIZE); + ServiceState.UPGRADING); } - serviceManager.upgradeVersion = event.getVersion(); return State.UPGRADING; } catch (Throwable e) { LOG.error("[SERVICE]: Upgrade to version {} failed", event.getVersion(), @@ -169,8 +181,19 @@ public State transition(ServiceManager serviceManager, if (currState.equals(ServiceState.STABLE)) { return State.STABLE; } + if (currState.equals(ServiceState.EXPRESS_UPGRADING)) { + org.apache.hadoop.yarn.service.api.records.Component component = + serviceManager.compsToUpgradeInOrder.peek(); + if (!component.getState().equals(ComponentState.NEEDS_UPGRADE) && + !component.getState().equals(ComponentState.UPGRADING)) { + serviceManager.compsToUpgradeInOrder.remove(); + } + serviceManager.upgradeNextCompIfAny(); + } if (currState.equals(ServiceState.UPGRADING_AUTO_FINALIZE) || - event.getType().equals(ServiceEventType.START)) { + event.getType().equals(ServiceEventType.START) || + (currState.equals(ServiceState.EXPRESS_UPGRADING) && + serviceManager.compsToUpgradeInOrder.isEmpty())) { ServiceState targetState = checkIfStable(serviceManager.serviceSpec); if (targetState.equals(ServiceState.STABLE)) { if (serviceManager.finalizeUpgrade()) { @@ -184,6 +207,19 @@ public State transition(ServiceManager serviceManager, } } + private void upgradeNextCompIfAny() { + if (!compsToUpgradeInOrder.isEmpty()) { + org.apache.hadoop.yarn.service.api.records.Component component = + compsToUpgradeInOrder.peek(); + + ComponentEvent needUpgradeEvent = new ComponentEvent( + component.getName(), ComponentEventType.UPGRADE).setTargetSpec( + component).setUpgradeVersion(upgradeVersion).setExpressUpgrade(true); + context.scheduler.getDispatcher().getEventHandler().handle( + needUpgradeEvent); + } + } + /** * @return whether finalization of upgrade was successful. */ @@ -250,23 +286,18 @@ public void checkAndUpdateServiceState() { } void processUpgradeRequest(String upgradeVersion, - boolean autoFinalize) throws IOException { + boolean autoFinalize, boolean expressUpgrade) throws IOException { Service targetSpec = ServiceApiUtil.loadServiceUpgrade( context.fs, context.service.getName(), upgradeVersion); List - compsThatNeedUpgrade = componentsFinder. + compsNeedUpgradeList = componentsFinder. findTargetComponentSpecs(context.service, targetSpec); - ServiceEvent event = new ServiceEvent(ServiceEventType.UPGRADE) - .setVersion(upgradeVersion) - .setAutoFinalize(autoFinalize); - context.scheduler.getDispatcher().getEventHandler().handle(event); - if (compsThatNeedUpgrade != null && !compsThatNeedUpgrade.isEmpty()) { - if (autoFinalize) { - event.setAutoFinalize(true); - } - compsThatNeedUpgrade.forEach(component -> { + // remove all components from need upgrade list if there restart policy + // doesn't all upgrade. + if (compsNeedUpgradeList != null) { + compsNeedUpgradeList.removeIf(component -> { org.apache.hadoop.yarn.service.api.records.Component.RestartPolicyEnum restartPolicy = component.getRestartPolicy(); @@ -274,25 +305,65 @@ void processUpgradeRequest(String upgradeVersion, Component.getRestartPolicyHandler(restartPolicy); // Do not allow upgrades for components which have NEVER/ON_FAILURE // restart policy - if (restartPolicyHandler.allowUpgrades()) { + if (!restartPolicyHandler.allowUpgrades()) { + LOG.info("The component {} has a restart policy that doesnt " + + "allow upgrades {} ", component.getName(), + component.getRestartPolicy().toString()); + return true; + } + + return false; + }); + } + + ServiceEvent event = new ServiceEvent(ServiceEventType.UPGRADE) + .setVersion(upgradeVersion) + .setAutoFinalize(autoFinalize) + .setExpressUpgrade(expressUpgrade); + + if (expressUpgrade) { + // In case of express upgrade components need to be upgraded in order. + // Once the service manager gets notified that a component finished + // upgrading, it then issues event to upgrade the next component. + Map + compsNeedUpgradeByName = new HashMap<>(); + if (compsNeedUpgradeList != null) { + compsNeedUpgradeList.forEach(component -> + compsNeedUpgradeByName.put(component.getName(), component)); + } + List resolvedComps = ServiceApiUtil + .resolveCompsDependency(targetSpec); + + Queue + orderedCompUpgrade = new LinkedList<>(); + resolvedComps.forEach(compName -> { + org.apache.hadoop.yarn.service.api.records.Component component = + compsNeedUpgradeByName.get(compName); + if (component != null ) { + orderedCompUpgrade.add(component); + } + }); + event.setCompsToUpgradeInOrder(orderedCompUpgrade); + } + + context.scheduler.getDispatcher().getEventHandler().handle(event); + + if (compsNeedUpgradeList != null && !compsNeedUpgradeList.isEmpty()) { + if (!expressUpgrade) { + compsNeedUpgradeList.forEach(component -> { ComponentEvent needUpgradeEvent = new ComponentEvent( component.getName(), ComponentEventType.UPGRADE).setTargetSpec( component).setUpgradeVersion(event.getVersion()); context.scheduler.getDispatcher().getEventHandler().handle( needUpgradeEvent); - } else { - LOG.info("The component {} has a restart " - + "policy that doesnt allow upgrades {} ", component.getName(), - component.getRestartPolicy().toString()); - } - }); - } else { + + }); + } + } else if (autoFinalize) { // nothing to upgrade if upgrade auto finalize is requested, trigger a // state check. - if (autoFinalize) { - context.scheduler.getDispatcher().getEventHandler().handle( - new ServiceEvent(ServiceEventType.CHECK_STABLE)); - } + context.scheduler.getDispatcher().getEventHandler().handle( + new ServiceEvent(ServiceEventType.CHECK_STABLE)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 0801ad052db..384659f8aba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -219,7 +219,7 @@ public void buildInstance(ServiceContext context, Configuration configuration) nmClient.getClient().cleanupRunningContainersOnStop(false); addIfService(nmClient); - dispatcher = new AsyncDispatcher("Component dispatcher"); + dispatcher = createAsyncDispatcher(); dispatcher.register(ServiceEventType.class, new ServiceEventHandler()); dispatcher.register(ComponentEventType.class, new ComponentEventHandler()); @@ -253,6 +253,9 @@ public void buildInstance(ServiceContext context, Configuration configuration) YarnServiceConf.CONTAINER_RECOVERY_TIMEOUT_MS, YarnServiceConf.DEFAULT_CONTAINER_RECOVERY_TIMEOUT_MS, app.getConfiguration(), getConfig()); + + serviceManager = createServiceManager(); + context.setServiceManager(serviceManager); } protected YarnRegistryViewForProviders createYarnRegistryOperations( @@ -262,6 +265,14 @@ protected YarnRegistryViewForProviders createYarnRegistryOperations( context.attemptId); } + protected ServiceManager createServiceManager() { + return new ServiceManager(context); + } + + protected AsyncDispatcher createAsyncDispatcher() { + return new AsyncDispatcher("Component dispatcher"); + } + protected NMClientAsync createNMClient() { return NMClientAsync.createNMClientAsync(new NMClientCallback()); } @@ -344,8 +355,6 @@ public void serviceStart() throws Exception { // Since AM has been started and registered, the service is in STARTED state app.setState(ServiceState.STARTED); - serviceManager = new ServiceManager(context); - context.setServiceManager(serviceManager); // recover components based on containers sent from RM recoverComponents(response); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java index b6ae38bdeee..0b3c0377fab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java @@ -30,5 +30,5 @@ @javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") public enum ServiceState { ACCEPTED, STARTED, STABLE, STOPPED, FAILED, FLEX, UPGRADING, - UPGRADING_AUTO_FINALIZE; + UPGRADING_AUTO_FINALIZE, EXPRESS_UPGRADING; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 5668d9fa3ae..a27ed87aa63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.service.client; import com.google.common.annotations.VisibleForTesting; + import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -215,48 +216,31 @@ public int actionBuild(Service service) return EXIT_SUCCESS; } - @Override - public int initiateUpgrade(String appName, String fileName, - boolean autoFinalize) - throws IOException, YarnException { - Service upgradeService = loadAppJsonFromLocalFS(fileName, appName, - null, null); - if (autoFinalize) { - upgradeService.setState(ServiceState.UPGRADING_AUTO_FINALIZE); - } else { - upgradeService.setState(ServiceState.UPGRADING); - } - return initiateUpgrade(upgradeService); - } - - public int initiateUpgrade(Service service) throws YarnException, - IOException { + private ApplicationReport upgradePrecheck(Service service) + throws YarnException, IOException { boolean upgradeEnabled = getConfig().getBoolean( - YARN_SERVICE_UPGRADE_ENABLED, - YARN_SERVICE_UPGRADE_ENABLED_DEFAULT); + YARN_SERVICE_UPGRADE_ENABLED, YARN_SERVICE_UPGRADE_ENABLED_DEFAULT); if (!upgradeEnabled) { throw new YarnException(ErrorStrings.SERVICE_UPGRADE_DISABLED); } - Service persistedService = - ServiceApiUtil.loadService(fs, service.getName()); + Service persistedService = ServiceApiUtil.loadService(fs, + service.getName()); if (!StringUtils.isEmpty(persistedService.getId())) { - cachedAppInfo.put(persistedService.getName(), new AppInfo( - ApplicationId.fromString(persistedService.getId()), - persistedService.getKerberosPrincipal().getPrincipalName())); + cachedAppInfo.put(persistedService.getName(), + new AppInfo(ApplicationId.fromString(persistedService.getId()), + persistedService.getKerberosPrincipal().getPrincipalName())); } if (persistedService.getVersion().equals(service.getVersion())) { - String message = - service.getName() + " is already at version " + service.getVersion() - + ". There is nothing to upgrade."; + String message = service.getName() + " is already at version " + + service.getVersion() + ". There is nothing to upgrade."; LOG.error(message); throw new YarnException(message); } Service liveService = getStatus(service.getName()); if (!liveService.getState().equals(ServiceState.STABLE)) { - String message = service.getName() + " is at " + - liveService.getState() + String message = service.getName() + " is at " + liveService.getState() + " state and upgrade can only be initiated when service is STABLE."; LOG.error(message); throw new YarnException(message); @@ -266,11 +250,67 @@ public int initiateUpgrade(Service service) throws YarnException, ServiceApiUtil.validateAndResolveService(service, fs, getConfig()); ServiceApiUtil.createDirAndPersistApp(fs, serviceUpgradeDir, service); - ApplicationReport appReport = - yarnClient.getApplicationReport(getAppId(service.getName())); + ApplicationReport appReport = yarnClient + .getApplicationReport(getAppId(service.getName())); if (StringUtils.isEmpty(appReport.getHost())) { throw new YarnException(service.getName() + " AM hostname is empty"); } + return appReport; + } + + @Override + public int actionUpgradeExpress(String appName, File path) + throws IOException, YarnException { + Service service = + loadAppJsonFromLocalFS(path.getAbsolutePath(), appName, null, null); + service.setState(ServiceState.UPGRADING_AUTO_FINALIZE); + actionUpgradeExpress(service); + return EXIT_SUCCESS; + } + + public int actionUpgradeExpress(Service service) throws YarnException, + IOException { + ApplicationReport appReport = upgradePrecheck(service); + ClientAMProtocol proxy = createAMProxy(service.getName(), appReport); + UpgradeServiceRequestProto.Builder requestBuilder = + UpgradeServiceRequestProto.newBuilder(); + requestBuilder.setVersion(service.getVersion()); + if (service.getState().equals(ServiceState.UPGRADING_AUTO_FINALIZE)) { + requestBuilder.setAutoFinalize(true); + } + if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) { + requestBuilder.setExpressUpgrade(true); + requestBuilder.setAutoFinalize(true); + } + UpgradeServiceResponseProto responseProto = proxy.upgrade( + requestBuilder.build()); + if (responseProto.hasError()) { + LOG.error("Service {} express upgrade to version {} failed because {}", + service.getName(), service.getVersion(), responseProto.getError()); + throw new YarnException("Failed to express upgrade service " + + service.getName() + " to version " + service.getVersion() + + " because " + responseProto.getError()); + } + return EXIT_SUCCESS; + } + + @Override + public int initiateUpgrade(String appName, String fileName, + boolean autoFinalize) + throws IOException, YarnException { + Service upgradeService = loadAppJsonFromLocalFS(fileName, appName, + null, null); + if (autoFinalize) { + upgradeService.setState(ServiceState.UPGRADING_AUTO_FINALIZE); + } else { + upgradeService.setState(ServiceState.UPGRADING); + } + return initiateUpgrade(upgradeService); + } + + public int initiateUpgrade(Service service) throws YarnException, + IOException { + ApplicationReport appReport = upgradePrecheck(service); ClientAMProtocol proxy = createAMProxy(service.getName(), appReport); UpgradeServiceRequestProto.Builder requestBuilder = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java index 41a2fcd104b..acf3404fe93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.ExecutionType; import static org.apache.hadoop.yarn.service.api.records.Component @@ -43,6 +44,7 @@ import org.apache.hadoop.yarn.service.api.records.ContainerState; import org.apache.hadoop.yarn.service.api.records.ResourceInformation; import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType; import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId; import org.apache.hadoop.yarn.service.ContainerFailureTracker; import org.apache.hadoop.yarn.service.ServiceContext; @@ -546,13 +548,21 @@ public void transition(Component component, ComponentEvent event) { @Override public void transition(Component component, ComponentEvent event) { component.upgradeInProgress.set(true); + component.upgradeEvent = event; component.componentSpec.setState(org.apache.hadoop.yarn.service.api. records.ComponentState.NEEDS_UPGRADE); component.numContainersThatNeedUpgrade.set( component.componentSpec.getNumberOfContainers()); - component.componentSpec.getContainers().forEach(container -> - container.setState(ContainerState.NEEDS_UPGRADE)); - component.upgradeEvent = event; + component.componentSpec.getContainers().forEach(container -> { + container.setState(ContainerState.NEEDS_UPGRADE); + if (event.isExpressUpgrade()) { + ComponentInstanceEvent upgradeEvent = new ComponentInstanceEvent( + ContainerId.fromString(container.getId()), + ComponentInstanceEventType.UPGRADE); + LOG.info("Upgrade container {}", container.getId()); + component.dispatcher.getEventHandler().handle(upgradeEvent); + } + }); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java index 84caa77b205..643961d505a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java @@ -35,6 +35,7 @@ private ContainerId containerId; private org.apache.hadoop.yarn.service.api.records.Component targetSpec; private String upgradeVersion; + private boolean expressUpgrade; public ContainerId getContainerId() { return containerId; @@ -113,4 +114,13 @@ public ComponentEvent setUpgradeVersion(String upgradeVersion) { this.upgradeVersion = upgradeVersion; return this; } + + public boolean isExpressUpgrade() { + return expressUpgrade; + } + + public ComponentEvent setExpressUpgrade(boolean expressUpgrade) { + this.expressUpgrade = expressUpgrade; + return this; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java index 11a6caa901d..ed5e68e98f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java @@ -380,6 +380,11 @@ public void transition(ComponentInstance compInstance, @Override public void transition(ComponentInstance compInstance, ComponentInstanceEvent event) { + if (!compInstance.containerSpec.getState().equals( + ContainerState.NEEDS_UPGRADE)) { + //nothing to upgrade. this may happen with express upgrade. + return; + } compInstance.containerSpec.setState(ContainerState.UPGRADING); compInstance.component.decContainersReady(false); ComponentEvent upgradeEvent = compInstance.component.getUpgradeEvent(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java index 92195693274..b588e88ae7f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java @@ -638,6 +638,32 @@ public static void validateInstancesUpgrade(List return containerNeedUpgrade; } + /** + * Validates the components that are requested are stable for upgrade. + * It returns the instances of the components which are in ready state. + */ + public static List validateAndResolveCompsStable( + Service liveService, Collection compNames) throws YarnException { + Preconditions.checkNotNull(compNames); + HashSet requestedComps = Sets.newHashSet(compNames); + List containerNeedUpgrade = new ArrayList<>(); + for (Component liveComp : liveService.getComponents()) { + if (requestedComps.contains(liveComp.getName())) { + if (!liveComp.getState().equals(ComponentState.STABLE)) { + // Nothing to upgrade + throw new YarnException(String.format( + ERROR_COMP_DOES_NOT_NEED_UPGRADE, liveComp.getName())); + } + liveComp.getContainers().forEach(liveContainer -> { + if (liveContainer.getState().equals(ContainerState.READY)) { + containerNeedUpgrade.add(liveContainer); + } + }); + } + } + return containerNeedUpgrade; + } + private static String parseComponentName(String componentInstanceName) throws YarnException { int idx = componentInstanceName.lastIndexOf('-'); @@ -651,4 +677,22 @@ private static String parseComponentName(String componentInstanceName) public static String $(String s) { return "${" + s +"}"; } + + public static List resolveCompsDependency(Service service) { + List components = new ArrayList(); + for (Component component : service.getComponents()) { + int depSize = component.getDependencies().size(); + if (!components.contains(component.getName())) { + components.add(component.getName()); + } + if (depSize != 0) { + for (String depComp : component.getDependencies()) { + if (!components.contains(depComp)) { + components.add(0, depComp); + } + } + } + } + return components; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto index 6166dedd1de..169f765b8a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto @@ -66,6 +66,7 @@ message StopResponseProto { message UpgradeServiceRequestProto { optional string version = 1; optional bool autoFinalize = 2; + optional bool expressUpgrade = 3; } message UpgradeServiceResponseProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java index fc509f19420..a37cabe38c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java @@ -19,23 +19,26 @@ package org.apache.hadoop.yarn.service; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.ComponentState; +import org.apache.hadoop.yarn.service.api.records.ContainerState; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType; import org.apache.hadoop.yarn.service.exceptions.SliderException; -import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import java.io.IOException; -import java.util.Map; - -import static org.mockito.Mockito.mock; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeoutException; /** * Tests for {@link ServiceManager}. @@ -46,117 +49,120 @@ public ServiceTestUtils.ServiceFSWatcher rule = new ServiceTestUtils.ServiceFSWatcher(); - @Test - public void testUpgrade() throws IOException, SliderException { - ServiceManager serviceManager = createTestServiceManager("testUpgrade"); - upgrade(serviceManager, "v2", false, false); + @Test (timeout = TIMEOUT) + public void testUpgrade() throws Exception { + ServiceContext context = createServiceContext("testUpgrade"); + initUpgrade(context, "v2", false, false, false); Assert.assertEquals("service not upgraded", ServiceState.UPGRADING, - serviceManager.getServiceSpec().getState()); + context.getServiceManager().getServiceSpec().getState()); } - @Test + @Test (timeout = TIMEOUT) public void testRestartNothingToUpgrade() - throws IOException, SliderException { - ServiceManager serviceManager = createTestServiceManager( + throws Exception { + ServiceContext context = createServiceContext( "testRestartNothingToUpgrade"); - upgrade(serviceManager, "v2", false, false); - - //make components stable - serviceManager.getServiceSpec().getComponents().forEach(comp -> { - comp.setState(ComponentState.STABLE); - }); - serviceManager.handle(new ServiceEvent(ServiceEventType.START)); + initUpgrade(context, "v2", false, false, false); + ServiceManager manager = context.getServiceManager(); + //make components stable by upgrading all instances + upgradeAllInstances(context); + + context.scheduler.getDispatcher().getEventHandler().handle( + new ServiceEvent(ServiceEventType.START)); + GenericTestUtils.waitFor(()-> + context.service.getState().equals(ServiceState.STABLE), + CHECK_EVERY_MILLIS, TIMEOUT); Assert.assertEquals("service not re-started", ServiceState.STABLE, - serviceManager.getServiceSpec().getState()); + manager.getServiceSpec().getState()); } - @Test - public void testAutoFinalizeNothingToUpgrade() throws IOException, - SliderException { - ServiceManager serviceManager = createTestServiceManager( + @Test(timeout = TIMEOUT) + public void testAutoFinalizeNothingToUpgrade() throws Exception { + ServiceContext context = createServiceContext( "testAutoFinalizeNothingToUpgrade"); - upgrade(serviceManager, "v2", false, true); - - //make components stable - serviceManager.getServiceSpec().getComponents().forEach(comp -> - comp.setState(ComponentState.STABLE)); - serviceManager.handle(new ServiceEvent(ServiceEventType.CHECK_STABLE)); + initUpgrade(context, "v2", false, true, false); + ServiceManager manager = context.getServiceManager(); + //make components stable by upgrading all instances + upgradeAllInstances(context); + + GenericTestUtils.waitFor(()-> + context.service.getState().equals(ServiceState.STABLE), + CHECK_EVERY_MILLIS, TIMEOUT); Assert.assertEquals("service stable", ServiceState.STABLE, - serviceManager.getServiceSpec().getState()); + manager.getServiceSpec().getState()); } - @Test + @Test(timeout = TIMEOUT) public void testRestartWithPendingUpgrade() - throws IOException, SliderException { - ServiceManager serviceManager = createTestServiceManager("testRestart"); - upgrade(serviceManager, "v2", true, false); - serviceManager.handle(new ServiceEvent(ServiceEventType.START)); + throws Exception { + ServiceContext context = createServiceContext("testRestart"); + initUpgrade(context, "v2", true, false, false); + ServiceManager manager = context.getServiceManager(); + + context.scheduler.getDispatcher().getEventHandler().handle( + new ServiceEvent(ServiceEventType.START)); + context.scheduler.getDispatcher().stop(); Assert.assertEquals("service should still be upgrading", - ServiceState.UPGRADING, serviceManager.getServiceSpec().getState()); + ServiceState.UPGRADING, manager.getServiceSpec().getState()); } - @Test - public void testCheckState() throws IOException, SliderException { - ServiceManager serviceManager = createTestServiceManager( - "testCheckState"); - upgrade(serviceManager, "v2", true, false); + @Test(timeout = TIMEOUT) + public void testFinalize() throws Exception { + ServiceContext context = createServiceContext("testCheckState"); + initUpgrade(context, "v2", true, false, false); + ServiceManager manager = context.getServiceManager(); Assert.assertEquals("service not upgrading", ServiceState.UPGRADING, - serviceManager.getServiceSpec().getState()); + manager.getServiceSpec().getState()); - // make components stable - serviceManager.getServiceSpec().getComponents().forEach(comp -> { - comp.setState(ComponentState.STABLE); - }); - ServiceEvent checkStable = new ServiceEvent(ServiceEventType.CHECK_STABLE); - serviceManager.handle(checkStable); - Assert.assertEquals("service should still be upgrading", - ServiceState.UPGRADING, serviceManager.getServiceSpec().getState()); + //make components stable by upgrading all instances + upgradeAllInstances(context); // finalize service - ServiceEvent restart = new ServiceEvent(ServiceEventType.START); - serviceManager.handle(restart); - Assert.assertEquals("service not stable", - ServiceState.STABLE, serviceManager.getServiceSpec().getState()); + context.scheduler.getDispatcher().getEventHandler().handle( + new ServiceEvent(ServiceEventType.START)); + GenericTestUtils.waitFor(()-> + context.service.getState().equals(ServiceState.STABLE), + CHECK_EVERY_MILLIS, TIMEOUT); + Assert.assertEquals("service not re-started", ServiceState.STABLE, + manager.getServiceSpec().getState()); - validateUpgradeFinalization(serviceManager.getName(), "v2"); + validateUpgradeFinalization(manager.getName(), "v2"); } - @Test - public void testCheckStateAutoFinalize() throws IOException, SliderException { - ServiceManager serviceManager = createTestServiceManager( - "testCheckState"); - serviceManager.getServiceSpec().setState( + @Test(timeout = TIMEOUT) + public void testAutoFinalize() throws Exception { + ServiceContext context = createServiceContext("testCheckStateAutoFinalize"); + ServiceManager manager = context.getServiceManager(); + manager.getServiceSpec().setState( ServiceState.UPGRADING_AUTO_FINALIZE); - upgrade(serviceManager, "v2", true, true); - Assert.assertEquals("service not upgrading", - ServiceState.UPGRADING_AUTO_FINALIZE, - serviceManager.getServiceSpec().getState()); + initUpgrade(context, "v2", true, true, false); // make components stable - serviceManager.getServiceSpec().getComponents().forEach(comp -> - comp.setState(ComponentState.STABLE)); - ServiceEvent checkStable = new ServiceEvent(ServiceEventType.CHECK_STABLE); - serviceManager.handle(checkStable); + upgradeAllInstances(context); + + GenericTestUtils.waitFor(() -> + context.service.getState().equals(ServiceState.STABLE), + CHECK_EVERY_MILLIS, TIMEOUT); Assert.assertEquals("service not stable", - ServiceState.STABLE, serviceManager.getServiceSpec().getState()); + ServiceState.STABLE, manager.getServiceSpec().getState()); - validateUpgradeFinalization(serviceManager.getName(), "v2"); + validateUpgradeFinalization(manager.getName(), "v2"); } @Test - public void testInvalidUpgrade() throws IOException, SliderException { - ServiceManager serviceManager = createTestServiceManager( - "testInvalidUpgrade"); - serviceManager.getServiceSpec().setState( + public void testInvalidUpgrade() throws Exception { + ServiceContext serviceContext = createServiceContext("testInvalidUpgrade"); + ServiceManager manager = serviceContext.getServiceManager(); + manager.getServiceSpec().setState( ServiceState.UPGRADING_AUTO_FINALIZE); Service upgradedDef = ServiceTestUtils.createExampleApplication(); - upgradedDef.setName(serviceManager.getName()); + upgradedDef.setName(manager.getName()); upgradedDef.setVersion("v2"); upgradedDef.setLifetime(2L); writeUpgradedDef(upgradedDef); try { - serviceManager.processUpgradeRequest("v2", true); + manager.processUpgradeRequest("v2", true, false); } catch (Exception ex) { Assert.assertTrue(ex instanceof UnsupportedOperationException); return; @@ -164,6 +170,32 @@ public void testInvalidUpgrade() throws IOException, SliderException { Assert.fail(); } + @Test(timeout = TIMEOUT) + public void testExpressUpgrade() throws Exception { + ServiceContext context = createServiceContext("testExpressUpgrade"); + ServiceManager manager = context.getServiceManager(); + manager.getServiceSpec().setState( + ServiceState.EXPRESS_UPGRADING); + initUpgrade(context, "v2", true, true, true); + + List comps = ServiceApiUtil.resolveCompsDependency(context.service); + // wait till instances of first component are in upgrade + String comp1 = comps.get(0); + upgradeInstancesOf(context, comp1); + + // wait till instances of second component are in upgrade + String comp2 = comps.get(1); + upgradeInstancesOf(context, comp2); + + GenericTestUtils.waitFor(() -> + context.service.getState().equals(ServiceState.STABLE), + CHECK_EVERY_MILLIS, TIMEOUT); + + Assert.assertEquals("service not stable", + ServiceState.STABLE, manager.getServiceSpec().getState()); + validateUpgradeFinalization(manager.getName(), "v2"); + } + private void validateUpgradeFinalization(String serviceName, String expectedVersion) throws IOException { Service savedSpec = ServiceApiUtil.loadService(rule.getFs(), serviceName); @@ -172,15 +204,16 @@ private void validateUpgradeFinalization(String serviceName, Assert.assertNotNull("app id not present", savedSpec.getId()); Assert.assertEquals("state not stable", ServiceState.STABLE, savedSpec.getState()); - savedSpec.getComponents().forEach(compSpec -> { - Assert.assertEquals("comp not stable", ComponentState.STABLE, - compSpec.getState()); - }); + savedSpec.getComponents().forEach(compSpec -> + Assert.assertEquals("comp not stable", ComponentState.STABLE, + compSpec.getState())); } - private void upgrade(ServiceManager serviceManager, String version, - boolean upgradeArtifact, boolean autoFinalize) - throws IOException, SliderException { + private void initUpgrade(ServiceContext context, String version, + boolean upgradeArtifact, boolean autoFinalize, boolean expressUpgrade) + throws IOException, SliderException, TimeoutException, + InterruptedException { + ServiceManager serviceManager = context.getServiceManager(); Service upgradedDef = ServiceTestUtils.createExampleApplication(); upgradedDef.setName(serviceManager.getName()); upgradedDef.setVersion(version); @@ -191,39 +224,81 @@ private void upgrade(ServiceManager serviceManager, String version, }); } writeUpgradedDef(upgradedDef); - serviceManager.processUpgradeRequest(version, autoFinalize); + serviceManager.processUpgradeRequest(version, autoFinalize, expressUpgrade); ServiceEvent upgradeEvent = new ServiceEvent(ServiceEventType.UPGRADE); - upgradeEvent.setVersion(version); - if (autoFinalize) { - upgradeEvent.setAutoFinalize(true); - } - serviceManager.handle(upgradeEvent); + upgradeEvent.setVersion(version).setExpressUpgrade(expressUpgrade) + .setAutoFinalize(autoFinalize); + + GenericTestUtils.waitFor(()-> { + ServiceState serviceState = context.service.getState(); + if (serviceState.equals(ServiceState.UPGRADING) || + serviceState.equals(ServiceState.UPGRADING_AUTO_FINALIZE) || + serviceState.equals(ServiceState.EXPRESS_UPGRADING)) { + return true; + } + return false; + }, CHECK_EVERY_MILLIS, TIMEOUT); + } + + private void upgradeAllInstances(ServiceContext context) throws + TimeoutException, InterruptedException { + // upgrade the instances + context.scheduler.getLiveInstances().forEach(((containerId, instance) -> { + ComponentInstanceEvent event = new ComponentInstanceEvent(containerId, + ComponentInstanceEventType.UPGRADE); + context.scheduler.getDispatcher().getEventHandler().handle(event); + })); + + // become ready + context.scheduler.getLiveInstances().forEach(((containerId, instance) -> { + ComponentInstanceEvent event = new ComponentInstanceEvent(containerId, + ComponentInstanceEventType.BECOME_READY); + + context.scheduler.getDispatcher().getEventHandler().handle(event); + })); + GenericTestUtils.waitFor(()-> { + for (ComponentInstance instance: + context.scheduler.getLiveInstances().values()) { + if (!instance.getContainerState().equals(ContainerState.READY)) { + return false; + } + } + return true; + }, CHECK_EVERY_MILLIS, TIMEOUT); } - private ServiceManager createTestServiceManager(String name) - throws IOException { - ServiceContext context = new ServiceContext(); - context.service = createBaseDef(name); - context.fs = rule.getFs(); - - context.scheduler = new ServiceScheduler(context) { - @Override - protected YarnRegistryViewForProviders createYarnRegistryOperations( - ServiceContext context, RegistryOperations registryClient) { - return mock(YarnRegistryViewForProviders.class); + private void upgradeInstancesOf(ServiceContext context, String compName) + throws TimeoutException, InterruptedException { + Collection compInstances = context.scheduler + .getAllComponents().get(compName).getAllComponentInstances(); + GenericTestUtils.waitFor(() -> { + for (ComponentInstance instance : compInstances) { + if (!instance.getContainerState().equals(ContainerState.UPGRADING)) { + return false; + } } - }; + return true; + }, CHECK_EVERY_MILLIS, TIMEOUT); - context.scheduler.init(rule.getConf()); + // instances of comp1 get upgraded and become ready event is triggered + // become ready + compInstances.forEach(instance -> { + ComponentInstanceEvent event = new ComponentInstanceEvent( + instance.getContainer().getId(), + ComponentInstanceEventType.BECOME_READY); - Map - componentState = context.scheduler.getAllComponents(); - context.service.getComponents().forEach(component -> { - componentState.put(component.getName(), - new org.apache.hadoop.yarn.service.component.Component(component, - 1L, context)); + context.scheduler.getDispatcher().getEventHandler().handle(event); }); - return new ServiceManager(context); + } + + private ServiceContext createServiceContext(String name) + throws Exception { + Service service = createBaseDef(name); + ServiceContext context = new MockRunningServiceContext(rule, + service); + context.scheduler.getDispatcher().setDrainEventsOnStop(); + context.scheduler.getDispatcher().start(); + return context; } public static Service createBaseDef(String name) { @@ -257,4 +332,6 @@ private void writeUpgradedDef(Service upgradedDef) upgradedDef); } + private static final int TIMEOUT = 200000; + private static final int CHECK_EVERY_MILLIS = 100; } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java index 8b13b2495b8..216d88fc4c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java @@ -415,6 +415,41 @@ public void testUpgrade() throws Exception { client.actionDestroy(service.getName()); } + @Test(timeout = 200000) + public void testExpressUpgrade() throws Exception { + setupInternal(NUM_NMS); + getConf().setBoolean(YARN_SERVICE_UPGRADE_ENABLED, true); + ServiceClient client = createClient(getConf()); + + Service service = createExampleApplication(); + client.actionCreate(service); + waitForServiceToBeStable(client, service); + + // upgrade the service + Component component = service.getComponents().iterator().next(); + service.setState(ServiceState.EXPRESS_UPGRADING); + service.setVersion("v2"); + component.getConfiguration().getEnv().put("key1", "val1"); + Component component2 = service.getComponent("compb"); + component2.getConfiguration().getEnv().put("key2", "val2"); + client.actionUpgradeExpress(service); + + // wait for upgrade to complete + waitForServiceToBeStable(client, service); + Service active = client.getStatus(service.getName()); + Assert.assertEquals("component not stable", ComponentState.STABLE, + active.getComponent(component.getName()).getState()); + Assert.assertEquals("compa does not have new env", "val1", + active.getComponent(component.getName()).getConfiguration() + .getEnv("key1")); + Assert.assertEquals("compb does not have new env", "val2", + active.getComponent(component2.getName()).getConfiguration() + .getEnv("key2")); + LOG.info("Stop/destroy service {}", service); + client.actionStop(service.getName(), true); + client.actionDestroy(service.getName()); + } + // Test to verify ANTI_AFFINITY placement policy // 1. Start mini cluster with 3 NMs and scheduler placement-constraint handler // 2. Create an example service with 3 containers diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java similarity index 87% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java index c2a80e7fa78..98e7474237f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java @@ -15,11 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.service; +package org.apache.hadoop.yarn.service.utils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.ServiceTestUtils; import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.KerberosPrincipal; @@ -30,8 +31,6 @@ import org.apache.hadoop.yarn.service.api.records.Resource; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; -import org.apache.hadoop.yarn.service.utils.SliderFileSystem; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -39,6 +38,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -53,7 +53,7 @@ /** * Test for ServiceApiUtil helper methods. */ -public class TestServiceApiUtil { +public class TestServiceApiUtil extends ServiceTestUtils { private static final Logger LOG = LoggerFactory .getLogger(TestServiceApiUtil.class); private static final String EXCEPTION_PREFIX = "Should have thrown " + @@ -635,10 +635,12 @@ public void testKerberosPrincipalNameFormat() throws IOException { try { ServiceApiUtil.validateKerberosPrincipal(app.getKerberosPrincipal()); - Assert.fail(EXCEPTION_PREFIX + "service with invalid principal name format."); + Assert.fail(EXCEPTION_PREFIX + "service with invalid principal name " + + "format."); } catch (IllegalArgumentException e) { assertEquals( - String.format(RestApiErrorMessages.ERROR_KERBEROS_PRINCIPAL_NAME_FORMAT, + String.format( + RestApiErrorMessages.ERROR_KERBEROS_PRINCIPAL_NAME_FORMAT, kp.getPrincipalName()), e.getMessage()); } @@ -650,4 +652,92 @@ public void testKerberosPrincipalNameFormat() throws IOException { Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); } } + + @Test + public void testResolveCompsDependency() { + Service service = createExampleApplication(); + List dependencies = new ArrayList(); + dependencies.add("compb"); + Component compa = createComponent("compa"); + compa.setDependencies(dependencies); + Component compb = createComponent("compb"); + service.addComponent(compa); + service.addComponent(compb); + List order = ServiceApiUtil.resolveCompsDependency(service); + List expected = new ArrayList(); + expected.add("compb"); + expected.add("compa"); + for (int i = 0; i < expected.size(); i++) { + Assert.assertEquals("Components are not equal.", expected.get(i), + order.get(i)); + } + } + + @Test + public void testResolveCompsDependencyReversed() { + Service service = createExampleApplication(); + List dependencies = new ArrayList(); + dependencies.add("compa"); + Component compa = createComponent("compa"); + Component compb = createComponent("compb"); + compb.setDependencies(dependencies); + service.addComponent(compa); + service.addComponent(compb); + List order = ServiceApiUtil.resolveCompsDependency(service); + List expected = new ArrayList(); + expected.add("compa"); + expected.add("compb"); + for (int i = 0; i < expected.size(); i++) { + Assert.assertEquals("Components are not equal.", expected.get(i), + order.get(i)); + } + } + + @Test + public void testResolveCompsCircularDependency() { + Service service = createExampleApplication(); + List dependencies = new ArrayList(); + List dependencies2 = new ArrayList(); + dependencies.add("compb"); + dependencies2.add("compa"); + Component compa = createComponent("compa"); + compa.setDependencies(dependencies); + Component compb = createComponent("compb"); + compa.setDependencies(dependencies2); + service.addComponent(compa); + service.addComponent(compb); + List order = ServiceApiUtil.resolveCompsDependency(service); + List expected = new ArrayList(); + expected.add("compa"); + expected.add("compb"); + for (int i = 0; i < expected.size(); i++) { + Assert.assertEquals("Components are not equal.", expected.get(i), + order.get(i)); + } + } + + @Test + public void testResolveNoCompsDependency() { + Service service = createExampleApplication(); + Component compa = createComponent("compa"); + Component compb = createComponent("compb"); + service.addComponent(compa); + service.addComponent(compb); + List order = ServiceApiUtil.resolveCompsDependency(service); + List expected = new ArrayList(); + expected.add("compa"); + expected.add("compb"); + for (int i = 0; i < expected.size(); i++) { + Assert.assertEquals("Components are not equal.", expected.get(i), + order.get(i)); + } + } + + public static Service createExampleApplication() { + + Service exampleApp = new Service(); + exampleApp.setName("example-app"); + exampleApp.setVersion("v1"); + return exampleApp; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 26c99e31aa9..f51b2f9b168 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -33,7 +33,6 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse; @@ -52,10 +51,14 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; @@ -900,4 +903,58 @@ public abstract Resource getResourceProfile(String profile) @Unstable public abstract List getResourceTypeInfo() throws YarnException, IOException; + + /** + *

+ * The interface used by client to get node attributes in the cluster. + *

+ * + * @return cluster node attributes collection + * @throws YarnException when there is a failure in + * {@link ApplicationClientProtocol} + * @throws IOException when there is a failure in + * {@link ApplicationClientProtocol} + */ + @Public + @Unstable + public abstract Set getClusterAttributes() + throws YarnException, IOException; + + /** + *

+ * The interface used by client to get mapping of AttributeKey to associated + * NodeToAttributeValue list for specified node attributeKeys in the cluster. + *

+ * + * @param attributes AttributeKeys for which associated NodeToAttributeValue + * mapping value has to be retrieved. If empty or null is set then + * will return mapping for all attributeKeys in the cluster + * @return mapping of AttributeKey to List of associated + * NodeToAttributeValue's. + * @throws YarnException + * @throws IOException + */ + @Public + @Unstable + public abstract Map> getAttributesToNodes( + Set attributes) throws YarnException, IOException; + + /** + *

+ * The interface used by client to get all node to attribute mapping in + * existing cluster. + *

+ * + * @param hostNames HostNames for which host to attributes mapping has to + * be retrived.If empty or null is set then will return + * all nodes to attributes mapping in cluster. + * @return Node to attribute mappings + * @throws YarnException + * @throws IOException + */ + @Public + @Unstable + public abstract Map> getNodeToAttributes( + Set hostNames) throws YarnException, IOException; + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 1ceb46209b1..acfc3ff70be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -22,7 +22,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.EnumSet; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -52,8 +51,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; @@ -68,6 +69,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; @@ -96,15 +98,18 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; @@ -977,4 +982,28 @@ public Resource getResourceProfile(String profile) GetAllResourceTypeInfoRequest.newInstance(); return rmClient.getResourceTypeInfo(request).getResourceTypeInfo(); } + + @Override + public Set getClusterAttributes() + throws YarnException, IOException { + GetClusterNodeAttributesRequest request = + GetClusterNodeAttributesRequest.newInstance(); + return rmClient.getClusterNodeAttributes(request).getNodeAttributes(); + } + + @Override + public Map> getAttributesToNodes( + Set attributes) throws YarnException, IOException { + GetAttributesToNodesRequest request = + GetAttributesToNodesRequest.newInstance(attributes); + return rmClient.getAttributesToNodes(request).getAttributesToNodes(); + } + + @Override + public Map> getNodeToAttributes( + Set hostNames) throws YarnException, IOException { + GetNodesToAttributesRequest request = + GetNodesToAttributesRequest.newInstance(hostNames); + return rmClient.getNodesToAttributes(request).getNodeToAttributes(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 807938c7f0c..a0e4e02b0a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.client.cli; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; @@ -100,6 +101,7 @@ public static final String COMPONENT = "component"; public static final String ENABLE_FAST_LAUNCH = "enableFastLaunch"; public static final String UPGRADE_CMD = "upgrade"; + public static final String UPGRADE_EXPRESS = "express"; public static final String UPGRADE_INITIATE = "initiate"; public static final String UPGRADE_AUTO_FINALIZE = "autoFinalize"; public static final String UPGRADE_FINALIZE = "finalize"; @@ -247,6 +249,9 @@ public int run(String[] args) throws Exception { opts.addOption(UPGRADE_CMD, true, "Upgrades an application/long-" + "running service. It requires either -initiate, -instances, or " + "-finalize options."); + opts.addOption(UPGRADE_EXPRESS, true, "Works with -upgrade option to " + + "perform express upgrade. It requires the upgraded application " + + "specification file."); opts.addOption(UPGRADE_INITIATE, true, "Works with -upgrade option to " + "initiate the application upgrade. It requires the upgraded " + "application specification file."); @@ -639,9 +644,9 @@ public int run(String[] args) throws Exception { moveApplicationAcrossQueues(cliParser.getOptionValue(APP_ID), cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE)); } else if (cliParser.hasOption(UPGRADE_CMD)) { - if (hasAnyOtherCLIOptions(cliParser, opts, UPGRADE_CMD, UPGRADE_INITIATE, - UPGRADE_AUTO_FINALIZE, UPGRADE_FINALIZE, COMPONENT_INSTS, COMPONENTS, - APP_TYPE_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, UPGRADE_CMD, UPGRADE_EXPRESS, + UPGRADE_INITIATE, UPGRADE_AUTO_FINALIZE, UPGRADE_FINALIZE, + COMPONENT_INSTS, COMPONENTS, APP_TYPE_CMD)) { printUsage(title, opts); return exitCode; } @@ -649,7 +654,14 @@ public int run(String[] args) throws Exception { AppAdminClient client = AppAdminClient.createAppAdminClient(appType, getConf()); String appName = cliParser.getOptionValue(UPGRADE_CMD); - if (cliParser.hasOption(UPGRADE_INITIATE)) { + if (cliParser.hasOption(UPGRADE_EXPRESS)) { + File file = new File(cliParser.getOptionValue(UPGRADE_EXPRESS)); + if (!file.exists()) { + System.err.println(file.getAbsolutePath() + " does not exist."); + return exitCode; + } + return client.actionUpgradeExpress(appName, file); + } else if (cliParser.hasOption(UPGRADE_INITIATE)) { if (hasAnyOtherCLIOptions(cliParser, opts, UPGRADE_CMD, UPGRADE_INITIATE, UPGRADE_AUTO_FINALIZE, APP_TYPE_CMD)) { printUsage(title, opts); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java index a29b0db7362..4d939498453 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java @@ -36,6 +36,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -52,6 +53,7 @@ public static final String LIST_LABELS_CMD = "list-node-labels"; public static final String DIRECTLY_ACCESS_NODE_LABEL_STORE = "directly-access-node-label-store"; + public static final String LIST_CLUSTER_ATTRIBUTES="list-node-attributes"; public static final String CMD = "cluster"; private boolean accessLocal = false; static CommonNodeLabelsManager localNodeLabelsManager = null; @@ -71,6 +73,8 @@ public int run(String[] args) throws Exception { opts.addOption("lnl", LIST_LABELS_CMD, false, "List cluster node-label collection"); + opts.addOption("lna", LIST_CLUSTER_ATTRIBUTES, false, + "List cluster node-attribute collection"); opts.addOption("h", HELP_CMD, false, "Displays help for all commands."); opts.addOption("dnl", DIRECTLY_ACCESS_NODE_LABEL_STORE, false, "This is DEPRECATED, will be removed in future releases. Directly access node label store, " @@ -102,6 +106,8 @@ public int run(String[] args) throws Exception { if (parsedCli.hasOption(LIST_LABELS_CMD)) { printClusterNodeLabels(); + } else if(parsedCli.hasOption(LIST_CLUSTER_ATTRIBUTES)){ + printClusterNodeAttributes(); } else if (parsedCli.hasOption(HELP_CMD)) { printUsage(opts); return 0; @@ -112,6 +118,17 @@ public int run(String[] args) throws Exception { return 0; } + private void printClusterNodeAttributes() throws IOException, YarnException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter( + new OutputStreamWriter(baos, Charset.forName("UTF-8"))); + for (NodeAttributeInfo attribute : client.getClusterAttributes()) { + pw.println(attribute.toString()); + } + pw.close(); + sysout.println(baos.toString("UTF-8")); + } + void printClusterNodeLabels() throws YarnException, IOException { List nodeLabels = null; if (accessLocal) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java new file mode 100644 index 00000000000..13d5e24c1c5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java @@ -0,0 +1,715 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.cli; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.UnrecognizedOptionException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeType; +import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.AttributeMappingOperationType; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * CLI to map attributes to Nodes. + */ +public class NodeAttributesCLI extends Configured implements Tool { + + protected static final String INVALID_MAPPING_ERR_MSG = + "Invalid Node to attribute mapping : "; + + protected static final String USAGE_YARN_NODE_ATTRIBUTES = + "Usage: yarn nodeattributes "; + + protected static final String MISSING_ARGUMENT = + "Missing argument for command"; + + protected static final String NO_MAPPING_ERR_MSG = + "No node-to-attributes mappings are specified"; + + private static final String DEFAULT_SEPARATOR = System.lineSeparator(); + public static final String INVALID_COMMAND_USAGE = "Invalid Command Usage : "; + /** + * Output stream for errors, for use in tests. + */ + private PrintStream errOut = System.err; + + public NodeAttributesCLI() { + super(); + } + + protected void setErrOut(PrintStream errOut) { + this.errOut = errOut; + } + + protected AdminCommandHandler getAdminCommandHandler() { + return new AdminCommandHandler(); + } + + protected ClientCommandHandler getClientCommandHandler() { + return new ClientCommandHandler(); + } + + void printUsage(String cmd, boolean desc, CommandHandler... handlers) + throws UnsupportedEncodingException { + StringBuilder usageBuilder = new StringBuilder(); + usageBuilder.append(USAGE_YARN_NODE_ATTRIBUTES); + boolean satisfied = false; + for (CommandHandler cmdHandlers : handlers) { + satisfied |= cmdHandlers.getHelp(cmd, usageBuilder, desc); + } + if (!satisfied) { + printUsage(desc, handlers); + } else { + print(usageBuilder); + } + } + + private void printUsage(boolean desc, CommandHandler... handlers) + throws UnsupportedEncodingException { + StringBuilder usageBuilder = new StringBuilder(); + usageBuilder.append(USAGE_YARN_NODE_ATTRIBUTES); + for (CommandHandler cmdHandlers : handlers) { + cmdHandlers.getHelp(usageBuilder, desc); + } + + // append help with usage + usageBuilder.append(DEFAULT_SEPARATOR) + .append(" -help [cmd] List help of commands"); + print(usageBuilder); + } + + private void print(StringBuilder usageBuilder) + throws UnsupportedEncodingException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = + new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8"))); + pw.write(usageBuilder.toString()); + pw.close(); + errOut.println(baos.toString("UTF-8")); + } + + private Options buildOptions(CommandHandler... handlers) { + Options opts = new Options(); + for (CommandHandler handler : handlers) { + Options handlerOpts = handler.getOptions(); + handlerOpts.getOptions().iterator() + .forEachRemaining(option -> opts.addOption((Option) option)); + } + return opts; + } + + public int run(String[] args) throws Exception { + + int exitCode = -1; + + AdminCommandHandler adminCmdHandler = getAdminCommandHandler(); + ClientCommandHandler clientCmdHandler = getClientCommandHandler(); + + // Build options + Options opts = buildOptions(adminCmdHandler, clientCmdHandler); + + if (args.length < 1) { + printUsage(false, adminCmdHandler, clientCmdHandler); + return -1; + } + + // Handle command separate + if (handleHelpCommand(args, adminCmdHandler, clientCmdHandler)) { + return 0; + } + + CommandLine cliParser; + CommandHandler handler = null; + try { + cliParser = new GnuParser().parse(opts, args); + handler = adminCmdHandler.canHandleCommand(cliParser) ? + adminCmdHandler : + clientCmdHandler.canHandleCommand(cliParser) ? + clientCmdHandler : + null; + if (handler == null) { + errOut.println(INVALID_COMMAND_USAGE); + printUsage(false, adminCmdHandler, clientCmdHandler); + return exitCode; + } else { + return handler.handleCommand(cliParser); + } + } catch (UnrecognizedOptionException e) { + errOut.println(INVALID_COMMAND_USAGE); + printUsage(false, adminCmdHandler, clientCmdHandler); + return exitCode; + } catch (MissingArgumentException ex) { + errOut.println(MISSING_ARGUMENT); + printUsage(true, adminCmdHandler, clientCmdHandler); + return exitCode; + } catch (IllegalArgumentException arge) { + errOut.println(arge.getLocalizedMessage()); + // print admin command detail + printUsage(true, handler); + return exitCode; + } catch (Exception e) { + errOut.println(e.toString()); + printUsage(true, handler); + return exitCode; + } + } + + private boolean handleHelpCommand(String[] args, CommandHandler... handlers) + throws UnsupportedEncodingException { + if (args[0].equals("-help")) { + if (args.length == 2) { + printUsage(args[1], true, handlers); + } else { + printUsage(true, handlers); + } + return true; + } + return false; + } + + public static void main(String[] args) throws Exception { + int result = ToolRunner.run(new NodeAttributesCLI(), args); + System.exit(result); + } + + /** + * Abstract class for command handler. + */ + public static abstract class CommandHandler extends Configured { + + private Options options; + + private LinkedList order = new LinkedList<>(); + private String header; + + protected CommandHandler(String header) { + this(new YarnConfiguration()); + this.header = header; + } + + protected CommandHandler(Configuration conf) { + super(conf); + options = buildOptions(); + } + + public boolean canHandleCommand(CommandLine parse) { + ArrayList