diff --git a/common/pom.xml b/common/pom.xml index ad9f6c0..5ab70c2 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -46,6 +46,11 @@ ${commons-cli.version} + commons-io + commons-io + ${commons-io.version} + + commons-lang commons-lang ${commons-lang.version} diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/CipherSuite.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/CipherSuite.java new file mode 100644 index 0000000..1b6a69a --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/CipherSuite.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.io.crypto; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Defines properties of a CipherSuite. Modeled after the ciphers in + * {@link javax.crypto.Cipher}. + */ +@InterfaceAudience.Private +public enum CipherSuite { + UNKNOWN("Unknown", 0), + AES_CTR_NOPADDING("AES/CTR/NoPadding", 16); + + private final String name; + private final int algoBlockSize; + + private Integer unknownValue = null; + + CipherSuite(String name, int algoBlockSize) { + this.name = name; + this.algoBlockSize = algoBlockSize; + } + + public void setUnknownValue(int unknown) { + this.unknownValue = unknown; + } + + public int getUnknownValue() { + return unknownValue; + } + + /** + * @return name of cipher suite, as in {@link javax.crypto.Cipher} + */ + public String getName() { + return name; + } + + /** + * @return size of an algorithm block in bytes + */ + public int getAlgorithmBlockSize() { + return algoBlockSize; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("{"); + builder.append("name: " + name); + builder.append(", algorithmBlockSize: " + algoBlockSize); + if (unknownValue != null) { + builder.append(", unknownValue: " + unknownValue); + } + builder.append("}"); + return builder.toString(); + } + + public static void checkName(String name) { + CipherSuite[] suites = CipherSuite.values(); + for (CipherSuite suite : suites) { + if (suite.getName().equals(name)) { + return; + } + } + throw new IllegalArgumentException("Invalid cipher suite name: " + name); + } + + /** + * Convert to CipherSuite from name, {@link #algoBlockSize} is fixed for + * certain cipher suite, just need to compare the name. + * @param name cipher suite name + * @return CipherSuite cipher suite + */ + public static CipherSuite convert(String name) { + CipherSuite[] suites = CipherSuite.values(); + for (CipherSuite suite : suites) { + if (suite.getName().equals(name)) { + return suite; + } + } + throw new IllegalArgumentException("Invalid cipher suite name: " + name); + } + + /** + * Returns suffix of cipher suite configuration. + * @return String configuration suffix + */ + public String getConfigSuffix() { + String[] parts = name.split("/"); + StringBuilder suffix = new StringBuilder(); + for (String part : parts) { + suffix.append(".").append(part.toLowerCase()); + } + + return suffix.toString(); + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/CryptoCodec.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/CryptoCodec.java new file mode 100644 index 0000000..41fb84f --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/CryptoCodec.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hive.common.io.crypto; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configurable; + +/** + * A common interface for a cryptographic algorithm. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class CryptoCodec implements Configurable { + + /** + * @return the CipherSuite for this codec. + */ + public abstract CipherSuite getCipherSuite(); + + /** + * Return this codec's algorithm + */ + public abstract String getAlgorithm(); + + /** + * Return the key length required by this cipher, in bytes + */ + public abstract int getKeyLength(); + + /** + * Return the expected initialization vector length, in bytes, or 0 if not applicable + */ + public abstract int getIvLength(); + + /** + * Get an encryptor for encrypting data. + */ + public abstract Encryptor createEncryptor(); + + /** + * Return a decryptor for decrypting data. + */ + public abstract Decryptor createDecryptor(); + +} diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/CryptoCodecFactory.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/CryptoCodecFactory.java new file mode 100644 index 0000000..b326803 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/CryptoCodecFactory.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hive.common.io.crypto; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.util.ReflectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CryptoCodecFactory { + public static Logger LOG = LoggerFactory.getLogger(CryptoCodecFactory.class); + + /** + * Get crypto codec for specified algorithm/mode/padding. + * + * @param conf + * the configuration + * @param CipherSuite + * algorithm/mode/padding + * @return CryptoCodec the codec object. Null value will be returned if no + * crypto codec classes with cipher suite configured. + */ + public static CryptoCodec getInstance(Configuration conf, + CipherSuite cipherSuite) { + Class clazz = getCodecClass(conf, cipherSuite); + if (clazz == null) { + return null; + } + CryptoCodec codec = null; + try { + CryptoCodec c = ReflectionUtils.newInstance(clazz, conf); + if (c.getCipherSuite().getName().equals(cipherSuite.getName())) { + if (codec == null) { + LOG.debug("Using crypto codec {}.", clazz.getName()); + codec = c; + } + } else { + LOG.warn("Crypto codec {} doesn't meet the cipher suite {}.", + clazz.getName(), cipherSuite.getName()); + } + } catch (Exception e) { + LOG.warn("Crypto codec {} is not available.", clazz.getName()); + } + + if (codec != null) { + return codec; + } + + throw new RuntimeException("No available crypto codec which meets " + + "the cipher suite " + cipherSuite.getName() + "."); + } + + /** + * Get crypto codec for algorithm/mode/padding in config value + * hive.security.crypto.cipher.suite + * + * @param conf + * the configuration + * @return CryptoCodec the codec object Null value will be returned if no + * crypto codec classes with cipher suite configured. + */ + public static CryptoCodec getInstance(Configuration conf) { + String name = conf.get(ConfVars.HIVE_SECURITY_CRYPTO_CIPHER_SUITE.varname, + ConfVars.HIVE_SECURITY_CRYPTO_CIPHER_SUITE.getDefaultValue()); + return getInstance(conf, CipherSuite.convert(name)); + } + + private static Class getCodecClass( + Configuration conf, CipherSuite cipherSuite) { + Class result = null; + String codecString = conf.get(ConfVars.HIVE_SECURITY_CRYPTO_CODEC.varname, + ConfVars.HIVE_SECURITY_CRYPTO_CODEC.getDefaultValue()); + + try { + result = (Class) conf.getClassByName(codecString); + } catch (ClassCastException e) { + LOG.warn("Class " + codecString + " is not a CryptoCodec."); + } catch (ClassNotFoundException e) { + LOG.warn("Crypto codec " + codecString + " not found."); + } + + return result; + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/Decryptor.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/Decryptor.java new file mode 100644 index 0000000..fd96105 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/Decryptor.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.io.crypto; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Decryptors apply a cipher to an InputStream to recover plaintext. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Decryptor { + + /** + * Get the secret key + */ + public abstract Key getKey(); + + /** + * Set the secret key + * @param key + */ + public abstract void setKey(Key key); + + /** + * Get the expected length for the initialization vector + * @return the expected length for the initialization vector + */ + public abstract int getIvLength(); + + /** + * Get the cipher's internal block size + * @return the cipher's internal block size + */ + public abstract int getBlockSize(); + + /** + * Set the initialization vector + * @param iv + */ + public abstract void setIv(byte[] iv); + + /** + * Reset state, reinitialize with the key and iv + */ + public abstract void reset(); + + /** + * Create a stream for decryption + * @param in + */ + public abstract InputStream createDecryptionStream(InputStream in); + + /** + * Decrypt a stream of ciphertext + * @param in + * @param out + */ + public void decrypt(InputStream in, OutputStream out, int outLen) + throws IOException { + InputStream is = createDecryptionStream(in); + byte buf[] = new byte[8*1024]; + long remaining = outLen; + try { + while (remaining > 0) { + int toRead = (int)(remaining < buf.length ? remaining : buf.length); + int read = is.read(buf, 0, toRead); + if (read < 0) { + break; + } + out.write(buf, 0, read); + remaining -= read; + } + } finally { + is.close(); + } + } + + /** + * Decrypt a stream to a array of byte + * @param in + * @param out + */ + public void decrypt(InputStream in, byte[] dest, int destOffset, + int destSize) throws IOException { + InputStream is = createDecryptionStream(in); + try { + IOUtils.readFully(is, dest, destOffset, destSize); + } finally { + is.close(); + } + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/Encryptor.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/Encryptor.java new file mode 100644 index 0000000..5bce6dc --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/Encryptor.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.io.crypto; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Encryptors apply a cipher to an OutputStream to produce ciphertext. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Encryptor { + public static Logger LOG = LoggerFactory.getLogger(Encryptor.class); + + /** + * Get the secret key + */ + public abstract Key getKey(); + + /** + * Set the secret key + * @param key + */ + public abstract void setKey(Key key); + + /** + * Get the expected length for the initialization vector + * @return the expected length for the initialization vector + */ + public abstract int getIvLength(); + + /** + * Get the cipher's internal block size + * @return the cipher's internal block size + */ + public abstract int getBlockSize(); + + /** + * Get the initialization vector + */ + public abstract byte[] getIv(); + + /** + * Set the initialization vector + * @param iv + */ + public abstract void setIv(byte[] iv); + + /** + * Reset state, reinitialize with the key and iv + */ + public abstract void reset(); + + /** + * Create a stream for encryption + * @param out + */ + public abstract OutputStream createEncryptionStream(OutputStream out); + + /** + * Encrypt a stream of plaintext + * @param in + * @param out + */ + public void encrypt(InputStream in, OutputStream out) throws IOException { + OutputStream os = createEncryptionStream(out); + try { + IOUtils.copy(in, os); + } finally { + os.close(); + } + } + + /** + * Encrypt a array of byte of plaintext + * @param src + * @param offset + * @param length + * @param out + */ + public void encrypt(byte[] src, int offset, int length, + OutputStream out) throws IOException { + OutputStream os = createEncryptionStream(out); + try { + os.write(src, offset, length); + } finally { + os.close(); + } + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/Key.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/Key.java new file mode 100644 index 0000000..f056669 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/Key.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hive.common.io.crypto; + +import java.util.Arrays; + +/** + * Key is class contains key name and key material. + * + */ +public class Key { + private final String name; + private final byte[] material; + + public Key(String name, byte[] material) { + this.name = name; + this.material = Arrays.copyOf(material, material.length);; + } + + public String getName() { + return name; + } + + public byte[] getMaterial() { + return material; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(); + buf.append("key"); + buf.append("="); + if (material == null) { + buf.append("null"); + } else { + for(byte b: material) { + buf.append(' '); + int right = b & 0xff; + if (right < 0x10) { + buf.append('0'); + } + buf.append(Integer.toHexString(right)); + } + } + return buf.toString(); + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/AesDecryptor.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/AesDecryptor.java new file mode 100644 index 0000000..1479be5 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/AesDecryptor.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.io.crypto.aes; + +import java.io.InputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.util.Arrays; + +import javax.crypto.Cipher; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.common.io.crypto.Decryptor; +import org.apache.hadoop.hive.common.io.crypto.Key; + +import com.google.common.base.Preconditions; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class AesDecryptor extends Decryptor { + + private final Cipher cipher; + private Key key; + private byte[] iv; + private boolean initialized = false; + + public AesDecryptor(Cipher cipher) { + this.cipher = cipher; + } + + public AesDecryptor(Cipher cipher, Key key, byte[] iv) { + this.cipher = cipher; + this.key = key; + this.iv = Arrays.copyOf(iv, iv.length); + } + + public Cipher getCipher() { + return cipher; + } + + @Override + public Key getKey() { + return key; + } + + @Override + public void setKey(Key key) { + Preconditions.checkNotNull(key, "Key cannot be null"); + if (key != null) { + Preconditions.checkArgument(key.getMaterial().length == JceAesCtrCryptoCodec.KEY_LENGTH, + "Invalid key length"); + } + this.key = key; + } + + @Override + public int getIvLength() { + return JceAesCtrCryptoCodec.IV_LENGTH; + } + + @Override + public int getBlockSize() { + return JceAesCtrCryptoCodec.BLOCK_SIZE; + } + + @Override + public void setIv(byte[] iv) { + Preconditions.checkNotNull(iv, "IV cannot be null"); + Preconditions.checkArgument(iv.length == JceAesCtrCryptoCodec.IV_LENGTH, + "Invalid IV length"); + this.iv = Arrays.copyOf(iv, iv.length); + } + + @Override + public void reset() { + init(); + } + + @Override + public InputStream createDecryptionStream(InputStream in) { + if (!initialized) { + init(); + } + return new javax.crypto.CipherInputStream(in, cipher); + } + + protected void init() { + try { + if (iv == null) { + throw new NullPointerException("IV is null"); + } + cipher.init(javax.crypto.Cipher.DECRYPT_MODE, + new SecretKeySpec(key.getMaterial(), "AES"), new IvParameterSpec(iv)); + } catch (InvalidKeyException e) { + throw new RuntimeException(e); + } catch (InvalidAlgorithmParameterException e) { + throw new RuntimeException(e); + } + initialized = true; + } + +} diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/AesEncryptor.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/AesEncryptor.java new file mode 100644 index 0000000..3d5cce1 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/AesEncryptor.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.io.crypto.aes; + +import java.io.OutputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.SecureRandom; +import java.util.Arrays; + +import javax.crypto.Cipher; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.common.io.crypto.Encryptor; +import org.apache.hadoop.hive.common.io.crypto.Key; + +import com.google.common.base.Preconditions; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class AesEncryptor extends Encryptor { + + private final Cipher cipher; + private final SecureRandom rng; + private Key key; + private byte[] iv; + private boolean initialized = false; + + public AesEncryptor(Cipher cipher, SecureRandom rng) { + this.cipher = cipher; + this.rng = rng; + } + + public AesEncryptor(Cipher cipher, SecureRandom rng, Key key, byte[] iv) { + this.cipher = cipher; + this.rng = rng; + this.key = key; + this.iv = Arrays.copyOf(iv, iv.length); + } + + public Cipher getCipher() { + return cipher; + } + + @Override + public Key getKey() { + return key; + } + + @Override + public void setKey(Key key) { + Preconditions.checkNotNull(key, "Key cannot be null"); + if (key != null) { + Preconditions.checkArgument(key.getMaterial().length == JceAesCtrCryptoCodec.KEY_LENGTH, + "Invalid key length"); + } + this.key = key; + } + + @Override + public int getIvLength() { + return JceAesCtrCryptoCodec.IV_LENGTH; + } + + @Override + public int getBlockSize() { + return JceAesCtrCryptoCodec.BLOCK_SIZE; + } + + @Override + public byte[] getIv() { + return iv; + } + + @Override + public void setIv(byte[] iv) { + if (iv != null) { + Preconditions.checkArgument(iv.length == JceAesCtrCryptoCodec.IV_LENGTH, + "Invalid IV length"); + } + this.iv = Arrays.copyOf(iv, iv.length); + } + + @Override + public void reset() { + init(); + } + + @Override + public OutputStream createEncryptionStream(OutputStream out) { + if (!initialized) { + init(); + } + return new javax.crypto.CipherOutputStream(out, cipher); + } + + protected void init() { + try { + if (iv == null) { + iv = new byte[getIvLength()]; + rng.nextBytes(iv); + } + cipher.init(javax.crypto.Cipher.ENCRYPT_MODE, + new SecretKeySpec(key.getMaterial(), "AES"), new IvParameterSpec(iv)); + } catch (InvalidKeyException e) { + throw new RuntimeException(e); + } catch (InvalidAlgorithmParameterException e) { + throw new RuntimeException(e); + } + initialized = true; + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/JceAesCtrCryptoCodec.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/JceAesCtrCryptoCodec.java new file mode 100644 index 0000000..493a02b --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/JceAesCtrCryptoCodec.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.io.crypto.aes; + +import java.security.GeneralSecurityException; +import java.security.SecureRandom; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.io.crypto.CipherSuite; +import org.apache.hadoop.hive.common.io.crypto.CryptoCodec; +import org.apache.hadoop.hive.common.io.crypto.Decryptor; +import org.apache.hadoop.hive.common.io.crypto.Encryptor; + +import com.google.common.annotations.VisibleForTesting; + +/** + * AES-128, provided by the JCE + *

+ * Algorithm instances are pooled for reuse, so the cipher provider and mode + * are configurable but fixed at instantiation. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class JceAesCtrCryptoCodec extends CryptoCodec { + private static final Log LOG = LogFactory.getLog(JceAesCtrCryptoCodec.class); + protected static final CipherSuite SUITE = CipherSuite.AES_CTR_NOPADDING; + + public static final int KEY_LENGTH = 16; + public static final int KEY_LENGTH_BITS = KEY_LENGTH * 8; + public static final int BLOCK_SIZE = 16; + public static final int IV_LENGTH = 16; + + public static final String CIPHER_PROVIDER_KEY = "hive.security.crypto.jce.provider"; + public static final String RNG_ALGORITHM_KEY = "hive.security.crypto.rng.algorithm"; + public static final String RNG_PROVIDER_KEY = "hive.security.crypto.rng.provider"; + + private Configuration conf; + private String rngAlgorithm; + private String cipherProvider; + private SecureRandom rng; + + public JceAesCtrCryptoCodec() { + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + // The JCE provider, null if default + cipherProvider = conf.get(CIPHER_PROVIDER_KEY); + // RNG algorithm + rngAlgorithm = conf.get(RNG_ALGORITHM_KEY, "SHA1PRNG"); + // RNG provider, null if default + String rngProvider = conf.get(RNG_PROVIDER_KEY); + try { + if (rngProvider != null) { + rng = SecureRandom.getInstance(rngAlgorithm, rngProvider); + } else { + rng = SecureRandom.getInstance(rngAlgorithm); + } + } catch (GeneralSecurityException e) { + LOG.warn("Could not instantiate specified RNG, falling back to default", e); + rng = new SecureRandom(); + } + } + + @Override + public CipherSuite getCipherSuite() { + return SUITE; + } + + @Override + public String getAlgorithm() { + return "AES"; + } + + @Override + public int getKeyLength() { + return KEY_LENGTH; + } + + @Override + public int getIvLength() { + return IV_LENGTH; + } + + @Override + public Encryptor createEncryptor() { + return new AesEncryptor(getJCECipherInstance(), rng); + } + + @Override + public Decryptor createDecryptor() { + return new AesDecryptor(getJCECipherInstance()); + } + + @VisibleForTesting + SecureRandom getRNG() { + return rng; + } + + private javax.crypto.Cipher getJCECipherInstance() { + try { + if (cipherProvider != null) { + return javax.crypto.Cipher.getInstance(SUITE.getName(), cipherProvider); + } + return javax.crypto.Cipher.getInstance(SUITE.getName()); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + } +} diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7f4afd9..0e95d87 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1729,7 +1729,13 @@ "When auto reducer parallelism is enabled this factor will be used to over-partition data in shuffle edges."), TEZ_MIN_PARTITION_FACTOR("hive.tez.min.partition.factor", 0.25f, "When auto reducer parallelism is enabled this factor will be used to put a lower limit to the number\n" + - "of reducers that tez specifies.") + "of reducers that tez specifies."), + HIVE_SECURITY_CRYPTO_CIPHER_SUITE("hive.security.crypto.cipher.suite", + "AES/CTR/NoPadding", + "Now just support AES/CTR/NoPadding"), + HIVE_SECURITY_CRYPTO_CODEC("hive.security.crypto.codec", + "hive.security.crypto.codec.aes.JceAesCryptoCodec", + "The crypto codec should match crypto cipher suite.") ; public final String varname; diff --git a/common/src/test/org/apache/hadoop/hive/common/io/crypto/TestCipherSuite.java b/common/src/test/org/apache/hadoop/hive/common/io/crypto/TestCipherSuite.java new file mode 100644 index 0000000..d47f328 --- /dev/null +++ b/common/src/test/org/apache/hadoop/hive/common/io/crypto/TestCipherSuite.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.io.crypto; + +import org.junit.Assert; +import org.junit.Test; + +public class TestCipherSuite { + @Test + public void testAesCtr() throws Exception { + CipherSuite SUITE = CipherSuite.AES_CTR_NOPADDING; + Assert.assertEquals("AES/CTR/NoPadding", SUITE.getName()); + Assert.assertEquals(16, SUITE.getAlgorithmBlockSize()); + + String trueAlgoName = "AES/CTR/NoPadding"; + String wrongAlogName = "AES/CTR/NoPadding/Wrong"; + CipherSuite.checkName(trueAlgoName); + try { + CipherSuite.checkName(wrongAlogName); + Assert.fail("wrong cipher algorithm"); + } catch (Exception e) { + // no-op + } + + SUITE = CipherSuite.convert(trueAlgoName); + Assert.assertEquals("AES/CTR/NoPadding", SUITE.getName()); + Assert.assertEquals(16, SUITE.getAlgorithmBlockSize()); + + try { + SUITE = CipherSuite.convert(wrongAlogName); + Assert.fail("wrong cipher algorithm"); + } catch (Exception e) { + // no-op + } + } +} diff --git a/common/src/test/org/apache/hadoop/hive/common/io/crypto/TestKey.java b/common/src/test/org/apache/hadoop/hive/common/io/crypto/TestKey.java new file mode 100644 index 0000000..a5c7dc5 --- /dev/null +++ b/common/src/test/org/apache/hadoop/hive/common/io/crypto/TestKey.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.io.crypto; + +import java.security.SecureRandom; +import java.util.Arrays; + +import org.junit.Assert; +import org.junit.Test; + +public class TestKey { + + @Test + public void testKey() throws Exception { + SecureRandom rng = SecureRandom.getInstance("SHA1PRNG"); + byte[] keyBytes = new byte[16]; + rng.nextBytes(keyBytes); + Key key = new Key("key01", keyBytes); + Assert.assertEquals("key01", key.getName()); + Assert.assertTrue(Arrays.equals(keyBytes, key.getMaterial())); + } +} diff --git a/common/src/test/org/apache/hadoop/hive/common/io/crypto/aes/TestAesCtrCryptoCodec.java b/common/src/test/org/apache/hadoop/hive/common/io/crypto/aes/TestAesCtrCryptoCodec.java new file mode 100644 index 0000000..cdaa762 --- /dev/null +++ b/common/src/test/org/apache/hadoop/hive/common/io/crypto/aes/TestAesCtrCryptoCodec.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.io.crypto.aes; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.security.SecureRandom; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.io.crypto.CipherSuite; +import org.apache.hadoop.hive.common.io.crypto.CryptoCodec; +import org.apache.hadoop.hive.common.io.crypto.CryptoCodecFactory; +import org.apache.hadoop.hive.common.io.crypto.Decryptor; +import org.apache.hadoop.hive.common.io.crypto.Encryptor; +import org.apache.hadoop.hive.common.io.crypto.Key; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestAesCtrCryptoCodec { + private SecureRandom rng; + private Key key; + private final byte[] iv = new byte[16]; + + @Before + public void setUp() throws Exception{ + rng = SecureRandom.getInstance("SHA1PRNG"); + byte[] keyBytes = new byte[16]; + rng.nextBytes(keyBytes); + key = new Key("key1", keyBytes); + rng.nextBytes(iv); + } + + @Test + public void testBasic() throws Exception { + Configuration conf = new Configuration(); + conf.set("hive.security.crypto.codec", JceAesCtrCryptoCodec.class.getName()); + conf.set("hive.security.crypto.cipher.suite", "AES/CTR/NoPadding"); + CryptoCodec codec = CryptoCodecFactory.getInstance(conf); + Assert.assertEquals(CipherSuite.AES_CTR_NOPADDING, codec.getCipherSuite()); + Assert.assertEquals("AES", codec.getAlgorithm()); + Assert.assertEquals(JceAesCtrCryptoCodec.KEY_LENGTH, codec.getKeyLength()); + Assert.assertEquals(JceAesCtrCryptoCodec.IV_LENGTH, codec.getIvLength()); + } + + @Test + public void testEncrypt() throws Exception { + Configuration conf = new Configuration(); + conf.set("hive.security.crypto.codec", JceAesCtrCryptoCodec.class.getName()); + conf.set("hive.security.crypto.cipher.suite", "AES/CTR/NoPadding"); + CryptoCodec codec = CryptoCodecFactory.getInstance(conf); + + Encryptor encryptor = codec.createEncryptor(); + encryptor.setKey(key); + encryptor.setIv(iv); + String text = "TestAesCtrCryptoCodec#testEncrypt"; + ByteArrayInputStream in = new ByteArrayInputStream(text.getBytes()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + encryptor.encrypt(in, out); + byte[] encryptedText = out.toByteArray(); + Assert.assertNotSame(text, new String(encryptedText)); + + Decryptor decryptor = codec.createDecryptor(); + decryptor.setKey(key); + decryptor.setIv(iv); + in = new ByteArrayInputStream(encryptedText); + out = new ByteArrayOutputStream(); + decryptor.decrypt(in, out, text.getBytes().length); + String decryptedText = new String(out.toByteArray()); + Assert.assertEquals(text, decryptedText); + } +} diff --git a/common/src/test/org/apache/hadoop/hive/common/io/crypto/aes/TestAesEncryptor.java b/common/src/test/org/apache/hadoop/hive/common/io/crypto/aes/TestAesEncryptor.java new file mode 100644 index 0000000..6aec4a3 --- /dev/null +++ b/common/src/test/org/apache/hadoop/hive/common/io/crypto/aes/TestAesEncryptor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.io.crypto.aes; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.security.SecureRandom; + +import javax.crypto.Cipher; + +import org.apache.hadoop.hive.common.io.crypto.Decryptor; +import org.apache.hadoop.hive.common.io.crypto.Encryptor; +import org.apache.hadoop.hive.common.io.crypto.Key; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestAesEncryptor { + private Cipher cipher; + private SecureRandom rng; + private Key key; + private final byte[] iv = new byte[16]; + + @Before + public void setUp() throws Exception{ + cipher = Cipher.getInstance("AES/CTR/NoPadding"); + rng = SecureRandom.getInstance("SHA1PRNG"); + byte[] keyBytes = new byte[16]; + rng.nextBytes(keyBytes); + key = new Key("key1", keyBytes); + rng.nextBytes(iv); + } + + @Test + public void testEncryptStream() throws Exception { + Encryptor encryptor = new AesEncryptor(cipher, rng); + encryptor.setKey(key); + encryptor.setIv(iv); + + String text = "TestAesEncryptor#testEncryptStream"; + ByteArrayInputStream in = new ByteArrayInputStream(text.getBytes()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + encryptor.encrypt(in, out); + byte[] encryptedText = out.toByteArray(); + Assert.assertNotSame(text, new String(encryptedText)); + + Decryptor decryptor = new AesDecryptor(cipher); + decryptor.setKey(key); + decryptor.setIv(iv); + in = new ByteArrayInputStream(encryptedText); + out = new ByteArrayOutputStream(); + decryptor.decrypt(in, out, text.getBytes().length); + String decryptedText = new String(out.toByteArray()); + Assert.assertEquals(text, decryptedText); + } + + @Test + public void testEncryptByteArray() throws Exception { + Encryptor encryptor = new AesEncryptor(cipher, rng); + encryptor.setKey(key); + encryptor.setIv(iv); + + String text = "TestAesEncryptor#testEncryptByteArray"; + byte[] textBytes = text.getBytes(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + encryptor.encrypt(textBytes, 0, textBytes.length, out); + byte[] encryptedText = out.toByteArray(); + Assert.assertNotSame(text, new String(encryptedText)); + + Decryptor decryptor = new AesDecryptor(cipher); + decryptor.setKey(key); + decryptor.setIv(iv); + InputStream in = new ByteArrayInputStream(encryptedText); + byte[] decryptedBytes = new byte[textBytes.length]; + decryptor.decrypt(in, decryptedBytes, 0, decryptedBytes.length); + String decryptedText = new String(decryptedBytes); + Assert.assertEquals(text, decryptedText); + } +}