Index: contrib/CHANGES.txt
===================================================================
--- contrib/CHANGES.txt (revision 901341)
+++ contrib/CHANGES.txt (working copy)
@@ -67,6 +67,9 @@
New features
+ * LUCENE-2228: Added AESDirectory along with its utility classes
+ (Jay Mundrawala)
+
* LUCENE-2102: Add a Turkish LowerCase Filter. TurkishLowerCaseFilter handles
Turkish and Azeri unique casing behavior correctly.
(Ahmet Arslan, Robert Muir via Simon Willnauer)
Index: contrib/misc/src/java/org/apache/lucene/store/AESDirectory.java
===================================================================
--- contrib/misc/src/java/org/apache/lucene/store/AESDirectory.java (revision 0)
+++ contrib/misc/src/java/org/apache/lucene/store/AESDirectory.java (revision 0)
@@ -0,0 +1,254 @@
+package org.apache.lucene.store;
+
+import java.io.File;
+import java.io.IOException;
+import org.apache.lucene.util.AESReader;
+import org.apache.lucene.util.AESWriter;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import javax.crypto.spec.SecretKeySpec;
+
+/**
+ * AESDirectory provides the ability for Lucene to read and write to an AES encrypted random access file using AESWriter and AESReader
+ *
+ * All rights reserved by the IIT IR Lab. (c)2009 Jordan Wilberding(jordan@ir.iit.edu) and Jay Mundrawala(mundra@ir.iit.edu)
+ *
+ * @author Jay Mundrawala
+ * @author Jordan Wilberding
+ */
+
+public class AESDirectory extends FSDirectory {
+
+ public static int BLOCKS_PER_PAGE = 64;
+ private SecretKeySpec key;
+ private int blocks_per_page;
+
+ public AESDirectory(File path, LockFactory lockFactory, byte[] key,
+ int blocks_per_page) throws IOException
+ {
+ super(path, lockFactory);
+ this.blocks_per_page = blocks_per_page;
+ this.key = new SecretKeySpec(key, "AES");
+ }
+
+ public AESDirectory(File path, LockFactory lockFactory, byte[] key) throws IOException
+ {
+ this(path, lockFactory, key, BLOCKS_PER_PAGE);
+ }
+
+ public AESDirectory(File path, byte[] key) throws IOException
+ {
+ this(path, null, key, BLOCKS_PER_PAGE);
+ }
+
+ public AESDirectory(File path, byte[] key, int blocks_per_page) throws IOException
+ {
+ this(path, null, key, blocks_per_page);
+ }
+ @Override
+ /** Returns the length in bytes of a file in the directory. */
+ public long fileLength(String name) {
+ ensureOpen();
+ File file = new File(directory, name);
+ long length;
+ AESReader aes;
+ try{
+ aes = new AESReader(file,key,blocks_per_page);
+ length = aes.length();
+ aes.close();
+ }catch(Exception e){
+ e.printStackTrace();
+ throw new RuntimeException("There was a problem reading the file: " + name);
+ }
+
+ return length;
+ }
+
+ @Override
+ public IndexOutput createOutput(String name) throws IOException {
+ initOutput(name);
+ return new AESIndexOutput(new File(directory, name), key, blocks_per_page);
+ }
+
+ @Override
+ public IndexInput openInput(String name, int bufferSize) throws IOException {
+ ensureOpen();
+ return new AESIndexInput(new File(directory, name), bufferSize, key, blocks_per_page);
+ }
+
+ protected static class AESIndexInput extends BufferedIndexInput {
+ protected static class Descriptor extends AESReader {
+ // remember if the file is open, so that we don't try to close it
+ // more than once
+ protected volatile boolean isOpen;
+ long position;
+ final long length;
+
+ public Descriptor(File file, SecretKeySpec key, int blocks_per_page) throws Exception {
+ super(file, key,blocks_per_page);
+ isOpen=true;
+ length=length();
+ }
+
+ public void close() throws IOException {
+ if (isOpen) {
+ isOpen=false;
+ super.close();
+ }
+ }
+
+ protected void finalize() throws Throwable {
+ try {
+ close();
+ } finally {
+ super.finalize();
+ }
+ }
+ }
+
+ protected final Descriptor file;
+ boolean isClone;
+
+ public AESIndexInput(File path, SecretKeySpec key, int blocks_per_page) throws IOException {
+ this(path, BufferedIndexInput.BUFFER_SIZE,key, blocks_per_page);
+ }
+
+ public AESIndexInput(File path, int bufferSize, SecretKeySpec key, int blocks_per_page) throws IOException {
+ super(bufferSize);
+ try{
+ file = new Descriptor(path, key, blocks_per_page);
+ }catch(IOException e){
+ throw e;
+ }catch(Exception e){
+ e.printStackTrace();
+ throw new RuntimeException("Bad File");
+ }
+ }
+
+ /** IndexInput methods */
+ protected void readInternal(byte[] b, int offset, int len)
+ throws IOException {
+ synchronized (file) {
+ try{
+ long position = getFilePointer();
+ if (position != file.position) {
+ file.seek(position);
+ file.position = position;
+ }
+ int total = 0;
+ do {
+ int i = file.read(b, offset+total, len-total);
+ if (i == -1)
+ throw new IOException("read past EOF");
+ file.position += i;
+ total += i;
+ } while (total < len);
+ }catch(IOException e){
+ throw e;
+ }catch(Exception e){
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ public void close() throws IOException {
+ // only close the file if this is not a clone
+ if (!isClone) file.close();
+ }
+
+ protected void seekInternal(long position) {
+ }
+
+ public long length() {
+ return file.length;
+ }
+
+ public Object clone() {
+ AESIndexInput clone = (AESIndexInput)super.clone();
+ clone.isClone = true;
+ return clone;
+ }
+
+
+ }
+
+
+ protected static class AESIndexOutput extends BufferedIndexOutput {
+ AESWriter file = null;
+
+ // remember if the file is open, so that we don't try to close it
+ // more than once
+ private volatile boolean isOpen;
+
+ public AESIndexOutput(File path, SecretKeySpec key, int blocks_per_page) throws IOException {
+ try{
+ file = new AESWriter(path,key,blocks_per_page);
+ }catch(IOException e){
+ throw e;
+ }catch(Exception e){
+ e.printStackTrace();
+ throw new IOException("Bad File");
+ }
+ isOpen = true;
+ }
+
+ /** output methods: */
+ public void flushBuffer(byte[] b, int offset, int size) throws IOException {
+ try{
+ file.write(b, offset, size);
+ file.flush();
+ }catch(Exception e){
+ throw new RuntimeException(e);
+ }
+ //file.flush();
+ }
+ public void close() throws IOException {
+ // only close the file if it has not been closed yet
+ if (isOpen) {
+ boolean success = false;
+ try {
+ super.close();
+ success = true;
+ } finally {
+ isOpen = false;
+ if (!success) {
+ try {
+ file.close();
+ } catch (Exception t) {
+ // Suppress so we don't mask original exception
+ }
+ } else{
+ try{
+ file.close();
+ }catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+ }
+
+ /** Random-access methods */
+ public void seek(long pos) throws IOException {
+ try{
+ super.seek(pos);
+ file.seek(pos);
+ }catch(Exception e){
+ throw new RuntimeException(e);
+ }
+ }
+ public long length() throws IOException {
+ return file.length();
+ }
+ public void setLength(long length) throws IOException {
+ try{
+ file.setLength(length);
+ }catch(IOException e){
+ throw e;
+ }catch(Exception e){
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
Index: contrib/misc/src/java/org/apache/lucene/util/AESWriter.java
===================================================================
--- contrib/misc/src/java/org/apache/lucene/util/AESWriter.java (revision 0)
+++ contrib/misc/src/java/org/apache/lucene/util/AESWriter.java (revision 0)
@@ -0,0 +1,583 @@
+package org.apache.lucene.util;
+
+import java.io.*;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.KeyGenerator;
+
+import java.util.*;
+
+import java.security.Security;
+/**
+ * AESWriter is responsible for writing AES encrypted files to disk.
+ * This class will abstract the encryption and decryption process from the user.
+ *
+ * All rights reserved by the IIT IR Lab. (c)2009 Jordan Wilberding(jordan@ir.iit.edu) and Jay Mundrawala(mundra@ir.iit.edu)
+ *
+ * @author Jay Mundrawala
+ * @author Jordan Wilberding
+ */
+
+public class AESWriter
+{
+ /* AES using 16 byte block sizes */
+ private static final int BLOCKSIZE = 16;
+ public static final int DEFAULT_PAGESIZE = 32;
+ //public boolean LOG = false;
+
+ private RandomAccessFile raf;
+
+ /* Encryption Cipher */
+ private Cipher ecipher;
+
+ /* Decryption Cipher. This is needed if a seek occurs and entire blocks are not
+ * overwritten. */
+ private Cipher dcipher;
+
+ /* Encryption pending buffer. If there is a block that is not entirly filled, this buffer
+ * will be used. */
+ private byte[] buffer;
+ private byte[] ciphertext;
+
+ /* Initialization vector for buffer */
+ private byte[] cur_iv;
+
+ /* Current byte in the buffer */
+ private int buffer_pos;
+
+ /* Start position of buffer */
+ private long buffer_start;
+
+ /* Number of byte in the buffer */
+ private int buffer_size;
+
+ /* Length of the encrypted file */
+ private long end;
+
+ /* Contains the state of the padding */
+ private boolean isPadded;
+
+ /* Key */
+ private SecretKeySpec key;
+
+ /* Used to generate init vectors */
+ private KeyGenerator ivgen;
+
+ /* Blocks(16 bytes) per page */
+ private int page_size;
+
+ /* Bytes per page */
+ private long page_size_in_bytes;
+
+ /* File pointer in encrypted file...not the physical one */
+ private long cur_fp;
+
+ /* was the buffer modified? */
+ private boolean modified;
+
+ /* An object to sync on */
+ private Boolean lock = new Boolean(false);
+
+ /* File name */
+ private String name;
+
+ /* force gets set if setLength is called.
+ * It means that the file cannot grow after setLength as been called */
+ private boolean force;
+
+ /* Used for logging */
+ //private RandomAccessFile log;
+
+
+ /**
+ * Creates an encrypted random access file that uses the AES encryption algorithm in CBC mode.
+ * @param file File to create.
+ * @param key Key used to initialize the ciphers.
+ * @param page_size Number of 16-byte blocks per page.
+ */
+ public AESWriter(File file, SecretKeySpec key, int page_size) throws Exception
+ {
+ this.raf = new RandomAccessFile(file,"rw");
+ this.key = key;
+ this.name = file.getName();
+
+ /* Used during debugging. Maybe if java had macros, I wouldnt have to comment this stuff out */
+ /*if(LOG){
+ log = new RandomAccessFile("testwritedata/" + name, "rw");
+ log.seek(log.length());
+ }*/
+
+ /* Only allow writing on new files. Lucene specifies that a new writer
+ * will be created only for new files.
+ */
+ if(raf.length() != 0)
+ throw new RuntimeException("File already Exists");
+
+ /* unpadding with dcipher does not work for some reason.
+ * It seems that it wants the last 2 blocks of memory before it
+ * decrypts. That is why unpadding is done manually */
+ this.ecipher = Cipher.getInstance("AES/CBC/NoPadding");
+ this.dcipher = Cipher.getInstance("AES/CBC/NoPadding");
+
+ /* Initialize the ciphers. We should clean this up depending on the mode.
+ * We need 2 only if the mode is allows reading and writing
+ */
+ this.page_size = page_size;
+ this.page_size_in_bytes = BLOCKSIZE*this.page_size;
+
+ /* Initialize the internal buffer. Decrypted blocks are stored here */
+ this.buffer = new byte[BLOCKSIZE*page_size];
+ this.ciphertext = new byte[buffer.length];
+
+ this.isPadded = false;
+ this.ivgen = KeyGenerator.getInstance("AES");
+ this.initCiphers(generateIV());
+ }
+
+
+ /**
+ * Generates a random initialization vector.
+ * @return a randomly generated IV
+ */
+ private IvParameterSpec generateIV(){
+ return new IvParameterSpec(ivgen.generateKey().getEncoded());
+
+ }
+
+ /**
+ * Initialize the encrption and decryption ciphers.
+ * @param ivps The initialization vector to use or null. If null, an IV will be randomly generated.
+ */
+ private void initCiphers(IvParameterSpec ivps) throws java.security.InvalidKeyException,
+ java.security.InvalidAlgorithmParameterException
+ {
+ if(ivps == null)
+ ivps = generateIV();
+
+ /* Store the IV bytes */
+ this.cur_iv = ivps.getIV();
+
+ /* Init the ciphers with ivps */
+ this.ecipher.init(Cipher.ENCRYPT_MODE, this.key,ivps);
+ this.dcipher.init(Cipher.DECRYPT_MODE, this.key,ivps);
+ }
+
+ /**
+ * Flushes all remaining data to disk, pads the file, and then closes the underlying file stream
+ */
+ public void close() throws java.io.IOException, javax.crypto.ShortBufferException,javax.crypto.IllegalBlockSizeException,
+ javax.crypto.BadPaddingException,java.security.InvalidKeyException,java.security.InvalidAlgorithmParameterException
+ {
+ synchronized(lock){
+
+ /*if(LOG)
+ log.writeBytes(name + " w:close\n");
+ */
+ if(!isPadded || modified)
+ writePage(true);
+ if(!this.isPadded){
+ throw new RuntimeException("NO PADDING: this.end=" + this.end + ";this.cur_fp=" + this.cur_fp + ";this.buffer_size="+this.buffer_size +
+ ";this.buffer_pos=" + this.buffer_pos + ";this.buffer_start=" + this.buffer_start + ";this.force=" + this.force);
+ }
+ this.raf.close();
+ }
+ }
+
+ /**
+ * Writes any data in the buffer to disk and any additional padding needed.
+ */
+ public void flush() throws java.io.IOException, javax.crypto.ShortBufferException, javax.crypto.IllegalBlockSizeException,
+ javax.crypto.BadPaddingException,java.security.InvalidKeyException,java.security.InvalidAlgorithmParameterException
+ {
+ synchronized(lock){
+ /*if(LOG)
+ log.writeBytes(name + " w:flush\n");
+ */
+ writePage(true);
+ }
+ }
+
+ /**
+ * Returns the file position in the encrypted file.
+ * @return Returns the current position in this file, where the next write will occur.
+ */
+ public long getFilePointer() throws java.io.IOException
+ {
+ return this.cur_fp;
+ }
+
+ /**
+ * The number of bytes in the file.
+ * @return the size off the file
+ */
+ public long length() throws java.io.IOException
+ {
+ return end;
+ }
+
+ /**
+ * Sets the position where the next write will occurs
+ * @param pos the position where the next write will occur
+ */
+ public void seek(long pos) throws java.io.IOException, javax.crypto.ShortBufferException,javax.crypto.IllegalBlockSizeException,
+ javax.crypto.BadPaddingException,java.security.InvalidKeyException,java.security.InvalidAlgorithmParameterException
+ {
+ synchronized(lock){
+ /*if(LOG)
+ log.writeBytes(name + " w:pos " + pos +"\n");
+ */
+ this._seek(pos);
+ }
+ }
+
+ /**
+ * Sets the position where the next write will occurs
+ * @param pos the position where the next write will occur
+ */
+ private void _seek(long pos) throws java.io.IOException, javax.crypto.ShortBufferException,javax.crypto.IllegalBlockSizeException,
+ javax.crypto.BadPaddingException,java.security.InvalidKeyException,java.security.InvalidAlgorithmParameterException
+ {
+ long _page; /* physical starting address of the page that contains pos */
+
+ /* flush data in buffer */
+ writePage(true);
+
+ /* Convert pos to physical block */
+ long _phys_pos = this.encryptedAddrToPhysicalAddr(pos);
+
+ /* Convert physical block to physical address */
+ _page = _phys_pos/(this.page_size_in_bytes + BLOCKSIZE);
+ _page *= (this.page_size_in_bytes + BLOCKSIZE);
+
+ /* set cur_fp to physical block address */
+ this.raf.seek(_page);
+ this.cur_fp = pos;
+ this.fillBuffer();
+ this.buffer_start = pos/page_size_in_bytes*page_size_in_bytes;
+
+ /* set buffer_pos to point to pos in the buffer */
+ this.buffer_pos = (int)(this.cur_fp % this.page_size_in_bytes);
+
+ }
+
+ /**
+ * Writes an array of bytes to this file.
+ * @param b array of bytes to write
+ * @param off offset in b to start
+ * @param len number of bytes to write
+ */
+ public void write(byte[] b, int off, int len) throws java.io.IOException, javax.crypto.ShortBufferException,
+ javax.crypto.IllegalBlockSizeException,java.security.InvalidKeyException,
+ java.security.InvalidAlgorithmParameterException, javax.crypto.BadPaddingException
+ {
+ int _len;
+ synchronized(lock){
+ /*if(LOG){
+ log.writeBytes(name + " w:write " + b.length + " " + off + " " + len + "\n");
+ log.write(b);
+ }*/
+
+ while(len > 0){
+ this.modified = true;
+
+ //Minimum of len and available is the number of bytes to write
+ _len = (int)Math.min(len, this.page_size_in_bytes - this.buffer_pos);
+ System.arraycopy(b, off, this.buffer, this.buffer_pos, _len);
+
+ if(this.cur_fp + _len > this.end){
+ this.isPadded = false;
+ }
+
+
+ len -= _len;
+ off += _len;
+ this.buffer_pos += _len;
+ this.cur_fp += _len;
+ this.buffer_size = Math.max(this.buffer_size, this.buffer_pos);
+ this.end = Math.max(this.cur_fp, this.end);
+
+ if(this.buffer_pos == this.page_size_in_bytes){
+ /* write current page */
+ this.writePage();
+ /* load next page */
+ this.fillBuffer();
+ }
+
+ }
+ }
+ }
+
+ private void writePage() throws java.io.IOException, javax.crypto.ShortBufferException,
+ javax.crypto.IllegalBlockSizeException,javax.crypto.BadPaddingException,java.security.InvalidKeyException,
+ java.security.InvalidAlgorithmParameterException
+ {
+ writePage(false);
+ }
+
+ /**
+ * Writes the internal buffer to disk.
+ * @param flush_pad if set to true, padding is guaranteed to be added. Otherwise, padding will only be added if
+ * there are not enough bytes to make a complete block.
+ */
+ private void writePage(boolean flush_pad) throws java.io.IOException, javax.crypto.ShortBufferException,
+ javax.crypto.IllegalBlockSizeException,javax.crypto.BadPaddingException,java.security.InvalidKeyException,
+ java.security.InvalidAlgorithmParameterException
+ {
+
+ this.modified = false;
+
+ /* set underlying file position */
+ this.raf.seek(encryptedAddrToPhysicalAddr(this.buffer_start) - BLOCKSIZE);
+ this.raf.write(this.cur_iv);
+
+ /* determine number of bytes to write */
+ int len = (this.buffer_size + BLOCKSIZE-1)/BLOCKSIZE*BLOCKSIZE;
+
+ /* number of padding bytes */
+ int no_padding = BLOCKSIZE - (this.buffer_size % BLOCKSIZE);
+
+
+ if(no_padding < BLOCKSIZE && no_padding > 0)
+ {
+ /* Does not need a full block of padding...easy */
+ this.isPadded = true;
+ for(int i = buffer_size; i < len;i++){
+ this.buffer[i]=(byte)no_padding;
+ }
+ }else if(flush_pad && no_padding == BLOCKSIZE && this.cur_fp/page_size_in_bytes == this.end/page_size_in_bytes){
+ /* Needs a full block of padding...only pad if on the last page and flush_pad is true */
+ if(!this.isPadded)
+ {
+ /* Write padding bytes to buffer */
+ for(int i = len; i< len + BLOCKSIZE; i++)
+ this.buffer[i] = BLOCKSIZE;
+ if(this.page_size_in_bytes - this.buffer_size >= BLOCKSIZE){
+ /* Make sure the page isnt full...should never happen */
+ this.isPadded = true;
+ len += BLOCKSIZE;
+ }else{
+ throw new RuntimeException("Page should not be full: " + buffer_size);
+ }
+ }
+ }
+
+ /* Encrypt data and write to disk */
+ this.ecipher.doFinal(this.buffer,0,len,this.ciphertext,0);
+ this.raf.write(this.ciphertext,0,len);
+
+ }
+
+ /**
+ * Reads a page from the underlying file and initialize the ciphers.
+ * If there is no page to read, the ciphers get initialized
+ * with a random IV. */
+ private void fillBuffer() throws java.io.IOException, javax.crypto.ShortBufferException,
+ javax.crypto.IllegalBlockSizeException, java.security.InvalidKeyException,
+ java.security.InvalidAlgorithmParameterException, javax.crypto.BadPaddingException
+ {
+ int _end;
+ long _cur_fp;
+ byte[] _iv;
+
+ _iv = new byte[BLOCKSIZE];
+
+ this.buffer_start = this.cur_fp/page_size_in_bytes*page_size_in_bytes;
+ _cur_fp = encryptedAddrToPhysicalAddr(this.buffer_start) - BLOCKSIZE;
+
+ /* Read IV */
+ this.raf.seek(_cur_fp);
+ if(_cur_fp >= this.raf.length()){
+ /* generate a random IV and write it to disk */
+ _iv = generateIV().getIV();
+ this.raf.write(_iv);
+ }else{
+ this.raf.readFully(_iv);
+ }
+
+ _cur_fp += BLOCKSIZE;
+
+ /* Read page */
+ this.buffer_size = _end = this.raf.read(this.buffer,0, (int)this.page_size_in_bytes);
+
+ if(_end == -1){
+ this.buffer_size = 0;
+ this.initCiphers(new IvParameterSpec(_iv));
+ this.buffer_pos = 0;
+ return;
+ }
+
+ /* reinit ciphers and decrypt page */
+ this.initCiphers(new IvParameterSpec(_iv));
+ this.buffer_pos = 0;
+ this.dcipher.doFinal(this.buffer,0,_end,this.buffer,0);
+ }
+
+
+ /**
+ * Set the size of the file. To use this method, it must be called immediately after this object is created.
+ * Also, when this method is called, the file length may not be extended.
+ * @param newLen The new size of the file.
+ */
+ public void setLength(long newLen) throws java.io.IOException,javax.crypto.ShortBufferException,
+ javax.crypto.BadPaddingException, java.security.InvalidKeyException, javax.crypto.IllegalBlockSizeException,
+ java.security.InvalidAlgorithmParameterException
+ {
+ synchronized(lock){
+ /*if(LOG)
+ log.writeBytes(name + " w:setLength " + newLen + "\n");
+ */
+ force = true;
+ long _len = newLen + BLOCKSIZE - (newLen % BLOCKSIZE);
+ int no_padding = (int)(BLOCKSIZE - (newLen % BLOCKSIZE));
+ long num_blocks = (_len + BLOCKSIZE - 1)/BLOCKSIZE;
+ long num_pages = (num_blocks + this.page_size-1)/this.page_size;
+
+ this.raf.setLength(_len + num_pages*BLOCKSIZE);
+ this.end = newLen;
+
+ byte[] _iv;
+ this.raf.seek(0);
+ this.buffer_start = 0;
+
+ long _cur = 0;
+
+ for(int i =0; i < num_pages; i++){
+ if(this.raf.getFilePointer() % (page_size_in_bytes + BLOCKSIZE) != 0)
+ throw new RuntimeException("Debug::Incorrect file postion;fp=" + this.raf.getFilePointer());
+
+ if(i == num_pages - 1){
+ /* Last page needs padding */
+ _iv = generateIV().getIV();
+ this.raf.write(_iv);
+ int _num = (int)(_len - _cur);
+ byte[] data = new byte[_num];
+ for(int j = _num -1; j >= _num - no_padding; j--)
+ {
+ data[j] = (byte)no_padding;
+ }
+ this.initCiphers(new IvParameterSpec(_iv));
+ this.ecipher.doFinal(data,0,_num,data,0);
+ this.raf.write(data);
+
+ this.isPadded = true;
+ _cur += _num;
+
+
+ }else{
+ /* generate and write random IV to file */
+ _iv = generateIV().getIV();
+ this.raf.write(_iv);
+ this.raf.seek(this.raf.getFilePointer() + page_size_in_bytes);
+
+ _cur+= page_size_in_bytes;
+ }
+
+
+ }
+ /* Seek to the beginning of the file */
+ this.raf.seek(0);
+ this.cur_fp = 0;
+ this.buffer_start = 0;
+ this.buffer_pos = 0;
+ /* load first page */
+ this.fillBuffer();
+ }
+ }
+
+ private static void printBarr(byte[] p){
+ for(int i =0; i < p.length; i++)
+ System.out.print(p[i] + " ");
+ System.out.println();
+ }
+
+ /**
+ * Calculates the number of init vectors preceding a given block. The block of virtual address m
+ * is determined by m/BLOCKSIZE.
+ * @param block how many IVs are found before this block
+ * @return number of IVs found before block
+ */
+ private long numPageIVBlocksAt(long block){
+ return (block/((long)page_size)) +1;
+ }
+
+ /**
+ * Calculates the physical address of the given virtual address.
+ * @param m the virtual file pointer
+ * @return the address of where m actually lies in the underlying file
+ */
+ private long encryptedAddrToPhysicalAddr(long m){
+ long block = m/BLOCKSIZE;
+ return ((block + (numPageIVBlocksAt(block))) * BLOCKSIZE) + (m % BLOCKSIZE);
+ }
+
+ /**
+ * A simple test program.
+ */
+
+ public static void main(String[] args) throws Exception{
+ String file;
+ int page_size;
+ long file_size;
+ Random rand = new Random(100);
+ file = args[0];
+ page_size = Integer.parseInt(args[1]);
+ file_size = Long.parseLong(args[2]);
+
+ SecretKeySpec key = new SecretKeySpec(new byte[]{
+ 0x00, 0x01, 0x00, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
+ },"AES");
+
+ AESWriter writer = new AESWriter(new File(file),key,page_size);
+ writer.setLength(file_size);
+ byte[] b = new byte[(int)file_size];
+ rand.nextBytes(b);
+ writer.write(b,0,b.length);
+
+ System.out.println("Filesize: " + writer.length());
+ int iterations = 200;
+
+ for(int i = 0; i < iterations; i++){
+ long pos = (long)rand.nextInt((int)file_size);
+ int len = rand.nextInt((int)file_size - (int)pos);
+ writer.seek(pos);
+ if(writer.getFilePointer() != pos){
+ System.err.println("FilePos not properly set");
+ System.exit(1);
+ }
+ //System.out.println("Pos=" + pos + ";Len=" + len);
+
+ for(int j = (int)pos; j < (int)pos+len; j++){
+ b[j] = (byte)(rand.nextInt() & 0xFF);
+ }
+ writer.write(b,(int)pos,len);
+ }
+
+
+ writer.close();
+
+ AESReader reader = new AESReader(new File(file),key,page_size);
+ if(reader.length() != file_size)
+ {
+ System.out.println("Incorrect Filesize");
+ System.exit(1);
+ }
+ for(int i =0; i < reader.length(); i++)
+ {
+ int br = (reader.read() & 0xFF);
+ int bs = b[i] & 0xFF;
+ if(br != bs){
+ System.out.println("data not correctly written");
+ System.out.println("Expected=" + bs + ";Got=" + br);
+ System.exit(1);
+ }
+ }
+ System.out.println("Success");
+
+ }
+
+}
Index: contrib/misc/src/java/org/apache/lucene/util/AESReader.java
===================================================================
--- contrib/misc/src/java/org/apache/lucene/util/AESReader.java (revision 0)
+++ contrib/misc/src/java/org/apache/lucene/util/AESReader.java (revision 0)
@@ -0,0 +1,511 @@
+package org.apache.lucene.util;
+
+import java.io.*;
+import java.io.DataOutput;
+import java.io.DataInput;
+import java.io.RandomAccessFile;
+import java.io.File;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.KeyGenerator;
+import java.security.Security;
+
+import java.util.Random;
+
+/**
+ * AESReader provides the ability to read and write to an AES encrypted random access file.
+ *
+ * All rights reserved by the IIT IR Lab. (c)2009 Jordan Wilberding(jordan@ir.iit.edu) and Jay Mundrawala(mundra@ir.iit.edu)
+ *
+ * @author Jay Mundrawala
+ * @author Jordan Wilberding
+ */
+public class AESReader
+{
+ /* AES using 16 byte block sizes */
+ private static final int BLOCKSIZE = 16;
+ //public boolean LOG = false;
+
+ /* Underlying file to write to */
+ private RandomAccessFile raf;
+
+ /* Decryption Cipher */
+ private Cipher dcipher;
+
+ /* Current Initialization Vector */
+ private byte[] cur_iv;
+
+ /* Encryption/Decryption buffer */
+ private byte[] buffer;
+
+ /* Internal filePos. We cannot use raf's because that one
+ * will always be aligned an a 16 byte boundary
+ */
+ private long filePos;
+
+ /* Start location of buffer in physical file */
+ private long bufferStart;
+
+ /* Number of valid bytes in the buffer */
+ private int bufferLength;
+
+ /* Current position in buffer */
+ private int bufferPosition;
+
+ /* address of last block. */
+ private long end;
+
+ /* Blocks per page */
+ private int page_size;
+
+ /* Key used to decrypt data */
+ private SecretKeySpec key;
+
+ /* Object to sync on */
+ private Boolean lock = new Boolean(false);
+
+ /* Name of file */
+ private String name;
+
+ //private RandomAccessFile log;
+
+ /**
+ * Creates an encrypted random access file reader that uses the AES encryption algorithm in CBC mode.
+ * @param file File to read.
+ * @param key Key used to initialize the ciphers.
+ * @param page_size Number of 16-byte blocks per page. Must be the same number used when writing the file.
+ */
+ public AESReader(File file, SecretKeySpec key, int page_size) throws IOException,
+ java.security.NoSuchAlgorithmException,java.security.InvalidKeyException,
+ javax.crypto.ShortBufferException,javax.crypto.NoSuchPaddingException,
+ java.security.InvalidAlgorithmParameterException,
+ javax.crypto.IllegalBlockSizeException,javax.crypto.IllegalBlockSizeException,
+ javax.crypto.BadPaddingException,javax.crypto.BadPaddingException
+ {
+ /*
+ if(LOG){
+ log = new RandomAccessFile("testreaddata/" + name,"rw");
+ log.seek(log.length());
+ }
+ */
+ long page_size_in_bytes;
+ long prev_pos;
+ int nread;
+ int buf_size;
+ int no_padding;
+ int no_data;
+
+ this.name = file.getName();
+ this.raf = new RandomAccessFile(file,"r");
+ page_size_in_bytes = page_size*BLOCKSIZE;
+ prev_pos = this.raf.getFilePointer();
+ this.dcipher = Cipher.getInstance("AES/CBC/NoPadding");
+ this.buffer = new byte[page_size*BLOCKSIZE];
+ this.cur_iv = new byte[BLOCKSIZE];
+ this.key = key;
+ this.page_size = page_size;
+
+ //check padding and determine end(file length)
+ this.raf.seek(Math.max(this.raf.length() - page_size_in_bytes,0));
+ this.raf.readFully(this.cur_iv);
+
+ nread = this.raf.read(buffer);
+ dcipher.init(Cipher.DECRYPT_MODE,key, new IvParameterSpec(this.cur_iv));
+ buf_size = dcipher.doFinal(buffer,0,nread,buffer,0);
+
+ if(buf_size != nread)
+ throw new IOException("Not enough bytes decrypted");
+
+ no_padding = buffer[buf_size - 1];
+ no_data = BLOCKSIZE - no_padding;
+
+ if(no_data < 0 || no_data > BLOCKSIZE)
+ throw new IOException("Bad padding: " + no_padding);
+
+ for(int i = buf_size - BLOCKSIZE + no_data; i < buf_size; i++){
+ if(no_padding != buffer[i])
+ throw new IOException(
+ "Bad padding @ byte " + (buf_size - i) + ". Expected: "
+ + no_padding + ". Value: " + buffer[i]
+ );
+
+ }
+
+ long blocks = this.raf.length()/BLOCKSIZE -1;
+ long pageivs = blocks/(page_size+1) + 1;
+ this.end = this.raf.length() - no_padding - (pageivs * BLOCKSIZE);
+
+
+
+ //refill the buffer
+ seek(prev_pos);
+
+ }
+
+ /**
+ * Creates an encrypted random access file reader that uses the AES encryption algorithm in CBC mode.
+ * @param name File to read.
+ * @param key Key used to initialize the ciphers.
+ * @param page_size Number of 16-byte blocks per page. Must be the same number used when writing the file.
+ */
+ public AESReader(String name,SecretKeySpec key, int page_size) throws IOException,
+ java.security.NoSuchAlgorithmException,java.security.InvalidKeyException,
+ javax.crypto.ShortBufferException, javax.crypto.NoSuchPaddingException,
+ java.security.InvalidAlgorithmParameterException,
+ javax.crypto.IllegalBlockSizeException,javax.crypto.IllegalBlockSizeException,
+ javax.crypto.BadPaddingException,javax.crypto.BadPaddingException
+ {
+ this(new File(name),key, page_size);
+ }
+
+ /**
+ * Close the underlying RandomAccessFile
+ */
+ public void close() throws java.io.IOException
+ {
+ /*if(LOG)
+ log.writeBytes(name + " r:close\n");
+ */
+ this.raf.close();
+ }
+
+ /**
+ * Get the current position in the file
+ */
+ public long getFilePointer() throws java.io.IOException
+ {
+ return this.filePos;
+ }
+
+ /**
+ * Get the number of bytes in the encrypted file. This size is equal to the physical file size
+ * minus the number of padding blocks and IV blocks
+ * @return size of file
+ */
+ public long length()
+ {
+ return this.end;
+ }
+
+ /** Read the next byte from the file.
+ * @return -1 if eof has been reached, the next byte otherwise.
+ */
+ public int read() throws java.io.IOException, javax.crypto.ShortBufferException,
+ javax.crypto.IllegalBlockSizeException,javax.crypto.BadPaddingException,
+ java.security.InvalidAlgorithmParameterException,java.security.InvalidKeyException
+ {
+ byte[] b = new byte[1];
+ int numRead = read(b);
+ if(numRead == -1)
+ return -1;
+
+ return (int) b[0] & 0xFF;
+
+ }
+
+ /**
+ * Try to fill the given buffer with the next bytes from the file.
+ * @param b byte array to fill with bytes
+ * @return -1 if eof has been reached, the number of bytes copied into the given buffer otherwise.
+ */
+ public int read(byte[] b) throws java.io.IOException, javax.crypto.ShortBufferException,
+ javax.crypto.IllegalBlockSizeException,javax.crypto.BadPaddingException,
+ java.security.InvalidAlgorithmParameterException,java.security.InvalidKeyException
+ {
+ return this.read(b, 0,b.length);
+ }
+
+
+ /**
+ * Read bytes from the file into the given byte array
+ * @param b byte array to copy bytes to
+ * @param offset position in b to start copying data
+ * @param len number of bytes to copy to the given byte array
+ * @return -1 if eof has been reached, the number of bytes copied into b otherwise.
+ */
+ public int read(byte[] b, int offset, int len) throws java.io.IOException,
+ javax.crypto.ShortBufferException, javax.crypto.IllegalBlockSizeException,
+ javax.crypto.BadPaddingException,java.security.InvalidKeyException,
+ java.security.InvalidAlgorithmParameterException
+ {
+
+ if(this.filePos >= this.end){
+ return -1;
+ }
+ if(len <= 0)
+ return 0;
+
+ synchronized(lock){
+ /* if(LOG)
+ log.writeBytes(name + " r:read "+ b.length + " " + offset + " " + len + "\n");
+ */
+ int remaining = len;
+ /* Time to get next page when position in the buffer is geq its length */
+ if(bufferPosition >= bufferLength)
+ refill();
+
+ if(len <= bufferLength - bufferPosition){
+ /* Enough bytes in the internal buffer...just copy them to b */
+ System.arraycopy(buffer,bufferPosition,b,offset,len);
+ bufferPosition += len;
+ filePos += len;
+ remaining = 0;
+ }else{
+ /* Will need to start loading next pages to read len bytes */
+ while(remaining > 0 && filePos < end){
+ int available = bufferLength - bufferPosition;
+ if(available > 0){
+ int to_read = Math.min(available,remaining);
+ System.arraycopy(buffer,bufferPosition,b,offset,to_read);
+ remaining -= to_read;
+ offset += to_read;
+ bufferPosition += to_read;
+ filePos += to_read;
+ }else{
+ refill();
+ }
+ }
+ }
+
+ return len - remaining;
+ }
+ }
+
+ /**
+ * Sets the file pointer so that the next byte read will be at pos.
+ * Seeking past the end of the file is not allowed.
+ * @param pos position to seek to.
+ */
+ public void seek(long pos) throws java.io.IOException,javax.crypto.ShortBufferException,
+ javax.crypto.IllegalBlockSizeException,javax.crypto.BadPaddingException,
+ java.security.InvalidKeyException,java.security.InvalidAlgorithmParameterException
+ {
+ /* flush buffer */
+ if(pos >= end || pos < 0){
+ throw new RuntimeException("Pos: " + pos + " end: " + end + " file: " + name);
+ }
+ synchronized(lock){
+ /*if(LOG)
+ log.writeBytes(name + " r:seek " + pos + "\n");
+ */
+ this.filePos = pos;
+ refill();
+
+ }
+ }
+
+ /**
+ * refill will make sure that this.filePos is in the internal buffer and decrypted. It reads
+ * 1 page from disk including the IV used to encrypt the page, and decrpyts the page which is
+ * then stored in the internal buffer.
+ */
+ private void refill() throws java.io.IOException,javax.crypto.ShortBufferException,
+ javax.crypto.IllegalBlockSizeException,javax.crypto.BadPaddingException,
+ java.security.InvalidKeyException,java.security.InvalidAlgorithmParameterException
+ {
+ int buf_size;
+ int nread;
+
+ /* address of filePos in the file */
+ long strt_addr = encryptedAddrToPhysicalAddr(this.filePos);
+
+ /* set bufferStart to the first byte of the IV of the page that contains
+ * filePos.
+ */
+ this.bufferStart = strt_addr/((long)page_size*BLOCKSIZE + BLOCKSIZE);
+ this.bufferStart *= ((long)page_size*BLOCKSIZE + BLOCKSIZE);
+
+ /* Seek to the IV */
+ this.raf.seek(bufferStart);
+ this.bufferStart += BLOCKSIZE;
+
+ /* read the IV */
+ this.raf.readFully(this.cur_iv);
+
+ /* initialize the cipher with the IV that was read */
+ this.dcipher.init(Cipher.DECRYPT_MODE,this.key,new IvParameterSpec(this.cur_iv));
+
+ /* Read and decrypt the ciphertext */
+ nread = this.raf.read(buffer);
+ buf_size = dcipher.doFinal(buffer,0,nread,buffer,0);
+
+ if(buf_size != nread)
+ throw new IOException("Not enough bytes decrypted");
+
+ this.bufferLength = buf_size;
+ this.bufferPosition = (int)(this.filePos % buffer.length);
+
+ }
+
+ /**
+ * Calculates the number of init vectors preceding a given block. The block of virtual address m
+ * is determined by m/BLOCKSIZE.
+ * @param block how many IVs are found before this block
+ * @return number of IVs found before block
+ */
+
+ private long numPageIVBlocksAt(long block){
+ return (block/((long)page_size)) +1;
+ }
+
+ /**
+ * Calculates the physical address of the given virtual address.
+ * @param m the virtual file pointer
+ * @return the address of where m actually lies in the underlying file
+ */
+ private long encryptedAddrToPhysicalAddr(long m){
+ long block = m/BLOCKSIZE;
+ return ((block + (numPageIVBlocksAt(block))) * BLOCKSIZE) + (m % BLOCKSIZE);
+ }
+
+ /**
+ * A simple test program.
+ */
+ public static void main(String[] args) throws Exception
+ {
+ int pagesize = 4;
+ int nbytes = 0;
+ String file_name = null;
+ File f;
+ RandomAccessFile raf;
+ byte[] b;
+ IvParameterSpec iv;
+ AESReader reader;
+ Random rand;
+ Cipher ecipher;
+
+ KeyGenerator ivgen = KeyGenerator.getInstance("AES");
+
+ SecretKeySpec key = new SecretKeySpec(new byte[]{
+ 0x00, 0x01, 0x00, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
+ },"AES");
+
+ if(args.length != 2 && args.length != 3){
+ System.out.println("Usage: java org.apache.lucene.AESReader file_name file_size [blocks per page]");
+ System.exit(1);
+ }else{
+ try{
+ file_name = args[0];
+ nbytes = Integer.parseInt(args[1]);
+ }catch(Exception e){
+ System.out.println("Usage: java org.apache.lucene.AESReader file_name file_size [blocks per page]");
+ System.exit(1);
+ }
+ try{
+ pagesize = Integer.parseInt(args[2]);
+ }catch(Exception e){
+
+ }
+ }
+ System.out.println("File: " + file_name);
+ System.out.println("File Size: " + nbytes);
+ System.out.println("Blocks Per Page: " + pagesize);
+
+ f = new File(file_name);
+ raf = new RandomAccessFile(f, "rw");
+ b = new byte[nbytes + BLOCKSIZE - (nbytes % BLOCKSIZE)];
+
+ System.out.println("Bytes: " + b.length);
+ rand = new Random();
+ rand.nextBytes(b);
+
+ ecipher = Cipher.getInstance("AES/CBC/NoPadding");
+
+
+ int no_padding = b.length - nbytes;
+ int page_size = BLOCKSIZE*pagesize;
+ int num_blocks = (b.length + BLOCKSIZE - 1)/BLOCKSIZE;
+ //System.out.println("Num blocks: " + num_blocks);
+ int num_pages = (num_blocks+pagesize-1)/pagesize;
+ int cur = 0;
+
+ for(int i = nbytes; i < b.length; i++){
+ b[i] = (byte)no_padding;
+ //System.out.println("adding padding: " + b[i]);
+ }
+
+ //System.out.println("Num Pages: " + num_pages);
+
+ for(int i = 0; i < num_pages; i++)
+ {
+ iv = generateIV(ivgen);
+ ecipher.init(Cipher.ENCRYPT_MODE, key, iv);
+ //System.out.println("Writing IV at " + raf.getFilePointer());
+ byte[] ivba = iv.getIV();
+ raf.write(ivba,0,ivba.length);
+ //System.out.println("Write IV");
+ /*
+ for(int j = 0; j < ivba.length; j++)
+ System.out.print(ivba[j] + " ");
+ System.out.println();
+ */
+
+ byte[] data;
+ if(i == num_pages -1){
+ int num = b.length - cur;
+ data = new byte[num];
+ ecipher.doFinal(b,cur,num,data,0);
+ cur += num;
+ }else{
+ data = new byte[page_size];
+ ecipher.doFinal(b,cur,page_size,data,0);
+ cur += page_size;
+ }
+ raf.write(data);
+ }
+
+
+
+ reader = new AESReader(f,key,pagesize);
+
+ if(reader.length() != nbytes){
+ System.err.println("Incorrect file size: Expected=" + nbytes + ";Got=" + reader.length());
+ }
+ System.out.println("Checking file sequentially");
+ for(int i = 0; i < nbytes; i++)
+ {
+ byte r = (byte)reader.read();
+ if(r != b[i]){
+ System.err.println(i + ": Read=" + r + ";Expected=" + b[i]);
+ System.exit(1);
+ }
+ }
+ System.out.println("Checking with seeks");
+ int iterations = nbytes;
+ for(int i = 0; i < iterations; i++){
+ if(i%100 == 0)
+ System.out.println("Iteration " + i);
+ byte[] data = new byte[b.length];
+ long pos = (long)rand.nextInt(nbytes);
+ int len = rand.nextInt(nbytes - (int)pos);
+ reader.seek(pos);
+ if(reader.getFilePointer() != pos){
+ System.err.println("FilePos not properly set");
+ System.exit(1);
+ }
+ //System.out.println("Pos=" + pos + ";Len=" + len);
+ reader.read(data,(int)pos,len);
+
+ for(int j = (int)pos; j < (int)pos+len; j++){
+ if(data[j] != b[j]){
+ System.err.println("data[j] != b[j];j=" + j);
+ System.exit(1);
+ }
+ }
+
+ }
+
+ System.err.println("Success");
+ System.exit(0);
+
+ }
+ private static IvParameterSpec generateIV(KeyGenerator ivgen){
+ return new IvParameterSpec(ivgen.generateKey().getEncoded());
+ }
+
+}