From 39f1aad52f6f02e55b25fe01ed39b9c1c84dc8a5 Mon Sep 17 00:00:00 2001 From: Mihai Soloi Date: Fri, 31 Aug 2012 19:29:35 +0200 Subject: [PATCH 1/3] addded maven sub-module for hbase simple search --- .gitignore | 14 + hbase-simple-search/README.txt | 13 + hbase-simple-search/pom.xml | 133 ++++ .../hbase/index/MessageSearchIndexListener.java | 533 ++++++++++++++ .../james/mailbox/hbase/store/HBaseIndexStore.java | 136 ++++ .../james/mailbox/hbase/store/HBaseNames.java | 35 + .../james/mailbox/hbase/store/MessageFields.java | 31 + .../hbase/store/endpoint/RowFilteringEndpoint.java | 142 ++++ .../hbase/store/endpoint/RowFilteringProtocol.java | 15 + .../src/main/resources/hbase-site.xml | 49 ++ .../index/MessageSearchIndexListenerTest.java | 776 +++++++++++++++++++++ .../james/mailbox/hbase/store/MessageBuilder.java | 62 ++ .../hbase/store/SimpleMailboxMembership.java | 295 ++++++++ .../src/test/resources/hbase-site.xml | 66 ++ .../src/test/resources/hdfs-site.xml | 36 + pom.xml | 1 + 16 files changed, 2337 insertions(+) create mode 100644 .gitignore create mode 100644 hbase-simple-search/README.txt create mode 100644 hbase-simple-search/pom.xml create mode 100644 hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/index/MessageSearchIndexListener.java create mode 100644 hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/HBaseIndexStore.java create mode 100644 hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/HBaseNames.java create mode 100644 hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/MessageFields.java create mode 100644 hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/endpoint/RowFilteringEndpoint.java create mode 100644 hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/endpoint/RowFilteringProtocol.java create mode 100644 hbase-simple-search/src/main/resources/hbase-site.xml create mode 100644 hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/index/MessageSearchIndexListenerTest.java create mode 100644 hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/store/MessageBuilder.java create mode 100644 hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/store/SimpleMailboxMembership.java create mode 100644 hbase-simple-search/src/test/resources/hbase-site.xml create mode 100644 hbase-simple-search/src/test/resources/hdfs-site.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b463c4f --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +/.settings +*/target +target +/.classpath +/.project +/build +/var +*log +.metadata +*.idea +*.iml +*.ipr +*.iws +*/*.iml diff --git a/hbase-simple-search/README.txt b/hbase-simple-search/README.txt new file mode 100644 index 0000000..2f4f423 --- /dev/null +++ b/hbase-simple-search/README.txt @@ -0,0 +1,13 @@ +The project is basically an inverted index in an HBase table to search through the mails in a mailbox. + +The structure of the index is as follows. + + 1. mailboxID is an java.util.UUID + 2. the fields are now Enums, and what is stored is a byte that identifies that enum field. + 3. each of the terms in the fields are tokenized using the lucene org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer, but some fields are not tokenized due to their nature(SENT_DATE for example) + +The row is composed of all the above byte arrays concatenated, so that searching can be done very fast through the HBase table, as well as lookup on the specific mailbox and field in the mail. The mailID is the qualifier in the static column family(only one column family) so that mail id's are found with relative ease. + +This is for the mail document in itself, the flags are stored in a single row in the table(one row for each mailbox) and can be found easily by a scan. Each of the rows now has an empty value, where in the possible future we'll be able to store data related to the term frequency in the document. + +What works currently are the searches based on the text, flags, headers, all criterions. These are implemented using Filters but I will be switching to Coprocessors till next Monday due to the benefit they provide of less data transfer over the network and distributed processing on each region. diff --git a/hbase-simple-search/pom.xml b/hbase-simple-search/pom.xml new file mode 100644 index 0000000..4f169d4 --- /dev/null +++ b/hbase-simple-search/pom.xml @@ -0,0 +1,133 @@ + + 4.0.0 + + + apache-james-mailbox + org.apache.james + 0.5-SNAPSHOT + ../pom.xml + + + apache-james-mailbox-hbase-simple-search + 0.5-SNAPSHOT + + Apache James Mailbox HBase Simple Search + Adds simple search capability to Apache James Mailbox HBase implementation + + + + maven-compiler-plugin + + 1.6 + 1.6 + + 2.5.1 + + + + + + lucene-repository + Lucene Maven + https://repository.apache.org/snapshots/ + + true + always + + + + + + 4.0-SNAPSHOT + + + + + org.apache.lucene + lucene-core + ${lucene.version} + + + org.apache.lucene + lucene-analyzers-common + ${lucene.version} + + + junit + junit + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-simple + test + + + org.apache.hbase + hbase + + + org.jruby + jruby-complete + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + commons-logging + commons-logging + + + + + org.apache.hbase + hbase + test-jar + test + + + org.jruby + jruby-complete + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + commons-logging + commons-logging + + + + + org.apache.hadoop + hadoop-test + test + + + org.apache.hadoop + hadoop-core + + + org.apache.james + apache-james-mailbox-lucene + + + org.apache.james + apache-james-mailbox-store + + + diff --git a/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/index/MessageSearchIndexListener.java b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/index/MessageSearchIndexListener.java new file mode 100644 index 0000000..e6cbc38 --- /dev/null +++ b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/index/MessageSearchIndexListener.java @@ -0,0 +1,533 @@ +package org.apache.james.mailbox.hbase.index; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.james.mailbox.MailboxSession; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.exception.UnsupportedSearchException; +import org.apache.james.mailbox.hbase.store.HBaseIndexStore; +import org.apache.james.mailbox.hbase.store.HBaseNames; +import org.apache.james.mailbox.hbase.store.MessageFields; +import org.apache.james.mailbox.model.MessageRange; +import org.apache.james.mailbox.model.SearchQuery; +import org.apache.james.mailbox.store.mail.MessageMapperFactory; +import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.Message; +import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex; +import org.apache.james.mailbox.store.search.SearchUtil; +import org.apache.james.mime4j.MimeException; +import org.apache.james.mime4j.dom.Header; +import org.apache.james.mime4j.dom.address.Address; +import org.apache.james.mime4j.dom.address.AddressList; +import org.apache.james.mime4j.dom.address.Group; +import org.apache.james.mime4j.dom.address.MailboxList; +import org.apache.james.mime4j.field.address.AddressFormatter; +import org.apache.james.mime4j.field.address.LenientAddressBuilder; +import org.apache.james.mime4j.message.SimpleContentHandler; +import org.apache.james.mime4j.parser.MimeStreamParser; +import org.apache.james.mime4j.stream.BodyDescriptor; +import org.apache.james.mime4j.stream.MimeConfig; +import org.apache.james.mime4j.util.MimeUtil; +import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.document.DateTools; +import org.apache.lucene.util.Version; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.mail.Flags; +import java.io.*; +import java.nio.charset.Charset; +import java.text.NumberFormat; +import java.text.ParseException; +import java.util.*; + +import static javax.mail.Flags.Flag; +import static javax.mail.Flags.Flag.*; +import static org.apache.james.mailbox.hbase.store.HBaseNames.EMPTY_COLUMN_VALUE; +import static org.apache.james.mailbox.hbase.store.MessageFields.*; + +public class MessageSearchIndexListener extends ListeningMessageSearchIndex { + + private final static Logger LOG = LoggerFactory.getLogger(MessageSearchIndexListener.class); + + private final static String MEDIA_TYPE_TEXT = "text"; + private final static String MEDIA_TYPE_MESSAGE = "message"; + private final static String DEFAULT_ENCODING = "US-ASCII"; + private HBaseIndexStore store; + + public MessageSearchIndexListener(MessageMapperFactory factory, HBaseIndexStore store) throws IOException { + super(factory); + this.store = store; + } + + @Override + public void add(MailboxSession session, Mailbox mailbox, Message message) throws MailboxException { + try { + store.storeMail(indexMessage(message)); + } catch (IOException e) { + throw new MailboxException("Problem adding the mail " + message.getUid() + + " in mailbox " + message.getMailboxId() + " to the storage!", e); + } finally { + try { + store.flushToStore(); + } catch (IOException e) { + LOG.warn("Storing index in table has failed."); + } + } + } + + private List indexMessage(Message message) throws MailboxException { + final List puts = Lists.newArrayList(); + final UUID mailboxId = message.getMailboxId(); + final long messageId = message.getUid(); + //add flags + Put put = new Put(Bytes.add(uuidToBytes(mailboxId), new byte[]{FLAGS_FIELD.id})); + put.add(HBaseNames.COLUMN_FAMILY.name, Bytes.toBytes(messageId), Bytes.toBytes(parseFlagsContent(message))); + puts.add(put); + //add full content + for (Map.Entry entry : parseFullContent(message).entries()) { + put = new Put(Bytes.add(uuidToBytes(mailboxId), new byte[]{entry.getKey().id}, Bytes.toBytes(entry.getValue()))); + put.add(HBaseNames.COLUMN_FAMILY.name, Bytes.toBytes(messageId), EMPTY_COLUMN_VALUE.name); + puts.add(put); + } + return puts; + } + + public static byte[] uuidToBytes(UUID uuid) { + return Bytes.add(Bytes.toBytes(uuid.getMostSignificantBits()), + Bytes.toBytes(uuid.getLeastSignificantBits())); + } + + public static UUID rowToUUID(byte[] row) { + byte[] uuidz = Bytes.head(row, 16); + return new UUID(Bytes.toLong(Bytes.head(uuidz, 8)), Bytes.toLong(Bytes.tail(uuidz, 8))); + } + + public static MessageFields rowToField(byte[] row) { + byte[] fieldRead = Bytes.tail(Bytes.head(row, 17), 1); + for (MessageFields field : MessageFields.values()) + if (field.id == fieldRead[0]) + return field; + return NOT_FOUND; + } + + public static String rowToTerm(byte[] row) { + byte[] term = Bytes.tail(row, row.length - 17); + return Bytes.toString(term); + } + + public static String row(byte[] row){ + return rowToUUID(row)+rowToField(row).name()+rowToTerm(row); + } + + @Override + public void delete(MailboxSession session, Mailbox mailbox, MessageRange range) throws MailboxException { + // delete a message from mailbox - maybe just mark it in a list and perform the delete on HBase compactions + for (Long messageId : range) { + ResultScanner scanner = null; + try { + scanner = store.retrieveMails(uuidToBytes(mailbox.getMailboxId()), messageId); + for (Result result : scanner) { + store.deleteMail(result.getRow(), messageId); + } + } catch (IOException e) { + LOG.warn("Couldn't delete mail from mailbox"); + } finally { + try { + store.flushToStore(); + scanner.close(); + } catch (IOException e) { + LOG.warn("Storing index in table has failed."); + } + } + } + } + + + /** + * all previous flags are deleted upon update + * + * @param session + * @param mailbox + * @param range + * @param flags + * @throws MailboxException + */ + @Override + public void update(MailboxSession session, Mailbox mailbox, MessageRange range, Flags flags) throws MailboxException { + // update the cells that changed - this means update the flags (and maybe other metadata). + // message body and headers are immutable so they do not change + for (Long messageId : range) { + Result result; + try { + result = store.retrieveFlags(uuidToBytes(mailbox.getMailboxId()), messageId); + store.updateFlags(result.getRow(), messageId, parseFlagsContent(flags)); + } catch (IOException e) { + throw new MailboxException("Couldn't retrieve flags", e); + } finally { + try { + store.flushToStore(); + } catch (IOException e) { + LOG.warn("Storing index in table has failed."); + } + } + } + } + + @Override + public Iterator search(MailboxSession session, Mailbox mailbox, SearchQuery searchQuery) throws MailboxException { + // return a list of search results + Set uids = Sets.newLinkedHashSet(); + ArrayListMultimap queries = ArrayListMultimap.create(); + for (SearchQuery.Criterion criterion : searchQuery.getCriterias()) { + queries.putAll(createQuery(criterion, mailbox, searchQuery.getRecentMessageUids())); + } + + try { + return store.retrieveMails(uuidToBytes(mailbox.getMailboxId()), queries); + } catch (Throwable throwable) { + throw new MailboxException("Exception thrown while searching through the index", (Exception) throwable); + } + } + + /** + * Return a query which is built based on the given {@link org.apache.james.mailbox.model.SearchQuery.Criterion} + */ + private Multimap createQuery(SearchQuery.Criterion criterion, Mailbox mailbox, Set recentUids) throws MailboxException { + if (criterion instanceof SearchQuery.InternalDateCriterion) + try { + return createInternalDateQuery((SearchQuery.InternalDateCriterion) criterion); + } catch (ParseException e) { + throw new MailboxException("Date not in valid format: ",e); + } + else if (criterion instanceof SearchQuery.TextCriterion) + return createTextQuery((SearchQuery.TextCriterion) criterion); + else if (criterion instanceof SearchQuery.FlagCriterion) { + SearchQuery.FlagCriterion crit = (SearchQuery.FlagCriterion) criterion; + return createFlagQuery(toString(crit.getFlag()), crit.getOperator().isSet(), mailbox, recentUids); + } else if (criterion instanceof SearchQuery.CustomFlagCriterion) { + SearchQuery.CustomFlagCriterion crit = (SearchQuery.CustomFlagCriterion) criterion; + return createFlagQuery(crit.getFlag(), crit.getOperator().isSet(), mailbox, recentUids); + } else if (criterion instanceof SearchQuery.HeaderCriterion) + return createHeaderQuery((SearchQuery.HeaderCriterion) criterion); + else if (criterion instanceof SearchQuery.AllCriterion) //searches on all mail uids on that mailbox + return ArrayListMultimap.create(); + + throw new UnsupportedSearchException(); + } + + private Multimap createInternalDateQuery(SearchQuery.InternalDateCriterion crit) throws UnsupportedSearchException, ParseException { + final Multimap dateQuery = ArrayListMultimap.create(); + SearchQuery.DateOperator dop = crit.getOperator(); + DateTools.Resolution resolution = toResolution(dop.getDateResultion()); + String time = resolution.name() +"|" + DateTools.stringToTime(DateTools.dateToString(dop.getDate(), resolution)); + switch(dop.getType()) { + case ON: + dateQuery.put(SENT_DATE_FIELD,"0"+time); + break; + case BEFORE: + dateQuery.put(SENT_DATE_FIELD,"1"+time); + break; + case AFTER: + dateQuery.put(SENT_DATE_FIELD,"2"+time); + break; + default: + throw new UnsupportedSearchException(); + } + return dateQuery; + } + + private DateTools.Resolution toResolution(SearchQuery.DateResolution res) { + switch (res) { + case Year: + return DateTools.Resolution.YEAR; + case Month: + return DateTools.Resolution.MONTH; + case Day: + return DateTools.Resolution.DAY; + case Hour: + return DateTools.Resolution.HOUR; + case Minute: + return DateTools.Resolution.MINUTE; + case Second: + return DateTools.Resolution.SECOND; + default: + return DateTools.Resolution.MILLISECOND; + } + } + + private Multimap createFlagQuery(String flag, boolean isSet, Mailbox mailbox, Set recentUids) { + final Multimap flagsQuery = ArrayListMultimap.create(); + flagsQuery.put(FLAGS_FIELD, isSet ? flag : EMPTY_COLUMN_VALUE.toString()); + return flagsQuery; + } + + private Multimap createTextQuery(SearchQuery.TextCriterion crit) { + String value = crit.getOperator().getValue().toUpperCase(Locale.ENGLISH); + Multimap textQuery = ArrayListMultimap.create(); + switch (crit.getType()) { + case BODY: + tokenize(BODY_FIELD, value, textQuery); + break; + case FULL: + tokenize(BODY_FIELD, value, textQuery); + tokenize(HEADERS_FIELD, value, textQuery); + break; + } + return textQuery; + } + + private Multimap createHeaderQuery(SearchQuery.HeaderCriterion crit) throws UnsupportedSearchException { + SearchQuery.HeaderOperator op = crit.getOperator(); + MessageFields field = getHeaderField(crit.getHeaderName()); + Multimap headerQuery = ArrayListMultimap.create(); + if (op instanceof SearchQuery.ContainsOperator) { + String containedInHeader = ((SearchQuery.ContainsOperator) op).getValue().toUpperCase(Locale.ENGLISH); + headerQuery.put(field, containedInHeader); + } else if (op instanceof SearchQuery.ExistsOperator) + headerQuery.put(field, ""); + else if (op instanceof SearchQuery.AddressOperator) { + String address = ((SearchQuery.AddressOperator) op).getAddress().toUpperCase(Locale.ENGLISH); + tokenize(field, address, headerQuery); + } else // Operator not supported + throw new UnsupportedSearchException(); + return headerQuery; + } + + private static void tokenize(MessageFields field, String value, Multimap map) { + tokenize(field, new StringReader(value), map); + } + + private static void tokenize(MessageFields field, Reader reader, Multimap map) { + UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(Version.LUCENE_40, reader); + tokenizer.addAttribute(CharTermAttribute.class); + try { + while (tokenizer.incrementToken()) + map.put(field, tokenizer.getAttribute(CharTermAttribute.class).toString().toUpperCase(Locale.ENGLISH)); + } catch (IOException ioe) { + LOG.warn("Problem tokenizing " + field.name(), ioe); + } finally { + IOUtils.closeQuietly(tokenizer); + } + } + + private MessageFields getHeaderField(String headerName) { + if ("To".equalsIgnoreCase(headerName)) + return TO_FIELD; + else if ("From".equalsIgnoreCase(headerName)) + return FROM_FIELD; + else if ("Cc".equalsIgnoreCase(headerName)) + return CC_FIELD; + else if ("Bcc".equalsIgnoreCase(headerName)) + return BCC_FIELD; + else if ("Subject".equalsIgnoreCase(headerName)) + return BASE_SUBJECT_FIELD; + return PREFIX_HEADER_FIELD; + } + + private ArrayListMultimap parseFullContent(final Message message) throws MailboxException { + final ArrayListMultimap map = ArrayListMultimap.create(); + + // content handler which will mailbox the headers and the body of the message + SimpleContentHandler handler = new SimpleContentHandler() { + public void headers(Header header) { + + String firstFromMailbox = ""; + String firstToMailbox = ""; + String firstCcMailbox = ""; + String firstFromDisplay = ""; + String firstToDisplay = ""; + + for (org.apache.james.mime4j.stream.Field f : header) { + String headerName = f.getName().toUpperCase(Locale.ENGLISH); + String headerValue = f.getBody().toUpperCase(Locale.ENGLISH); + String fullValue = f.toString().toUpperCase(Locale.ENGLISH); + tokenize(HEADERS_FIELD, fullValue, map); + tokenize(PREFIX_HEADER_FIELD, headerValue, map); + + MessageFields field = getHeaderField(headerName); + + // Check if we can mailbox the the address in the right manner + if (field != null) { + // not sure if we really should reparse it. It maybe be better to check just for the right type. + // But this impl was easier in the first place + AddressList aList = LenientAddressBuilder.DEFAULT.parseAddressList(MimeUtil.unfold(f.getBody())); + for (int i = 0; i < aList.size(); i++) { + Address address = aList.get(i); + if (address instanceof org.apache.james.mime4j.dom.address.Mailbox) { + org.apache.james.mime4j.dom.address.Mailbox mailbox = (org.apache.james.mime4j.dom.address.Mailbox) address; + String value = AddressFormatter.DEFAULT.encode(mailbox).toUpperCase(Locale.ENGLISH); + tokenize(field, value, map); + if (i == 0) { + String mailboxAddress = SearchUtil.getMailboxAddress(mailbox); + String mailboxDisplay = SearchUtil.getDisplayAddress(mailbox); + + switch (field) { + case TO_FIELD: + firstToMailbox = mailboxAddress; + firstToDisplay = mailboxDisplay; + break; + case FROM_FIELD: + firstFromMailbox = mailboxAddress; + firstFromDisplay = mailboxDisplay; + break; + case CC_FIELD: + firstCcMailbox = mailboxAddress; + break; + } + } + } else if (address instanceof Group) { + MailboxList mList = ((Group) address).getMailboxes(); + for (int a = 0; a < mList.size(); a++) { + org.apache.james.mime4j.dom.address.Mailbox mailbox = mList.get(a); + String value = AddressFormatter.DEFAULT.encode(mailbox).toUpperCase(Locale.ENGLISH); + map.put(field, value); + + if (i == 0 && a == 0) { + String mailboxAddress = SearchUtil.getMailboxAddress(mailbox); + String mailboxDisplay = SearchUtil.getDisplayAddress(mailbox); + + switch (field) { + case TO_FIELD: + firstToMailbox = mailboxAddress; + firstToDisplay = mailboxDisplay; + break; + case FROM_FIELD: + firstFromMailbox = mailboxAddress; + firstFromDisplay = mailboxDisplay; + break; + case CC_FIELD: + firstCcMailbox = mailboxAddress; + break; + } + } + } + } + } + + tokenize(field, headerValue, map); + + } else if (headerName.equalsIgnoreCase("Subject")) { + map.put(BASE_SUBJECT_FIELD, SearchUtil.getBaseSubject(headerValue)); + } + } + map.put(SENT_DATE_FIELD, addLongPadding(message.getInternalDate().getTime())); + map.put(FIRST_FROM_MAILBOX_NAME_FIELD, firstFromMailbox); + map.put(FIRST_TO_MAILBOX_NAME_FIELD, firstToMailbox); + map.put(FIRST_CC_MAILBOX_NAME_FIELD, firstCcMailbox); + map.put(FIRST_FROM_MAILBOX_DISPLAY_FIELD, firstFromDisplay); + map.put(FIRST_TO_MAILBOX_DISPLAY_FIELD, firstToDisplay); + + } + + @Override + public void body(BodyDescriptor desc, InputStream in) throws MimeException, IOException { + String mediaType = desc.getMediaType(); + if (MEDIA_TYPE_TEXT.equalsIgnoreCase(mediaType) || MEDIA_TYPE_MESSAGE.equalsIgnoreCase(mediaType)) { + String cset = desc.getCharset(); + if (cset == null) { + cset = DEFAULT_ENCODING; + } + Charset charset; + try { + charset = Charset.forName(cset); + } catch (Exception e) { + // Invalid charset found so fallback toe the DEFAULT_ENCODING + charset = Charset.forName(DEFAULT_ENCODING); + } + + // Read the content one line after the other and add it to the document + tokenize(BODY_FIELD, new BufferedReader(new InputStreamReader(in, charset)), map); + } + } + + }; + MimeConfig config = new MimeConfig(); + config.setMaxLineLen(-1); + //config.setStrictParsing(false); + config.setMaxContentLen(-1); + MimeStreamParser parser = new MimeStreamParser(config); + parser.setContentDecoding(true); + parser.setContentHandler(handler); + + try { + // parse the message to mailbox headers and body + parser.parse(message.getFullContent()); + } catch (MimeException e) { + // This should never happen as it was parsed before too without problems. + throw new MailboxException("Unable to mailbox content of message", e); + } catch (IOException e) { + // This should never happen as it was parsed before too without problems. + // anyway let us just skip the body and headers in the mailbox + throw new MailboxException("Unable to mailbox content of message", e); + } + + return map; + } + + /** + * adding padding because of full row comparison + * all longs have to have the same length in digits + * + * @param time + * @return + */ + public static String addLongPadding(long time){ + NumberFormat format= NumberFormat.getInstance(Locale.ENGLISH); + format.setMinimumIntegerDigits(19); + return format.format(time).replace(",",""); + } + + private String parseFlagsContent(Message message) { + return parseFlagsContent(message.createFlags()); + } + + private String parseFlagsContent(Flags flags) { + final StringBuilder sb = new StringBuilder(); + Flag[] systemFlags = flags.getSystemFlags(); + String[] userFlags = flags.getUserFlags(); + + if (systemFlags.length == 0 && userFlags.length == 0) + sb.append(EMPTY_COLUMN_VALUE.toString()); + else { + for (Flag systemFlag : systemFlags) + sb.append(toString(systemFlag)); + + for (String userFlag : userFlags) + sb.append(userFlag); + } + return sb.toString(); + } + + /** + * Convert the given {@link Flag} to a String + * + * @param flag + * @return flagString + */ + private String toString(Flag flag) { + if (ANSWERED.equals(flag)) { + return "\\ANSWERED"; + } else if (DELETED.equals(flag)) { + return "\\DELETED"; + } else if (DRAFT.equals(flag)) { + return "\\DRAFT"; + } else if (FLAGGED.equals(flag)) { + return "\\FLAGGED"; + } else if (RECENT.equals(flag)) { + return "\\RECENT"; + } else if (SEEN.equals(flag)) { + return "\\FLAG"; + } else { + return flag.toString(); + } + } +} diff --git a/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/HBaseIndexStore.java b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/HBaseIndexStore.java new file mode 100644 index 0000000..6b768e1 --- /dev/null +++ b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/HBaseIndexStore.java @@ -0,0 +1,136 @@ +package org.apache.james.mailbox.hbase.store; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Sets; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.filter.BinaryPrefixComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.james.mailbox.hbase.store.endpoint.RowFilteringProtocol; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.*; + +import static org.apache.james.mailbox.hbase.store.HBaseNames.COLUMN_FAMILY; +import static org.apache.james.mailbox.hbase.store.MessageFields.FLAGS_FIELD; + +public class HBaseIndexStore { + private static final Logger LOG = LoggerFactory.getLogger(HBaseIndexStore.class); + private static HBaseIndexStore store; + private static HTableInterface table; + + private HBaseIndexStore() { + } + + public static synchronized HBaseIndexStore getInstance(final Configuration configuration) + throws IOException { + if (store == null) { + store = new HBaseIndexStore(); + HBaseAdmin admin = new HBaseAdmin(configuration); + + HTableDescriptor htd = new HTableDescriptor(HBaseNames.INDEX_TABLE.name); + HColumnDescriptor columnDescriptor = new HColumnDescriptor(COLUMN_FAMILY.name); + htd.addFamily(columnDescriptor); + admin.createTable(htd); + table = new HTable(configuration, HBaseNames.INDEX_TABLE.name); + } + return store; + } + + public Object clone() throws CloneNotSupportedException { + throw new CloneNotSupportedException(); + } + + /** + * writes the rows as puts in HBase where the qualifier is composed of the mailID + * + * @param puts + * @throws IOException + */ + public void storeMail(List puts) throws IOException { + for (Put put : puts) { + table.put(put); + } + } + + Iterator retrieveMails(final byte[] mailboxId) throws Throwable { + Map> results = table.coprocessorExec(RowFilteringProtocol.class, mailboxId, + Bytes.add(mailboxId, new byte[]{(byte) 0xFF}), + new Batch.Call>() { + @Override + public Set call(RowFilteringProtocol instance) throws IOException { + return instance.filterByMailbox(mailboxId); + } + }); + + return extractMessageIds(results); + } + + public ResultScanner retrieveMails(byte[] mailboxId, long messageId) throws IOException { + Preconditions.checkArgument(messageId != 0l); + Scan scan = new Scan(); + scan.addColumn(COLUMN_FAMILY.name, Bytes.toBytes(messageId)); + RowFilter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, + new BinaryPrefixComparator(mailboxId)); + scan.setFilter(filter); + return table.getScanner(scan); + } + + public Iterator retrieveMails(final byte[] mailboxId, + final ArrayListMultimap queries) + throws Throwable { + if (queries.isEmpty()) + return retrieveMails(mailboxId); + + Map> results = table.coprocessorExec(RowFilteringProtocol.class, mailboxId, + Bytes.add(mailboxId, new byte[]{(byte) 0xFF}), + new Batch.Call>() { + @Override + public Set call(RowFilteringProtocol instance) throws IOException { + return instance.filterByQueries(mailboxId, queries); + } + }); + + return extractMessageIds(results); + } + + private Iterator extractMessageIds(Map> results){ + Set uids = Sets.newHashSet(); + for (Map.Entry> entry : results.entrySet()) { + uids.addAll(entry.getValue()); + } + return uids.iterator(); + } + + public void deleteMail(byte[] row, long messageId) throws IOException { + Delete delete = new Delete(row); + delete.deleteColumn(COLUMN_FAMILY.name, Bytes.toBytes(messageId)); + table.delete(delete); + } + + public void flushToStore() throws IOException { + table.flushCommits(); + } + + public Result retrieveFlags(byte[] mailboxId, long messageId) throws IOException { + Get get = new Get(Bytes.add(mailboxId, new byte[]{FLAGS_FIELD.id})); + get.addColumn(COLUMN_FAMILY.name, Bytes.toBytes(messageId)); + return table.get(get); + } + + public void updateFlags(byte[] row, long messageId, String flags) throws IOException { + Put put = new Put(row); + put.add(COLUMN_FAMILY.name, Bytes.toBytes(messageId), Bytes.toBytes(flags)); + table.put(put); + } + + +} diff --git a/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/HBaseNames.java b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/HBaseNames.java new file mode 100644 index 0000000..15377a4 --- /dev/null +++ b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/HBaseNames.java @@ -0,0 +1,35 @@ +/****************************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one or more * + * contributor license agreements. See the NOTICE file distributed with * + * this work for additional information regarding copyright ownership. * + * The ASF licenses this file to You under the Apache License, Version 2.0 * + * (the "License"); you may not use this file except in compliance with * + * the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +package org.apache.james.mailbox.hbase.store; + +import org.apache.hadoop.hbase.util.Bytes; + +public enum HBaseNames { + INDEX_TABLE("INDEX"), COLUMN_FAMILY("F"), EMPTY_COLUMN_VALUE(""); + + public final byte[] name; + + private HBaseNames(String name) { + this.name = Bytes.toBytes(name); + } + + @Override + public String toString() { + return Bytes.toString(name); + } +} \ No newline at end of file diff --git a/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/MessageFields.java b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/MessageFields.java new file mode 100644 index 0000000..fa9fd40 --- /dev/null +++ b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/MessageFields.java @@ -0,0 +1,31 @@ +package org.apache.james.mailbox.hbase.store; + +public enum MessageFields { + NOT_FOUND((byte) 0), + FLAGS_FIELD((byte) 1), + BODY_FIELD((byte) 2), + PREFIX_HEADER_FIELD((byte) 3), + HEADERS_FIELD((byte) 4), + TO_FIELD((byte) 5), + CC_FIELD((byte) 6), + FROM_FIELD((byte) 7), + BCC_FIELD((byte) 8), + BASE_SUBJECT_FIELD((byte) 9), + SENT_DATE_FIELD((byte) 10), + FIRST_FROM_MAILBOX_NAME_FIELD((byte) 11), + FIRST_TO_MAILBOX_NAME_FIELD((byte) 12), + FIRST_CC_MAILBOX_NAME_FIELD((byte) 13), + FIRST_FROM_MAILBOX_DISPLAY_FIELD((byte) 14), + FIRST_TO_MAILBOX_DISPLAY_FIELD((byte) 15); + + public final byte id; + + private MessageFields(byte id) { + this.id = id; + } + + @Override + public String toString() { + return Integer.toString(id); + } +} \ No newline at end of file diff --git a/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/endpoint/RowFilteringEndpoint.java b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/endpoint/RowFilteringEndpoint.java new file mode 100644 index 0000000..92d4b06 --- /dev/null +++ b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/endpoint/RowFilteringEndpoint.java @@ -0,0 +1,142 @@ +package org.apache.james.mailbox.hbase.store.endpoint; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Sets; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor; +import org.apache.hadoop.hbase.filter.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.james.mailbox.hbase.store.HBaseNames; +import org.apache.james.mailbox.hbase.store.MessageFields; +import org.apache.lucene.document.DateTools; + +import java.io.IOException; +import java.text.NumberFormat; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import static org.apache.james.mailbox.hbase.index.MessageSearchIndexListener.*; +import static org.apache.james.mailbox.hbase.store.HBaseNames.COLUMN_FAMILY; +import static org.apache.james.mailbox.hbase.store.MessageFields.SENT_DATE_FIELD; + +public class RowFilteringEndpoint extends BaseEndpointCoprocessor implements RowFilteringProtocol { + + private final static Date MAX_DATE; + private final static Date MIN_DATE; + + static { + Calendar cal = Calendar.getInstance(); + cal.set(9999, 11, 31); + MAX_DATE = cal.getTime(); + + cal.set(0000, 0, 1); + MIN_DATE = cal.getTime(); + } + + @Override + public Set filterByQueries(byte[] mailboxId, ArrayListMultimap queries) throws IOException { + Scan scan = new Scan(); + scan.addFamily(COLUMN_FAMILY.name); + FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ONE); + for (Map.Entry query : queries.entries()) { + String term = query.getValue().toUpperCase(Locale.ENGLISH); + byte[] field = new byte[]{query.getKey().id}; + byte[] prefix = Bytes.add(mailboxId, field); + switch (query.getKey()) { + case FLAGS_FIELD: + final FilterList flagList = new FilterList(FilterList.Operator.MUST_PASS_ALL); + RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(prefix)); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new SubstringComparator(term)); + flagList.addFilter(rowFilter); + flagList.addFilter(valueFilter); + list.addFilter(flagList); + break; + case SENT_DATE_FIELD: + int separatorIndex = term.indexOf("|"); + long time = Long.parseLong(term.substring(separatorIndex + 1)); + long max = getMaxResolution(term.substring(1, separatorIndex), time); + + FilterList timeList = new FilterList(FilterList.Operator.MUST_PASS_ALL); + long lowerBound = 0, upperBound = 0; + switch (term.charAt(0)) { + case '0'://ON + lowerBound = time; + upperBound = max; + break; + case '1'://BEFORE + lowerBound = MIN_DATE.getTime(); + upperBound = time; + break; + case '2'://AFTER + lowerBound = max; + upperBound = MAX_DATE.getTime(); + break; + } + timeList.addFilter(new RowFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.add(prefix, Bytes.toBytes(addLongPadding(lowerBound)))))); + timeList.addFilter(new RowFilter(CompareFilter.CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.add(prefix, Bytes.toBytes(addLongPadding(upperBound)))))); + list.addFilter(timeList); + break; + default: + RowFilter rowFilterPrefix = new RowFilter(CompareFilter.CompareOp.EQUAL, + new BinaryPrefixComparator(Bytes.add(prefix, Bytes.toBytes(term)))); + RowFilter rowFilterRegex = new RowFilter(CompareFilter.CompareOp.EQUAL, + new RegexStringComparator(Bytes.toString(prefix) + ".*?" + term + ".*+")); + list.addFilter(rowFilterPrefix); + list.addFilter(rowFilterRegex); + break; + } + } + scan.setFilter(list); + + return extractIds(scan); + } + + public long getMaxResolution(String name, long time) { + long diff = 1l; + final Calendar max = Calendar.getInstance(); + max.setTimeInMillis(time); + switch (DateTools.Resolution.valueOf(name)) { + case YEAR: + max.set(Calendar.YEAR, max.get(Calendar.YEAR) + 1); + return max.getTimeInMillis(); + case MONTH: + max.set(Calendar.MONTH, max.get(Calendar.MONTH) + 1); + return max.getTimeInMillis(); + case DAY: + return time + TimeUnit.DAYS.toMillis(diff); + case HOUR: + return time + TimeUnit.HOURS.toMillis(diff); + case MINUTE: + return time + TimeUnit.MINUTES.toMillis(diff); + case SECOND: + return time + TimeUnit.SECONDS.toMillis(diff); + default: + return time; + } + } + + @Override + public Set filterByMailbox(byte[] mailboxId) throws IOException { + Scan scan = new Scan(); + scan.addFamily(COLUMN_FAMILY.name); + RowFilter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, + new BinaryPrefixComparator(mailboxId)); + scan.setFilter(filter); + return extractIds(scan); + } + + private Set extractIds(Scan scan) throws IOException { + Set uids = Sets.newLinkedHashSet(); + ResultScanner scanner = getEnvironment().getTable(HBaseNames.INDEX_TABLE.name).getScanner(scan); + for (Result result : scanner) + for (byte[] qualifier : result.getFamilyMap(HBaseNames.COLUMN_FAMILY.name).keySet()) + uids.add(Bytes.toLong(qualifier)); + return uids; + } +} diff --git a/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/endpoint/RowFilteringProtocol.java b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/endpoint/RowFilteringProtocol.java new file mode 100644 index 0000000..a4750cc --- /dev/null +++ b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/endpoint/RowFilteringProtocol.java @@ -0,0 +1,15 @@ +package org.apache.james.mailbox.hbase.store.endpoint; + +import com.google.common.collect.ArrayListMultimap; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.james.mailbox.hbase.store.MessageFields; + +import java.io.IOException; +import java.util.Set; + +public interface RowFilteringProtocol extends CoprocessorProtocol{ + + public Set filterByQueries(byte[] mailboxId, ArrayListMultimap queries) throws IOException; + + public Set filterByMailbox(byte[] mailboxId) throws IOException; +} diff --git a/hbase-simple-search/src/main/resources/hbase-site.xml b/hbase-simple-search/src/main/resources/hbase-site.xml new file mode 100644 index 0000000..1857bac --- /dev/null +++ b/hbase-simple-search/src/main/resources/hbase-site.xml @@ -0,0 +1,49 @@ + + + + + + + hbase.rootdir + hdfs://localhost:9000/hbase + + + + hbase.master.port + 60000 + + + + hbase.regionserver.info.port + 6030 + + + + hbase.regionserver.info.bindAddress + 0.0.0.0 + + + diff --git a/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/index/MessageSearchIndexListenerTest.java b/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/index/MessageSearchIndexListenerTest.java new file mode 100644 index 0000000..89449a5 --- /dev/null +++ b/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/index/MessageSearchIndexListenerTest.java @@ -0,0 +1,776 @@ +package org.apache.james.mailbox.hbase.index; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.james.mailbox.hbase.store.HBaseIndexStore; +import org.apache.james.mailbox.hbase.store.MessageBuilder; +import org.apache.james.mailbox.hbase.store.MessageFields; +import org.apache.james.mailbox.hbase.store.SimpleMailboxMembership; +import org.apache.james.mailbox.model.MailboxACL; +import org.apache.james.mailbox.model.SearchQuery; +import org.apache.james.mailbox.model.SimpleMailboxACL; +import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.junit.*; + +import javax.mail.Flags; +import java.nio.charset.Charset; +import java.util.*; + +import static org.apache.james.mailbox.hbase.index.MessageSearchIndexListener.*; +import static org.apache.james.mailbox.hbase.store.HBaseNames.COLUMN_FAMILY; +import static org.junit.Assert.*; + +public class MessageSearchIndexListenerTest { + private MessageSearchIndexListener index; + private static HBaseIndexStore store; + + private SimpleMailbox mailbox = new SimpleMailbox(new UUID(0,0)); + private SimpleMailbox mailbox2 = new SimpleMailbox(new UUID(1,0)); + private SimpleMailbox mailbox3 = new SimpleMailbox(new UUID(2,0)); + + + private static final String FROM_ADDRESS = "Harry "; + + private static final String SUBJECT_PART = "Mixed"; + + private static final String CUSTARD = "CUSTARD"; + + private static final String RHUBARD = "Rhubard"; + + private static final long mailId = 10l; + + private static final String BODY = "This is a simple email\r\n " + + "It has " + RHUBARD + ".\r\n" + "It has " + CUSTARD + ".\r\n" + + "It needs naught else.\r\n"; + + private static HBaseTestingUtility HTU = new HBaseTestingUtility(); + + @BeforeClass + public static void setUpEnvironment() throws Exception { + HTU.startMiniCluster(); + store = HBaseIndexStore.getInstance(HTU.getConfiguration()); + } + + @Before + public void setUp() throws Exception { + index = new MessageSearchIndexListener(null,store); + Map headersSubject = new HashMap(); + headersSubject.put("Subject", "test (fwd)"); + headersSubject.put("From", "test99 "); + headersSubject.put("To", "test2 , test3 "); + + Map headersTest = new HashMap(); + headersTest.put("Test", "test"); + headersTest.put("From", "test1 "); + headersTest.put("To", "test3 , test4 "); + headersTest.put("Cc", "test21 , test6 "); + + Map headersTestSubject = new HashMap(); + headersTestSubject.put("Test", "test"); + headersTestSubject.put("Subject", "test2"); + headersTestSubject.put("Date", "Thu, 14 Feb 1990 12:00:00 +0000 (GMT)"); + headersTestSubject.put("From", "test12 "); + headersTestSubject.put("Cc", "test211 , test6 "); + + SimpleMailboxMembership m2 = new SimpleMailboxMembership(mailbox2.getMailboxId(), 1, 0, new Date(), 20, new Flags(Flags.Flag.ANSWERED), "My Body".getBytes(), headersSubject); + index.add(null, mailbox2, m2); + + SimpleMailboxMembership m = new SimpleMailboxMembership(mailbox.getMailboxId(), 1, 0, new Date(), 200, new Flags(Flags.Flag.ANSWERED), "My Body".getBytes(), headersSubject); + index.add(null, mailbox, m); + + Calendar cal = Calendar.getInstance(); + cal.set(1980, 2, 10); + SimpleMailboxMembership m3 = new SimpleMailboxMembership(mailbox.getMailboxId(), 2, 0, cal.getTime(), 20, new Flags(Flags.Flag.DELETED), "My Otherbody".getBytes(), headersTest); + index.add(null, mailbox, m3); + Calendar cal2 = Calendar.getInstance(); + cal2.set(8000, 2, 10); + SimpleMailboxMembership m4 = new SimpleMailboxMembership(mailbox.getMailboxId(), 3, 0, cal2.getTime(), 20, new Flags(Flags.Flag.DELETED), "My Otherbody2".getBytes(), headersTestSubject); + index.add(null, mailbox, m4); + + MessageBuilder builder = new MessageBuilder(); + builder.header("From", "test "); + builder.header("To", FROM_ADDRESS); + builder.header("Subject", "A " + SUBJECT_PART + " Multipart Mail"); + builder.header("Date", "Thu, 14 Feb 2008 12:00:00 +0000 (GMT)"); + builder.body = Charset.forName("us-ascii").encode(BODY).array(); + builder.uid = mailId; + builder.mailboxId = mailbox3.getMailboxId(); + + index.add(null, mailbox3, builder.build()); + } + + @AfterClass + public static void tearDownEnvironment() throws Exception { + HTU.shutdownMiniCluster(); + } + + @Test + public void testUUIDTransform() throws Exception{ + UUID uuid = new UUID(11,22); + assertEquals(uuid, rowToUUID(uuidToBytes(uuid))); + } + + @Test + public void testReadMailFromStore() throws Exception{ + for(Result result: store.retrieveMails(uuidToBytes(mailbox3.getMailboxId()),mailId)){ + NavigableMap family = result.getFamilyMap(COLUMN_FAMILY.name); + assertEquals(mailId, Bytes.toLong(family.firstEntry().getKey())); + byte[] row = result.getRow(); + UUID mailboxUUID = rowToUUID(row); + assertEquals(mailbox3.getMailboxId(),mailboxUUID); + } + } + + @Test + public void testBodyShouldMatchPhraseInBody() throws Exception { + SearchQuery query = new SearchQuery(); + query.andCriteria(SearchQuery.bodyContains(CUSTARD)); + Iterator result = index.search(null, mailbox3, query); + assertEquals(10L, result.next().longValue()); + assertFalse(result.hasNext()); + + query = new SearchQuery(); + query.andCriteria(SearchQuery.bodyContains(CUSTARD + CUSTARD)); + result = index.search(null, mailbox3, query); + assertFalse(result.hasNext()); + } + + @Test + public void testBodyMatchShouldBeCaseInsensitive() throws Exception { + SearchQuery query = new SearchQuery(); + query.andCriteria(SearchQuery.bodyContains(RHUBARD)); + Iterator result = index.search(null, mailbox3, query); + assertEquals(10L, result.next().longValue()); + assertFalse(result.hasNext()); + } + + @Test + public void testBodyShouldNotMatchPhraseOnlyInHeader() throws Exception { + SearchQuery query = new SearchQuery(); + query.andCriteria(SearchQuery.bodyContains(FROM_ADDRESS)); + Iterator result = index.search(null, mailbox3, query); + assertFalse(result.hasNext()); + + query = new SearchQuery(); + query.andCriteria(SearchQuery.bodyContains(SUBJECT_PART)); + result = index.search(null, mailbox3, query); + assertFalse(result.hasNext()); + } + + @Test + public void testTextShouldMatchPhraseInBody() throws Exception { + SearchQuery query = new SearchQuery(); + query.andCriteria(SearchQuery.mailContains(CUSTARD)); + Iterator result = index.search(null, mailbox3, query); + assertEquals(10L, result.next().longValue()); + assertFalse(result.hasNext()); + + query = new SearchQuery(); + query.andCriteria(SearchQuery.mailContains(CUSTARD + CUSTARD)); + result = index.search(null, mailbox3, query); + assertFalse(result.hasNext()); + } + + @Test + public void testTextMatchShouldBeCaseInsensitive() throws Exception { + SearchQuery query = new SearchQuery(); + query.andCriteria(SearchQuery.mailContains(RHUBARD)); + Iterator result = index.search(null, mailbox3, query); + assertEquals(10L, result.next().longValue()); + assertFalse(result.hasNext()); + + query.andCriteria(SearchQuery.mailContains(RHUBARD.toLowerCase(Locale.US))); + result = index.search(null, mailbox3, query); + assertEquals(10L, result.next().longValue()); + assertFalse(result.hasNext()); + } + + @Test + public void testSearchAddress() throws Exception { + + SearchQuery query = new SearchQuery(); + query.andCriteria(SearchQuery.address(SearchQuery.AddressType.To,FROM_ADDRESS)); + Iterator result = index.search(null, mailbox3, query); + assertEquals(10L, result.next().longValue()); + assertFalse(result.hasNext()); + + query = new SearchQuery(); + query.andCriteria(SearchQuery.address(SearchQuery.AddressType.To,"Harry")); + result = index.search(null, mailbox3, query); + assertEquals(10L, result.next().longValue()); + assertFalse(result.hasNext()); + + query = new SearchQuery(); + query.andCriteria(SearchQuery.address(SearchQuery.AddressType.To,"Harry@example.org")); + result = index.search(null, mailbox3, query); + assertEquals(10L, result.next().longValue()); + assertFalse(result.hasNext()); + } + + @Test + public void testSearchAddressFrom() throws Exception { + + SearchQuery query = new SearchQuery(); + query.andCriteria(SearchQuery.address(SearchQuery.AddressType.From,"ser-from@domain.or")); + Iterator result = index.search(null, mailbox3, query); + assertEquals(10L, result.next().longValue()); + assertFalse(result.hasNext()); + + + } + + @Test + public void testBodyShouldMatchPhraseOnlyInHeader() throws Exception { + + SearchQuery query = new SearchQuery(); + query.andCriteria(SearchQuery.mailContains(FROM_ADDRESS)); + Iterator result = index.search(null, mailbox3, query); + assertEquals(10L, result.next().longValue()); + assertFalse(result.hasNext()); + + query.andCriteria(SearchQuery.mailContains(SUBJECT_PART)); + result = index.search(null, mailbox3, query); + assertEquals(10L, result.next().longValue()); + assertFalse(result.hasNext()); + } + + @Test + public void testSearchAll() throws Exception { + SearchQuery query = new SearchQuery(); + query.andCriteria(SearchQuery.all()); + Iterator it2 = index.search(null, mailbox2, query); + assertTrue(it2.hasNext()); + assertEquals(1L, it2.next().longValue()); + assertFalse(it2.hasNext()); + } + + @Test + public void testSearchFlag() throws Exception { + + SearchQuery q = new SearchQuery(); + q.andCriteria(SearchQuery.flagIsSet(Flags.Flag.DELETED)); + Iterator it3 = index.search(null, mailbox, q); + assertEquals(2L, it3.next().longValue()); + assertEquals(3L, it3.next().longValue()); + assertFalse(it3.hasNext()); + } + + @Test + public void testSearchBody() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.bodyContains("body")); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Test + public void testSearchMail() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.mailContains("body")); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Test + public void testSearchHeaderContains() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.headerContains("Subject", "test")); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Test + public void testSearchHeaderExists() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.headerExists("Subject")); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Test + public void testSearchFlagUnset() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.flagIsUnSet(Flags.Flag.DRAFT)); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Test + public void testSearchInternalDateBefore() throws Exception { + SearchQuery q2 = new SearchQuery(); + Calendar cal = Calendar.getInstance(); + cal.setTime(new Date()); + q2.andCriteria(SearchQuery.internalDateBefore(cal.getTime(), SearchQuery.DateResolution.Day)); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(2L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + @Test + public void testSearchInternalDateAfter() throws Exception { + SearchQuery q2 = new SearchQuery(); + Calendar cal = Calendar.getInstance(); + cal.setTime(new Date()); + q2.andCriteria(SearchQuery.internalDateAfter(cal.getTime(), SearchQuery.DateResolution.Day)); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(3L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + @Test + public void testSearchInternalDateOn() throws Exception { + SearchQuery q2 = new SearchQuery(); + Calendar cal = Calendar.getInstance(); + cal.setTime(new Date()); + q2.andCriteria(SearchQuery.internalDateOn(cal.getTime(), SearchQuery.DateResolution.Day)); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSearchUidMatch() throws Exception { + SearchQuery q2 = new SearchQuery(); + Calendar cal = Calendar.getInstance(); + cal.setTime(new Date()); + q2.andCriteria(SearchQuery.uid(new SearchQuery.NumericRange[] {new SearchQuery.NumericRange(1)})); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSearchUidRange() throws Exception { + SearchQuery q2 = new SearchQuery(); + Calendar cal = Calendar.getInstance(); + cal.setTime(new Date()); + q2.andCriteria(SearchQuery.uid(new SearchQuery.NumericRange[] {new SearchQuery.NumericRange(1), new SearchQuery.NumericRange(2,3)})); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSearchSizeEquals() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.sizeEquals(200)); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSearchSizeLessThan() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.sizeLessThan(200)); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSearchSizeGreaterThan() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.sizeGreaterThan(6)); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortUid() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortUidReverse() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.Uid, true))); + q2.andCriteria(SearchQuery.all()); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(3L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortSentDate() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.SentDate, false))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortSentDateReverse() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.SentDate, true))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortBaseSubject() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.BaseSubject, false))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(2L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortBaseSubjectReverse() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.BaseSubject, true))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(3L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortMailboxFrom() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.MailboxFrom, false))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortMailboxFromReverse() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.MailboxFrom, true))); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortMailboxCc() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.MailboxCc, false))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortMailboxCcReverse() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.MailboxCc, true))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertEquals(1L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortMailboxTo() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.MailboxTo, false))); + + Iterator it4 = index.search(null, mailbox, q2); + + assertEquals(3L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortMailboxToReverse() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.MailboxTo, true))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(2L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortDisplayTo() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.DisplayTo, false))); + + Iterator it4 = index.search(null, mailbox, q2); + + assertEquals(3L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortDisplayToReverse() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.DisplayTo, true))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(2L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortDisplayFrom() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.DisplayFrom, false))); + + Iterator it4 = index.search(null, mailbox, q2); + + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortDisplayFromReverse() throws Exception { + + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.DisplayFrom, true))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortArrival() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.Arrival, false))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(2L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortArrivalReverse() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.Arrival, true))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(3L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortSize() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.Size, false))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + assertEquals(1L, it4.next().longValue()); + + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testSortSizeReverse() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.all()); + q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.Size, true))); + + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testNot() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.not(SearchQuery.uid(new SearchQuery.NumericRange[] { new SearchQuery.NumericRange(1)}))); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(2L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + private final class SimpleMailbox implements Mailbox { + private UUID id; + + public SimpleMailbox(UUID id) { + this.id = id; + } + + public UUID getMailboxId() { + return id; + } + + public String getNamespace() { + throw new UnsupportedOperationException("Not supported"); + } + + public void setNamespace(String namespace) { + throw new UnsupportedOperationException("Not supported"); + } + + public String getUser() { + throw new UnsupportedOperationException("Not supported"); + } + + public void setUser(String user) { + throw new UnsupportedOperationException("Not supported"); + } + + public String getName() { + return id.toString(); + } + + public void setName(String name) { + throw new UnsupportedOperationException("Not supported"); + + } + + public long getUidValidity() { + return 0; + } + + /* (non-Javadoc) + * @see org.apache.james.mailbox.store.mail.model.Mailbox#getACL() + */ + @Override + public MailboxACL getACL() { + return SimpleMailboxACL.OWNER_FULL_ACL; + } + + /* (non-Javadoc) + * @see org.apache.james.mailbox.store.mail.model.Mailbox#setACL(org.apache.james.mailbox.MailboxACL) + */ + @Override + public void setACL(MailboxACL acl) { + throw new UnsupportedOperationException("Not supported"); + } + } +} diff --git a/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/store/MessageBuilder.java b/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/store/MessageBuilder.java new file mode 100644 index 0000000..0cab78c --- /dev/null +++ b/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/store/MessageBuilder.java @@ -0,0 +1,62 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.hbase.store; + +import org.apache.james.mailbox.store.mail.model.Message; + +import javax.mail.Flags; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +public class MessageBuilder { + + public UUID mailboxId = new UUID(11,3); + public long uid = 776; + public Date internalDate = new Date(); + public int size = 8867; + public Flags flags = new Flags(); + public byte[] body = {}; + public final Map headers = new HashMap(); + public int lineNumber = 0; + + public Message build() throws Exception { + return new SimpleMailboxMembership(mailboxId, uid, -1, internalDate, size, flags, body, headers); + } + + public void header(String field, String value) { + headers.put(field, value); + } + + public void setKey(UUID mailboxId, int uid) { + this.uid = uid; + this.mailboxId = mailboxId; + } + + public void setFlags(boolean seen, boolean flagged, boolean answered, + boolean draft, boolean deleted, boolean recent) { + if (seen) flags.add(Flags.Flag.SEEN); + if (flagged) flags.add(Flags.Flag.FLAGGED); + if (answered) flags.add(Flags.Flag.ANSWERED); + if (draft) flags.add(Flags.Flag.DRAFT); + if (deleted) flags.add(Flags.Flag.DELETED); + if (recent) flags.add(Flags.Flag.RECENT); + } +} diff --git a/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/store/SimpleMailboxMembership.java b/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/store/SimpleMailboxMembership.java new file mode 100644 index 0000000..6f638c3 --- /dev/null +++ b/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/store/SimpleMailboxMembership.java @@ -0,0 +1,295 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.hbase.store; + +import org.apache.james.mailbox.store.mail.model.Message; +import org.apache.james.mailbox.store.mail.model.Property; +import org.apache.james.mailbox.store.mail.model.impl.SimpleProperty; + +import javax.mail.Flags; +import java.io.*; +import java.util.*; +import java.util.Map.Entry; + +public class SimpleMailboxMembership implements Message { + + private static final String TOSTRING_SEPARATOR = " "; + + public UUID mailboxId; + public long uid; + public Date internalDate; + public boolean recent = false; + public boolean answered = false; + public boolean deleted = false; + public boolean draft = false; + public boolean flagged = false; + public boolean seen = false; + + public SimpleMailboxMembership(UUID mailboxId, long uid, long modSeq, Date internalDate, int size, + Flags flags, byte[] body, final Map headers) { + super(); + this.mailboxId = mailboxId; + this.uid = uid; + this.internalDate = internalDate; + this.size = size; + this.body = body; + this.headers = (headers == null) ? new HashMap() : headers; + this.body = body; + setFlags(flags); + } + + /** + * @see org.apache.james.imap.Message.mail.model.Document#getInternalDate() + */ + public Date getInternalDate() { + return internalDate; + } + + /** + * @see org.apache.james.imap.Message.mail.model.Document#getMailboxId() + */ + public UUID getMailboxId() { + return mailboxId; + } + + /** + * @see org.apache.james.imap.Message.mail.model.Document#getUid() + */ + public long getUid() { + return uid; + } + + /** + * @see org.apache.james.imap.Message.mail.model.Document#isAnswered() + */ + public boolean isAnswered() { + return answered; + } + + /** + * @see org.apache.james.imap.Message.mail.model.Document#isDeleted() + */ + public boolean isDeleted() { + return deleted; + } + + /** + * @see org.apache.james.imap.Message.mail.model.Document#isDraft() + */ + public boolean isDraft() { + return draft; + } + + /** + * @see org.apache.james.imap.Message.mail.model.Document#isFlagged() + */ + public boolean isFlagged() { + return flagged; + } + + /** + * @see org.apache.james.imap.Message.mail.model.Document#isRecent() + */ + public boolean isRecent() { + return recent; + } + + /** + * @see org.apache.james.imap.Message.mail.model.Document#isSeen() + */ + public boolean isSeen() { + return seen; + } + + /** + * @see org.apache.james.imap.Message.mail.model.Document#unsetRecent() + */ + public void unsetRecent() { + recent = false; + } + + /** + * @see org.apache.james.imap.Message.mail.model.Document#setFlags(javax.mail.Flags) + */ + public void setFlags(Flags flags) { + answered = flags.contains(Flags.Flag.ANSWERED); + deleted = flags.contains(Flags.Flag.DELETED); + draft = flags.contains(Flags.Flag.DRAFT); + flagged = flags.contains(Flags.Flag.FLAGGED); + recent = flags.contains(Flags.Flag.RECENT); + seen = flags.contains(Flags.Flag.SEEN); + } + + /** + * @see org.apache.james.imap.Message.mail.model.Document#createFlags() + */ + public Flags createFlags() { + final Flags flags = new Flags(); + + if (isAnswered()) { + flags.add(Flags.Flag.ANSWERED); + } + if (isDeleted()) { + flags.add(Flags.Flag.DELETED); + } + if (isDraft()) { + flags.add(Flags.Flag.DRAFT); + } + if (isFlagged()) { + flags.add(Flags.Flag.FLAGGED); + } + if (isRecent()) { + flags.add(Flags.Flag.RECENT); + } + if (isSeen()) { + flags.add(Flags.Flag.SEEN); + } + return flags; + } + + @Override + public int hashCode() { + int result = mailboxId != null ? mailboxId.hashCode() : 0; + result = 31 * result + (int) (uid ^ (uid >>> 32)); + return result; + } + + @SuppressWarnings("unchecked") + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final Message other = (Message) obj; + return mailboxId == other.getMailboxId() && uid == other.getUid(); + } + + public String toString() { + return + "mailbox(" + + "mailboxId = " + this.mailboxId + TOSTRING_SEPARATOR + + "uid = " + this.uid + TOSTRING_SEPARATOR + + "internalDate = " + this.internalDate + TOSTRING_SEPARATOR + + "size = " + this.size + TOSTRING_SEPARATOR + + "answered = " + this.answered + TOSTRING_SEPARATOR + + "deleted = " + this.deleted + TOSTRING_SEPARATOR + + "draft = " + this.draft + TOSTRING_SEPARATOR + + "flagged = " + this.flagged + TOSTRING_SEPARATOR + + "recent = " + this.recent + TOSTRING_SEPARATOR + + "seen = " + this.seen + TOSTRING_SEPARATOR + + " )"; + } + + + public static final char[] NEW_LINE = {0x0D, 0x0A}; + + public byte[] body; + public Map headers; + public List properties; + public String subType = null; + public String mediaType = null; + public Long textualLineCount = null; + + private int size; + + private long modSeq; + + + /** + * @throws java.io.IOException + * @see org.apache.james.imap.Message.mail.model.Document#getBodyContent() + */ + public InputStream getBodyContent() throws IOException { + return new ByteArrayInputStream(body); + } + + /** + * @see org.apache.james.mailbox.store.mail.model.Message#getHeaderContent() + */ + public InputStream getHeaderContent() throws IOException { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final Writer writer = new OutputStreamWriter(baos, "us-ascii"); + + Iterator> hIt = headers.entrySet().iterator(); + while (hIt.hasNext()) { + Entry header = hIt.next(); + writer.write(header.getKey()); + writer.write(": "); + writer.write(header.getValue()); + writer.write(NEW_LINE); + } + writer.write(NEW_LINE); + writer.flush(); + return new ByteArrayInputStream(baos.toByteArray()); + + } + + public long getBodyOctets() { + return body.length; + } + + public String getSubType() { + return subType; + } + + public String getMediaType() { + return mediaType; + } + + public List getProperties() { + return new ArrayList(properties); + } + + public Long getTextualLineCount() { + return textualLineCount; + } + + public long getFullContentOctets() { + return size; + } + + /** + * @see Comparable#compareTo(Object) + */ + public int compareTo(Message other) { + return (int) (getUid() - other.getUid()); + } + + public long getModSeq() { + return modSeq; + } + + public void setModSeq(long modSeq) { + this.modSeq = modSeq; + } + + public void setUid(long uid) { + this.uid = uid; + } + + @Override + public InputStream getFullContent() throws IOException { + return new SequenceInputStream(getHeaderContent(), getBodyContent()); + } + + +} diff --git a/hbase-simple-search/src/test/resources/hbase-site.xml b/hbase-simple-search/src/test/resources/hbase-site.xml new file mode 100644 index 0000000..5f2499c --- /dev/null +++ b/hbase-simple-search/src/test/resources/hbase-site.xml @@ -0,0 +1,66 @@ + + + + + + + hbase.rootdir + hdfs://localhost:9000/hbase + + + + hbase.master.port + 60000 + + + + hbase.regionserver.info.port + 6030 + + + + hbase.regionserver.info.bindAddress + 0.0.0.0 + + + + + hbase.zookeeper.property.clientPort + 21818 + + + hbase.zookeeper.quorum + localhost + + + + + + hbase.coprocessor.region.classes + org.apache.james.mailbox.hbase.store.endpoint.RowFilteringEndpoint + + + diff --git a/hbase-simple-search/src/test/resources/hdfs-site.xml b/hbase-simple-search/src/test/resources/hdfs-site.xml new file mode 100644 index 0000000..ac4cccf --- /dev/null +++ b/hbase-simple-search/src/test/resources/hdfs-site.xml @@ -0,0 +1,36 @@ + + + + + + + dfs.permissions + true + + If "true", enable permission checking in HDFS. + If "false", permission checking is turned off, + but all other behavior is unchanged. + Switching from one parameter value to the other does not change the mode, + owner or group of files or directories. + + + + + dfs.datanode.data.dir.perm + 755 + Permissions for the directories on on the local filesystem where + the DFS data node store its blocks. The permissions can either be octal or symbolic. + + + + hbase.zookeeper.property.clientPort + 2181 + + + dfs.support.append + true + This branch of HDFS supports reliable append/sync. + + + + diff --git a/pom.xml b/pom.xml index 7222f3e..cbaad73 100644 --- a/pom.xml +++ b/pom.xml @@ -45,6 +45,7 @@ jcr maildir hbase + hbase-simple-search spring tool zoo-seq-provider -- 1.8.0.2 From 872ca28ade9a02b251adc591f0b2ae85b97a6b09 Mon Sep 17 00:00:00 2001 From: Mihai Soloi Date: Mon, 31 Dec 2012 08:57:51 +0100 Subject: [PATCH 2/3] added spring configuration file for hbase-simple-search, modified spring-mailbox.xml to include the new configuration --- .../james/mailbox/hbase/store/HBaseIndexStore.java | 25 ++++++++------ pom.xml | 5 +++ spring/pom.xml | 4 +++ .../james/spring-mailbox-hbase-simple-search.xml | 40 ++++++++++++++++++++++ .../META-INF/org/apache/james/spring-mailbox.xml | 3 +- 5 files changed, 65 insertions(+), 12 deletions(-) create mode 100644 spring/src/main/resources/META-INF/org/apache/james/spring-mailbox-hbase-simple-search.xml diff --git a/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/HBaseIndexStore.java b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/HBaseIndexStore.java index 6b768e1..0621fe4 100644 --- a/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/HBaseIndexStore.java +++ b/hbase-simple-search/src/main/java/org/apache/james/mailbox/hbase/store/HBaseIndexStore.java @@ -24,23 +24,26 @@ import static org.apache.james.mailbox.hbase.store.MessageFields.FLAGS_FIELD; public class HBaseIndexStore { private static final Logger LOG = LoggerFactory.getLogger(HBaseIndexStore.class); - private static HBaseIndexStore store; + private volatile static HBaseIndexStore store; private static HTableInterface table; private HBaseIndexStore() { } - public static synchronized HBaseIndexStore getInstance(final Configuration configuration) + public static HBaseIndexStore getInstance(final Configuration configuration) throws IOException { - if (store == null) { - store = new HBaseIndexStore(); - HBaseAdmin admin = new HBaseAdmin(configuration); - - HTableDescriptor htd = new HTableDescriptor(HBaseNames.INDEX_TABLE.name); - HColumnDescriptor columnDescriptor = new HColumnDescriptor(COLUMN_FAMILY.name); - htd.addFamily(columnDescriptor); - admin.createTable(htd); - table = new HTable(configuration, HBaseNames.INDEX_TABLE.name); + if (store == null) + synchronized (HBaseIndexStore.class){ + if (store == null) { + store = new HBaseIndexStore(); + HBaseAdmin admin = new HBaseAdmin(configuration); + + HTableDescriptor htd = new HTableDescriptor(HBaseNames.INDEX_TABLE.name); + HColumnDescriptor columnDescriptor = new HColumnDescriptor(COLUMN_FAMILY.name); + htd.addFamily(columnDescriptor); + admin.createTable(htd); + table = new HTable(configuration, HBaseNames.INDEX_TABLE.name); + } } return store; } diff --git a/pom.xml b/pom.xml index cbaad73..d6d358b 100644 --- a/pom.xml +++ b/pom.xml @@ -157,6 +157,11 @@ org.apache.james + apache-james-mailbox-hbase-simple-search + ${project.version} + + + org.apache.james apache-james-mailbox-tool ${project.version} diff --git a/spring/pom.xml b/spring/pom.xml index b1fd24b..5fdf356 100644 --- a/spring/pom.xml +++ b/spring/pom.xml @@ -60,6 +60,10 @@ org.apache.james apache-james-mailbox-hbase + + org.apache.james + apache-james-mailbox-hbase-simple-search + + + + + + + + + + + + + + + + diff --git a/spring/src/main/resources/META-INF/org/apache/james/spring-mailbox.xml b/spring/src/main/resources/META-INF/org/apache/james/spring-mailbox.xml index 02d165e..a90a13a 100644 --- a/spring/src/main/resources/META-INF/org/apache/james/spring-mailbox.xml +++ b/spring/src/main/resources/META-INF/org/apache/james/spring-mailbox.xml @@ -36,13 +36,14 @@ + - + hbase.rootdir hdfs://localhost:9000/hbase - + hbase.master.port 60000 - + hbase.regionserver.info.port 6030 diff --git a/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/index/MessageSearchIndexListenerTest.java b/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/index/MessageSearchIndexListenerTest.java index 89449a5..af0a125 100644 --- a/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/index/MessageSearchIndexListenerTest.java +++ b/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/index/MessageSearchIndexListenerTest.java @@ -5,7 +5,6 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; import org.apache.james.mailbox.hbase.store.HBaseIndexStore; import org.apache.james.mailbox.hbase.store.MessageBuilder; -import org.apache.james.mailbox.hbase.store.MessageFields; import org.apache.james.mailbox.hbase.store.SimpleMailboxMembership; import org.apache.james.mailbox.model.MailboxACL; import org.apache.james.mailbox.model.SearchQuery; @@ -17,7 +16,8 @@ import javax.mail.Flags; import java.nio.charset.Charset; import java.util.*; -import static org.apache.james.mailbox.hbase.index.MessageSearchIndexListener.*; +import static org.apache.james.mailbox.hbase.index.MessageSearchIndexListener.rowToUUID; +import static org.apache.james.mailbox.hbase.index.MessageSearchIndexListener.uuidToBytes; import static org.apache.james.mailbox.hbase.store.HBaseNames.COLUMN_FAMILY; import static org.junit.Assert.*; @@ -25,9 +25,9 @@ public class MessageSearchIndexListenerTest { private MessageSearchIndexListener index; private static HBaseIndexStore store; - private SimpleMailbox mailbox = new SimpleMailbox(new UUID(0,0)); - private SimpleMailbox mailbox2 = new SimpleMailbox(new UUID(1,0)); - private SimpleMailbox mailbox3 = new SimpleMailbox(new UUID(2,0)); + private SimpleMailbox mailbox = new SimpleMailbox(new UUID(0, 0)); + private SimpleMailbox mailbox2 = new SimpleMailbox(new UUID(1, 0)); + private SimpleMailbox mailbox3 = new SimpleMailbox(new UUID(2, 0)); private static final String FROM_ADDRESS = "Harry "; @@ -54,7 +54,7 @@ public class MessageSearchIndexListenerTest { @Before public void setUp() throws Exception { - index = new MessageSearchIndexListener(null,store); + index = new MessageSearchIndexListener(null, store); Map headersSubject = new HashMap(); headersSubject.put("Subject", "test (fwd)"); headersSubject.put("From", "test99 "); @@ -106,19 +106,19 @@ public class MessageSearchIndexListenerTest { } @Test - public void testUUIDTransform() throws Exception{ - UUID uuid = new UUID(11,22); + public void testUUIDTransform() throws Exception { + UUID uuid = new UUID(11, 22); assertEquals(uuid, rowToUUID(uuidToBytes(uuid))); } @Test - public void testReadMailFromStore() throws Exception{ - for(Result result: store.retrieveMails(uuidToBytes(mailbox3.getMailboxId()),mailId)){ - NavigableMap family = result.getFamilyMap(COLUMN_FAMILY.name); + public void testReadMailFromStore() throws Exception { + for (Result result : store.retrieveMails(uuidToBytes(mailbox3.getMailboxId()), mailId)) { + NavigableMap family = result.getFamilyMap(COLUMN_FAMILY.name); assertEquals(mailId, Bytes.toLong(family.firstEntry().getKey())); byte[] row = result.getRow(); UUID mailboxUUID = rowToUUID(row); - assertEquals(mailbox3.getMailboxId(),mailboxUUID); + assertEquals(mailbox3.getMailboxId(), mailboxUUID); } } @@ -190,19 +190,19 @@ public class MessageSearchIndexListenerTest { public void testSearchAddress() throws Exception { SearchQuery query = new SearchQuery(); - query.andCriteria(SearchQuery.address(SearchQuery.AddressType.To,FROM_ADDRESS)); + query.andCriteria(SearchQuery.address(SearchQuery.AddressType.To, FROM_ADDRESS)); Iterator result = index.search(null, mailbox3, query); assertEquals(10L, result.next().longValue()); assertFalse(result.hasNext()); query = new SearchQuery(); - query.andCriteria(SearchQuery.address(SearchQuery.AddressType.To,"Harry")); + query.andCriteria(SearchQuery.address(SearchQuery.AddressType.To, "Harry")); result = index.search(null, mailbox3, query); assertEquals(10L, result.next().longValue()); assertFalse(result.hasNext()); query = new SearchQuery(); - query.andCriteria(SearchQuery.address(SearchQuery.AddressType.To,"Harry@example.org")); + query.andCriteria(SearchQuery.address(SearchQuery.AddressType.To, "Harry@example.org")); result = index.search(null, mailbox3, query); assertEquals(10L, result.next().longValue()); assertFalse(result.hasNext()); @@ -212,7 +212,7 @@ public class MessageSearchIndexListenerTest { public void testSearchAddressFrom() throws Exception { SearchQuery query = new SearchQuery(); - query.andCriteria(SearchQuery.address(SearchQuery.AddressType.From,"ser-from@domain.or")); + query.andCriteria(SearchQuery.address(SearchQuery.AddressType.From, "ser-from@domain.or")); Iterator result = index.search(null, mailbox3, query); assertEquals(10L, result.next().longValue()); assertFalse(result.hasNext()); @@ -348,25 +348,23 @@ public class MessageSearchIndexListenerTest { assertFalse(it4.hasNext()); } - @Ignore("unsupported operation") @Test public void testSearchUidMatch() throws Exception { SearchQuery q2 = new SearchQuery(); Calendar cal = Calendar.getInstance(); cal.setTime(new Date()); - q2.andCriteria(SearchQuery.uid(new SearchQuery.NumericRange[] {new SearchQuery.NumericRange(1)})); + q2.andCriteria(SearchQuery.uid(new SearchQuery.NumericRange[]{new SearchQuery.NumericRange(1)})); Iterator it4 = index.search(null, mailbox, q2); assertEquals(1L, it4.next().longValue()); assertFalse(it4.hasNext()); } - @Ignore("unsupported operation") @Test public void testSearchUidRange() throws Exception { SearchQuery q2 = new SearchQuery(); Calendar cal = Calendar.getInstance(); cal.setTime(new Date()); - q2.andCriteria(SearchQuery.uid(new SearchQuery.NumericRange[] {new SearchQuery.NumericRange(1), new SearchQuery.NumericRange(2,3)})); + q2.andCriteria(SearchQuery.uid(new SearchQuery.NumericRange[]{new SearchQuery.NumericRange(1), new SearchQuery.NumericRange(2, 3)})); Iterator it4 = index.search(null, mailbox, q2); assertEquals(1L, it4.next().longValue()); assertEquals(2L, it4.next().longValue()); @@ -511,7 +509,7 @@ public class MessageSearchIndexListenerTest { @Ignore("unsupported operation") @Test - public void testSortMailboxFromReverse() throws Exception { + public void testSortMailboxFromReverse() throws Exception { SearchQuery q2 = new SearchQuery(); q2.andCriteria(SearchQuery.all()); q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.MailboxFrom, true))); @@ -540,7 +538,7 @@ public class MessageSearchIndexListenerTest { @Ignore("unsupported operation") @Test - public void testSortMailboxCcReverse() throws Exception { + public void testSortMailboxCcReverse() throws Exception { SearchQuery q2 = new SearchQuery(); q2.andCriteria(SearchQuery.all()); q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.MailboxCc, true))); @@ -572,7 +570,7 @@ public class MessageSearchIndexListenerTest { @Ignore("unsupported operation") @Test - public void testSortMailboxToReverse() throws Exception { + public void testSortMailboxToReverse() throws Exception { SearchQuery q2 = new SearchQuery(); q2.andCriteria(SearchQuery.all()); q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.MailboxTo, true))); @@ -603,7 +601,7 @@ public class MessageSearchIndexListenerTest { @Ignore("unsupported operation") @Test - public void testSortDisplayToReverse() throws Exception { + public void testSortDisplayToReverse() throws Exception { SearchQuery q2 = new SearchQuery(); q2.andCriteria(SearchQuery.all()); q2.setSorts(Arrays.asList(new SearchQuery.Sort(SearchQuery.Sort.SortClause.DisplayTo, true))); @@ -634,7 +632,7 @@ public class MessageSearchIndexListenerTest { @Ignore("unsupported operation") @Test - public void testSortDisplayFromReverse() throws Exception { + public void testSortDisplayFromReverse() throws Exception { SearchQuery q2 = new SearchQuery(); q2.andCriteria(SearchQuery.all()); @@ -710,7 +708,30 @@ public class MessageSearchIndexListenerTest { @Test public void testNot() throws Exception { SearchQuery q2 = new SearchQuery(); - q2.andCriteria(SearchQuery.not(SearchQuery.uid(new SearchQuery.NumericRange[] { new SearchQuery.NumericRange(1)}))); + q2.andCriteria(SearchQuery.not(SearchQuery.uid(new SearchQuery.NumericRange[]{new SearchQuery.NumericRange(2)}))); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(1L, it4.next().longValue()); + assertEquals(3L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testAnd() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.and(SearchQuery.uid(new SearchQuery.NumericRange[]{new SearchQuery.NumericRange(1, 2)}), + SearchQuery.uid(new SearchQuery.NumericRange[]{new SearchQuery.NumericRange(2, 3)}))); + Iterator it4 = index.search(null, mailbox, q2); + assertEquals(2L, it4.next().longValue()); + assertFalse(it4.hasNext()); + } + + @Ignore("unsupported operation") + @Test + public void testOr() throws Exception { + SearchQuery q2 = new SearchQuery(); + q2.andCriteria(SearchQuery.or(SearchQuery.uid(new SearchQuery.NumericRange[]{new SearchQuery.NumericRange(2)}), + SearchQuery.uid(new SearchQuery.NumericRange[]{new SearchQuery.NumericRange(3)}))); Iterator it4 = index.search(null, mailbox, q2); assertEquals(2L, it4.next().longValue()); assertEquals(3L, it4.next().longValue()); diff --git a/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/store/MessageBuilder.java b/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/store/MessageBuilder.java index 0cab78c..1679ed5 100644 --- a/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/store/MessageBuilder.java +++ b/hbase-simple-search/src/test/java/org/apache/james/mailbox/hbase/store/MessageBuilder.java @@ -27,8 +27,8 @@ import java.util.Map; import java.util.UUID; public class MessageBuilder { - - public UUID mailboxId = new UUID(11,3); + + public UUID mailboxId = new UUID(11, 3); public long uid = 776; public Date internalDate = new Date(); public int size = 8867; @@ -36,11 +36,11 @@ public class MessageBuilder { public byte[] body = {}; public final Map headers = new HashMap(); public int lineNumber = 0; - + public Message build() throws Exception { - return new SimpleMailboxMembership(mailboxId, uid, -1, internalDate, size, flags, body, headers); + return new SimpleMailboxMembership(mailboxId, uid, -1, internalDate, size, flags, body, headers); } - + public void header(String field, String value) { headers.put(field, value); } @@ -49,9 +49,9 @@ public class MessageBuilder { this.uid = uid; this.mailboxId = mailboxId; } - + public void setFlags(boolean seen, boolean flagged, boolean answered, - boolean draft, boolean deleted, boolean recent) { + boolean draft, boolean deleted, boolean recent) { if (seen) flags.add(Flags.Flag.SEEN); if (flagged) flags.add(Flags.Flag.FLAGGED); if (answered) flags.add(Flags.Flag.ANSWERED); diff --git a/hbase-simple-search/src/test/resources/hbase-site.xml b/hbase-simple-search/src/test/resources/hbase-site.xml index 5f2499c..5039529 100644 --- a/hbase-simple-search/src/test/resources/hbase-site.xml +++ b/hbase-simple-search/src/test/resources/hbase-site.xml @@ -48,12 +48,12 @@ - hbase.zookeeper.property.clientPort - 21818 + hbase.zookeeper.property.clientPort + 21818 - hbase.zookeeper.quorum - localhost + hbase.zookeeper.quorum + localhost -- 1.8.0.2