diff -uNr james-mailbox/README.md james-mailbox-elastic-search/README.md --- james-mailbox/README.md 2015-04-13 19:38:21.077673284 +0200 +++ james-mailbox-elastic-search/README.md 2015-04-13 19:50:12.234370713 +0200 @@ -13,6 +13,8 @@ ~~~ |-- api -- Mailbox API +|-- cassandra -- Mailbox implementation over Cassandra +|-- elasticsearch -- Email indexing module with ElasticSearch |-- hbase -- Mailbox implementation over HBase |-- jcr -- Mailbox implementation over Java Content Repository (JCR) |-- jpa -- Database Mailbox implementation using Java Persistence API @@ -61,6 +63,15 @@ * distributed SMTP/IMAP access * other +Mailbox Cassandra +================= + +Uses Apache Cassandra (http://cassandra.apache.org/) for storing email messages. Provides a scalable email storage. To have a fully +distributed email server you will also need, among others: + +* distributed SMTP/IMAP access +* other + Zookeeper Sequence Provider ========================== diff -uNr james-mailbox/api/src/main/java/org/apache/james/mailbox/exception/UnavailableIndexException.java james-mailbox-elastic-search/api/src/main/java/org/apache/james/mailbox/exception/UnavailableIndexException.java --- james-mailbox/api/src/main/java/org/apache/james/mailbox/exception/UnavailableIndexException.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/api/src/main/java/org/apache/james/mailbox/exception/UnavailableIndexException.java 2015-04-13 20:18:44.924444798 +0200 @@ -0,0 +1,44 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.mailbox.exception; + +/** + * This exception is thrown when the index can perform this type of search or operation + * but is unable to perform it because of other errors. + * + * Example given : when using ElasticSearch, a network problem might disallow you to query + * ElasticSearch, throwing this error. + */ +public class UnavailableIndexException extends MailboxException { + + private static final long serialVersionUID = 545863697215411L; + + public UnavailableIndexException() { + super(); + } + + public UnavailableIndexException(String message) { + super(message); + } + + public UnavailableIndexException(String message, Exception cause) { + super(message, cause); + } +} diff -uNr james-mailbox/elasticsearch/pom.xml james-mailbox-elastic-search/elasticsearch/pom.xml --- james-mailbox/elasticsearch/pom.xml 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/pom.xml 2015-04-13 20:17:16.401107636 +0200 @@ -0,0 +1,155 @@ + + + + apache-james-mailbox + org.apache.james + 0.6-SNAPSHOT + + 4.0.0 + + apache-james-mailbox-elasticsearch + + + 2.7.0 + + + + + sonatype + Sonatype Groups + https://oss.sonatype.org/content/groups/public/ + + + + + + ${javax.mail.groupId} + ${javax.mail.artifactId} + + + junit + junit + test + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-simple + test + + + ${project.groupId} + apache-james-mailbox-api + + + ${project.groupId} + apache-james-mailbox-store + + + ${project.groupId} + apache-james-mailbox-api + test + test-jar + + + ${project.groupId} + apache-james-mailbox-memory + test + + + io.searchbox + jest + 0.1.3 + + + org.elasticsearch + elasticsearch + 1.4.0 + + + org.apache.curator + curator-test + 2.7.0 + + + org.slf4j + log4j-over-slf4j + 1.7.7 + + + org.apache.kafka + kafka_2.10 + 0.8.1 + + + org.apache.curator + curator-framework + 2.7.0 + + + org.springframework + spring-beans + + + org.apache.james + apache-mime4j-core + + + org.apache.james + apache-mime4j-dom + + + + commons-io + commons-io + test + + + + + + + noTest + + + windows + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + + + + + + + maven-assembly-plugin + + + + fully.qualified.MainClass + + + + jar-with-dependencies + + + + + + + \ No newline at end of file diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/JestClientBuilder.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/JestClientBuilder.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/JestClientBuilder.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/JestClientBuilder.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,177 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search; + +import com.google.gson.JsonObject; +import io.searchbox.client.JestClient; +import io.searchbox.client.JestClientFactory; +import io.searchbox.client.config.HttpClientConfig; +import io.searchbox.indices.CreateIndex; +import io.searchbox.indices.mapping.PutMapping; +import org.apache.james.mailbox.store.json.model.MailboxConstants; +import org.apache.james.mailbox.store.json.model.MessageConstants; +import org.elasticsearch.common.settings.ImmutableSettings; + +/** + * Builds a Jest client for ElasticSearch. + */ +public class JestClientBuilder { + private JestClient jestClient; + + private int shardsNb; + private int replicaNb; + + public JestClient getJestClient() { + return jestClient; + } + + public JestClientBuilder(String hostString, int shardsNb, int replicaNb) { + JestClientFactory factory = new JestClientFactory(); + factory.setHttpClientConfig( + new HttpClientConfig + .Builder(hostString) + .multiThreaded(true) + .build() + ); + jestClient = factory.getObject(); + this.shardsNb = shardsNb; + this.replicaNb = replicaNb; + } + + /** + * Create the index with the given settings if not exist + * + * @return The updated MailboxJestClientBuilder + * @throws Exception + */ + public JestClientBuilder ensureSearchIndexExists() throws Exception { + ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder(); + settingsBuilder.put("number_of_shards", shardsNb); + settingsBuilder.put("number_of_replicas", replicaNb); + jestClient.execute( + new CreateIndex.Builder(MessageConstants.SEARCH_INDEX_NAME) + .settings( + settingsBuilder + .build() + .getAsMap() + ) + .build() + ); + return this; + } + + /** + * Create the mapping used for messages if not exist + * + * @return The updated MailboxJestClientBuilder + * @throws Exception + */ + public JestClientBuilder ensureTypeMessagesExists() throws Exception { + JsonObject mappingObject = new JsonObject(); + JsonObject messageMapping = new JsonObject(); + JsonObject parentSpec = new JsonObject(); + JsonObject propertiesObject = new JsonObject(); + mappingObject.add(MessageConstants.SEARCH_TYPE_NAME, messageMapping); + messageMapping.add("_parent", parentSpec); + parentSpec.addProperty("type", MailboxConstants.SEARCH_TYPE_NAME); + mappingObject.add("properties", propertiesObject); + propertiesObject.add(MessageConstants.UID, generateFieldDescription("long")); + propertiesObject.add(MessageConstants.MODSEQ, generateFieldDescription("long")); + propertiesObject.add(MessageConstants.FLAG_ANSWERED, generateFieldDescription("boolean")); + propertiesObject.add(MessageConstants.FLAG_DELETED, generateFieldDescription("boolean")); + propertiesObject.add(MessageConstants.FLAG_DRAFT, generateFieldDescription("boolean")); + propertiesObject.add(MessageConstants.FLAG_FLAGGED, generateFieldDescription("boolean")); + propertiesObject.add(MessageConstants.FLAG_RECENT, generateFieldDescription("boolean")); + propertiesObject.add(MessageConstants.FLAG_SEEN, generateFieldDescription("boolean")); + propertiesObject.add(MessageConstants.BODY_CONTENT, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.HEADER_CONTENT, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.MEDIA_TYPE, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.FULL_CONTENT_OCTET, generateFieldDescription("long")); + propertiesObject.add(MessageConstants.TEXTUAL_LINE_COUNT, generateFieldDescription("long")); + propertiesObject.add(MessageConstants.ARRIVAL, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.CC, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.BCC, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.FROM, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.TO, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.SUBJECT, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.SENT_DATE, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.DISPLAY_FROM, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.DISPLAY_TO, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.HEADERS_NAME, generateFieldDescription("string")); + propertiesObject.add(MessageConstants.UID, generateFieldDescription("string")); + JsonObject internalDateDescription = generateFieldDescription("date"); + internalDateDescription.addProperty("format", "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"); + propertiesObject.add(MessageConstants.DATE, internalDateDescription); + JsonObject mailboxIdDescription = new JsonObject(); + mailboxIdDescription.addProperty("index", "not_analyzed"); + propertiesObject.add(MessageConstants.MAILBOX_ID, mailboxIdDescription); + JsonObject nestedProperties = new JsonObject(); + nestedProperties.add(MessageConstants.HEADERS_NESTED_NAME, generateFieldDescription("string")); + nestedProperties.add(MessageConstants.HEADERS_NESTED_VALUE, generateFieldDescription("string")); + JsonObject nestedHeaderObject = generateFieldDescription("nested"); + nestedHeaderObject.add("properties", nestedProperties); + propertiesObject.add(MessageConstants.HEADERS, nestedHeaderObject); + PutMapping putMapping = new PutMapping.Builder( + MessageConstants.SEARCH_INDEX_NAME, + MessageConstants.SEARCH_TYPE_NAME, + mappingObject.toString() + ).build(); + jestClient.execute(putMapping); + return this; + } + + protected JsonObject generateFieldDescription(String fieldType) { + JsonObject fieldDescription = new JsonObject(); + fieldDescription.addProperty("type", fieldType); + return fieldDescription; + } + + /** + * Create the mapping used for documents if not exist + * + * @return The updated MailboxJestClientBuilder + * @throws Exception + */ + public JestClientBuilder ensureTypeMailboxsExists() throws Exception { + JsonObject mappingObject = new JsonObject(); + JsonObject messageMapping = new JsonObject(); + JsonObject propertiesObject = new JsonObject(); + mappingObject.add(MessageConstants.SEARCH_TYPE_NAME, messageMapping); + mappingObject.add("properties", propertiesObject); + mappingObject.add(MailboxConstants.ID, generateStringFieldDescription()); + mappingObject.add(MailboxConstants.USER, generateStringFieldDescription()); + mappingObject.add(MailboxConstants.NAME, generateStringFieldDescription()); + mappingObject.add(MailboxConstants.NAMESPACE, generateStringFieldDescription()); + PutMapping putMapping = new PutMapping.Builder( + MailboxConstants.SEARCH_INDEX_NAME, + MailboxConstants.SEARCH_TYPE_NAME, + mappingObject.toString() + ).build(); + jestClient.execute(putMapping); + return this; + } + + protected JsonObject generateStringFieldDescription() { + JsonObject fieldDescription = new JsonObject(); + fieldDescription.addProperty("type", "string"); + fieldDescription.addProperty("index", "not_analyzed"); + return fieldDescription; + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/MailboxBulkGenerator.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/MailboxBulkGenerator.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/MailboxBulkGenerator.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/MailboxBulkGenerator.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,79 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.bulk; + +import com.google.gson.JsonObject; +import io.searchbox.core.Bulk; +import io.searchbox.core.Delete; +import io.searchbox.core.Index; +import io.searchbox.core.Update; +import org.apache.james.mailbox.store.json.MailboxDocumentBuilder; +import org.apache.james.mailbox.store.json.model.MailboxConstants; +import org.apache.james.mailbox.store.mail.model.Mailbox; + +/** + * Generate Bulk requests operating on mailbox Type + */ +public class MailboxBulkGenerator { + + public Bulk generateBulkForMailboxRename(Mailbox mailbox) { + JsonObject scriptAsJson = new UpdateJsonBuilder() + .addUpdateScript(new UpdatedFields(MailboxConstants.NAME, UpdatedFields.Type.String, mailbox.getName())) + .addUpdateScript(new UpdatedFields(MailboxConstants.NAMESPACE, UpdatedFields.Type.String, mailbox.getNamespace())) + .addUpdateScript(new UpdatedFields(MailboxConstants.USER, UpdatedFields.Type.String, mailbox.getUser())) + .buildUpdateScript(); + Bulk.Builder bulkBuilder = new Bulk.Builder() + .defaultIndex(MailboxConstants.SEARCH_INDEX_NAME) + .defaultType(MailboxConstants.SEARCH_TYPE_NAME); + bulkBuilder.addAction(new Update.Builder(scriptAsJson) + .index(MailboxConstants.SEARCH_INDEX_NAME) + .type(MailboxConstants.SEARCH_TYPE_NAME) + .id(mailbox.getMailboxId().toString()) + .build() + ); + return bulkBuilder.build(); + } + + public Bulk generateBulkForMailboxAddition(Mailbox mailbox) { + JsonObject document = new MailboxDocumentBuilder(mailbox).build(); + Bulk.Builder bulkBuilder = new Bulk.Builder() + .defaultIndex(MailboxConstants.SEARCH_INDEX_NAME) + .defaultType(MailboxConstants.SEARCH_TYPE_NAME); + bulkBuilder.addAction(new Index.Builder(document) + .index(MailboxConstants.SEARCH_INDEX_NAME) + .type(MailboxConstants.SEARCH_TYPE_NAME) + .id(mailbox.getMailboxId().toString()) + .build() + ); + return bulkBuilder.build(); + } + + public Bulk generateBulkForMailboxDeletion(Mailbox mailbox) { + Bulk.Builder bulkBuilder = new Bulk.Builder() + .defaultIndex(MailboxConstants.SEARCH_INDEX_NAME) + .defaultType(MailboxConstants.SEARCH_TYPE_NAME); + bulkBuilder.addAction( + new Delete.Builder(mailbox.getMailboxId().toString()) + .index(MailboxConstants.SEARCH_INDEX_NAME) + .type(MailboxConstants.SEARCH_TYPE_NAME) + .build()); + return bulkBuilder.build(); + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/MessageBulkGenerator.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/MessageBulkGenerator.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/MessageBulkGenerator.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/MessageBulkGenerator.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,182 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.mailbox.elasticsearch.search.bulk; + +import com.google.gson.JsonObject; +import io.searchbox.core.Bulk; +import io.searchbox.core.Delete; +import io.searchbox.core.Index; +import io.searchbox.core.Update; +import org.apache.james.mailbox.MailboxSession; +import org.apache.james.mailbox.store.json.model.MessageConstants; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.model.MessageRange; +import org.apache.james.mailbox.store.json.MessageDocumentBuilder; +import org.apache.james.mailbox.store.mail.MessageMapper; +import org.apache.james.mailbox.store.mail.MessageMapperFactory; +import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.Message; +import org.apache.james.mailbox.store.search.extractor.TextExtractorFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Generate Bulk requests for message events + */ +public class MessageBulkGenerator { + + private MessageMapperFactory messageMapperFactory; + private TextExtractorFactory textExtractorFactory; + + public MessageBulkGenerator(MessageMapperFactory messageMapperFactory) { + this.messageMapperFactory = messageMapperFactory; + } + + public void setTextExtractorFactory(TextExtractorFactory textExtractorFactory) { + this.textExtractorFactory = textExtractorFactory; + } + + /** + * Calculate the id that will be used for this message in ElasticSearch + * + * @param message The message we want to generate an id. + * @return String used as id in ElasticSearch + */ + public static String calculateId( Message message ) { + return calculateId(message.getMailboxId().toString(), message.getUid()); + } + + /** + * Calculate the id used by ElasticSearch + * + * @param mailboxUUID Uuid of the mailbox + * @param messageUID Uid of the message + * @return String used as id in ElasticSearch + */ + public static String calculateId( String mailboxUUID, long messageUID ) { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(mailboxUUID); + stringBuilder.append("-"); + stringBuilder.append(messageUID); + return stringBuilder.toString(); + } + + /** + * Generate Bulk content to manage deletion + * + * @param mailbox Mailbox you want to delete messages into + * @param range Range of message you want to delete + * @return The bulk request you have to execute in order to delete these message + */ + public Bulk generateBulkForDeletions(Mailbox mailbox, MessageRange range) { + List toDelete = retrieveMessagesFromRange(mailbox, range); + Bulk.Builder bulkBuilder = new Bulk.Builder() + .defaultIndex(MessageConstants.SEARCH_INDEX_NAME) + .defaultType(MessageConstants.SEARCH_TYPE_NAME); + for (String messageKey : toDelete) { + bulkBuilder.addAction(new Delete.Builder(messageKey) + .index(MessageConstants.SEARCH_INDEX_NAME) + .type(MessageConstants.SEARCH_TYPE_NAME) + .build()); + } + return bulkBuilder.build(); + } + + /** + * Method used by subclass to generate Bulk requests for additions + * + * @param message Message you want to index + * @return The bulk request you have to execute in order to add this message to ElasticSearch + * @throws IOException Thrown when we can not read stream of data composing the message + * @throws MailboxException Thrown when Mime is malformed + */ + public Bulk generateBulkForAdditions(Message message) throws IOException, MailboxException { + JsonObject document = new MessageDocumentBuilder(message) + .setTextExtractorFactory(textExtractorFactory) + .addMetadata(MessageDocumentBuilder.MetadataMode.All) + .parseMessage(MessageDocumentBuilder.HeaderMode.All, MessageDocumentBuilder.ContentMode.BodyAndAttachment) + .addContent(MessageDocumentBuilder.ContentMode.BodyAndAttachment) + .build(); + Bulk.Builder bulkBuilder = new Bulk.Builder() + .defaultIndex(MessageConstants.SEARCH_INDEX_NAME) + .defaultType(MessageConstants.SEARCH_TYPE_NAME); + bulkBuilder.addAction(new Index.Builder(document) + .index(MessageConstants.SEARCH_INDEX_NAME) + .type(MessageConstants.SEARCH_TYPE_NAME) + .setParameter("parent", message.getMailboxId().toString()) + .id(calculateId(message)) + .build() + ); + return bulkBuilder.build(); + } + + /** + * Method used by subclass to generate Bulk requests for deletions + * + * @param mailbox Mailbox you want to delete messages into + * @param range Range of message to update into this mailbox + * @return Bulk request to use + */ + public Bulk generateBulkForUpdates(MailboxSession session, Mailbox mailbox, MessageRange range) throws MailboxException { + MessageMapper messageMapper = messageMapperFactory.getMessageMapper(session); + Bulk.Builder bulkBuilder = new Bulk.Builder() + .defaultIndex(MessageConstants.SEARCH_INDEX_NAME) + .defaultType(MessageConstants.SEARCH_TYPE_NAME); + Iterator> it = messageMapper.findInMailbox(mailbox, range, MessageMapper.FetchType.Metadata, -1); + while(it.hasNext()) { + Message message = it.next(); + addMessageUpdateToBulk(bulkBuilder, message); + } + return bulkBuilder.build(); + } + + private void addMessageUpdateToBulk(Bulk.Builder bulkBuilder, Message message) throws MailboxException { + bulkBuilder.addAction(new Update.Builder(getUpdateScript(message)) + .index(MessageConstants.SEARCH_INDEX_NAME) + .type(MessageConstants.SEARCH_TYPE_NAME) + .setParameter("parent", message.getMailboxId().toString()) + .id(calculateId(message)) + .build()); + } + + private JsonObject getUpdateScript(Message message) { + UpdateJsonBuilder updateJsonBuilder = new UpdateJsonBuilder(); + updateJsonBuilder.addUpdateScript(new UpdatedFields(MessageConstants.MODSEQ, UpdatedFields.Type.Numeric, message.getModSeq())); + updateJsonBuilder.addUpdateScript(new UpdatedFields(MessageConstants.FLAG_ANSWERED, UpdatedFields.Type.Boolean, message.isAnswered())); + updateJsonBuilder.addUpdateScript(new UpdatedFields(MessageConstants.FLAG_DRAFT, UpdatedFields.Type.Boolean, message.isDraft())); + updateJsonBuilder.addUpdateScript(new UpdatedFields(MessageConstants.FLAG_DELETED, UpdatedFields.Type.Boolean, message.isDeleted())); + updateJsonBuilder.addUpdateScript(new UpdatedFields(MessageConstants.FLAG_RECENT, UpdatedFields.Type.Boolean, message.isRecent())); + updateJsonBuilder.addUpdateScript(new UpdatedFields(MessageConstants.FLAG_FLAGGED, UpdatedFields.Type.Boolean, message.isFlagged())); + updateJsonBuilder.addUpdateScript(new UpdatedFields(MessageConstants.FLAG_SEEN, UpdatedFields.Type.Boolean, message.isSeen())); + return updateJsonBuilder.buildUpdateScript(); + } + + private List retrieveMessagesFromRange (Mailbox mailbox, MessageRange range) { + List result = new ArrayList(); + for(Long uid : range) { + result.add(MessageBulkGenerator.calculateId(mailbox.getMailboxId().toString(), uid)); + } + return result; + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/UpdateJsonBuilder.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/UpdateJsonBuilder.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/UpdateJsonBuilder.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/UpdateJsonBuilder.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,72 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.bulk; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +import java.util.ArrayList; +import java.util.List; + +/** + * Helper to build update scripts + */ +public class UpdateJsonBuilder { + + private List updatedFieldses; + + public UpdateJsonBuilder() { + updatedFieldses = new ArrayList(); + } + + public UpdateJsonBuilder addUpdateScript(UpdatedFields updatedFields) { + updatedFieldses.add(updatedFields); + return this; + } + + public JsonObject buildUpdateScript() { + JsonObject scriptAsJson = new JsonObject(); + scriptAsJson.add("doc", getUpdatedFields()); + return scriptAsJson; + } + + private JsonObject getUpdatedFields() { + JsonObject paramsAsJsonObject = new JsonObject(); + for(UpdatedFields updatedFields : updatedFieldses) { + paramsAsJsonObject.add(updatedFields.field,getCorrespondingElement(updatedFields)); + } + return paramsAsJsonObject; + } + + private JsonElement getCorrespondingElement(UpdatedFields updatedFields) { + switch (updatedFields.type) { + case Boolean: + return new JsonPrimitive((Boolean) updatedFields.value); + case Numeric: + return new JsonPrimitive((Long) updatedFields.value); + case Array: + return (JsonArray) updatedFields.value; + default: + return new JsonPrimitive((String) updatedFields.value); + } + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/UpdatedFields.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/UpdatedFields.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/UpdatedFields.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/bulk/UpdatedFields.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,42 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.bulk; + +/** + * Structure that hold information about an update script param + */ +public class UpdatedFields { + String field; + Type type; + Object value; + + public enum Type { + Boolean, + Numeric, + String, + Array + } + + public UpdatedFields(String field, Type type, Object value) { + this.field = field; + this.type = type; + this.value = value; + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/DSLBuilder.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/DSLBuilder.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/DSLBuilder.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/DSLBuilder.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,70 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl; + +import com.google.gson.JsonObject; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLFilteredQuery; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLMatchAllQuery; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLQuery; + +/** + * A DSL builder gives you the JSON you should use to perform your search + */ +public class DSLBuilder { + + private DSLFilter filter; + private DSLQuery query; + + public DSLBuilder setFilter(DSLFilter filter) { + this.filter = filter; + return this; + } + + public DSLBuilder setQuery(DSLQuery query) { + this.query = query; + return this; + } + + public JsonObject build() { + if(filter == null) { + if(query == null) { + return null; + } + return buildQuery(); + } + if(query == null) { + query = new DSLMatchAllQuery(); + } + return combineQueryAndFilter(); + } + + private JsonObject buildQuery() { + JsonObject queryObject = new JsonObject(); + queryObject.add(DSLQuery.QUERY, query.getQueryAsJson()); + return queryObject; + } + + private JsonObject combineQueryAndFilter() { + JsonObject body = new DSLFilteredQuery(query, filter).getQueryAsJson(); + JsonObject finalQuery = new JsonObject(); + finalQuery.add(DSLQuery.QUERY, body); + return finalQuery; + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLAndFilter.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLAndFilter.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLAndFilter.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLAndFilter.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,30 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.filter; + +import com.google.gson.JsonObject; + +/** + * Represents a AND filter + */ +public class DSLAndFilter extends DSLBooleanOperationFilter { + public JsonObject getQueryAsJson() { + return getBoolQueryAsJson(AND); + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLBooleanOperationFilter.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLBooleanOperationFilter.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLBooleanOperationFilter.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLBooleanOperationFilter.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,71 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.filter; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.util.ArrayList; +import java.util.List; + +/** + * Abstract class to factorize code used by DSLAndFilter, DSLOrFilter and DSLNotFilter + */ +public abstract class DSLBooleanOperationFilter implements DSLFilter { + private List filters; + + public DSLBooleanOperationFilter() { + this.filters = new ArrayList(); + } + + /** + * @return The sub filter count + */ + public int getFilterCount() { + return filters.size(); + } + + /** + * @param query A filter you want to add to the sub filters + */ + public void addFilter(DSLFilter query) { + filters.add(query); + } + + public abstract JsonObject getQueryAsJson(); + + protected JsonArray getFilterAsJsonArray() { + if(filters.size() == 0) { + return null; + } + JsonArray result = new JsonArray(); + for(DSLFilter filter : filters) { + result.add(filter.getQueryAsJson()); + } + return result; + } + + protected JsonObject getBoolQueryAsJson(String boolOperation) { + JsonArray jsonArray = getFilterAsJsonArray(); + JsonObject boolBody = new JsonObject(); + boolBody.add(boolOperation, jsonArray); + return boolBody; + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLFilter.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLFilter.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLFilter.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLFilter.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,48 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.filter; + +import com.google.gson.JsonObject; + +/** + * A DSLQuery is an object that represent a DSL filter. + * + * The basic operation is to get it as a JSON. + */ +public interface DSLFilter { + public final String AND = "and"; + public final String OR = "or"; + public final String NOT = "not"; + public final String TERM = "term"; + public final String CACHE = "_cache"; + public final String GTE = "gte"; + public final String LTE = "lte"; + public final String GT = "gt"; + public final String LT = "lt"; + public final String RANGE = "range"; + public final String NESTED = "nested"; + public final String PATH = "path"; + public final String FILTER = "filter"; + + /** + * + * @return JsonObject that will perform this filter + */ + public JsonObject getQueryAsJson(); +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLNestedFilter.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLNestedFilter.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLNestedFilter.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLNestedFilter.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,75 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.filter; + +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +/** + * Represents a DSL nested filter + */ +public class DSLNestedFilter implements DSLFilter { + + private String path; + private DSLFilter subFilter; + private boolean cache; + + public DSLNestedFilter(String path, DSLFilter subFilter) { + this(path, subFilter, false); + } + + public DSLNestedFilter(String path, DSLFilter subFilter, boolean cache) { + this.path = path; + this.subFilter = subFilter; + this.cache = cache; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public DSLFilter getSubFilter() { + return subFilter; + } + + public void setSubFilter(DSLFilter subFilter) { + this.subFilter = subFilter; + } + + public boolean isCache() { + return cache; + } + + public void setCache(boolean cache) { + this.cache = cache; + } + + public JsonObject getQueryAsJson() { + JsonObject filterJson = new JsonObject(); + filterJson.add(PATH, new JsonPrimitive(path)); + filterJson.add(FILTER, subFilter.getQueryAsJson()); + JsonObject result = new JsonObject(); + result.add(NESTED, filterJson); + return result; + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLNotFilter.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLNotFilter.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLNotFilter.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLNotFilter.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,38 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.filter; + +import com.google.gson.JsonObject; + +/** + * Represents a NOT filter + */ +public class DSLNotFilter implements DSLFilter { + private DSLFilter filter; + + public DSLNotFilter(DSLFilter filter) { + this.filter = filter; + } + + public JsonObject getQueryAsJson() { + JsonObject body = new JsonObject(); + body.add(NOT, filter.getQueryAsJson()); + return body; + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLOrFilter.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLOrFilter.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLOrFilter.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLOrFilter.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,30 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.filter; + +import com.google.gson.JsonObject; + +/** + * Represents a OR filter + */ +public class DSLOrFilter extends DSLBooleanOperationFilter { + public JsonObject getQueryAsJson() { + return getBoolQueryAsJson(OR); + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLRangeFilter.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLRangeFilter.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLRangeFilter.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLRangeFilter.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,176 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.filter; + +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +/** + * Represents a RangeFilter + */ +public class DSLRangeFilter implements DSLFilter{ + private String field; + private Number gtNumberValue; + private Number ltNumberValue; + private String gtString; + private String ltString; + private Comparator gtComparator; + private Comparator ltComparator; + private Type type; + + enum Comparator { + STRICT, + LAXIST + } + + enum Type { + NUMBER, + STRING + } + + public DSLRangeFilter(String field) { + this.field = field; + this.gtNumberValue = null; + this.ltNumberValue = null; + this.type = Type.NUMBER; + } + + public DSLRangeFilter gte(Number value) { + type = Type.NUMBER; + gtNumberValue = value; + gtComparator = Comparator.LAXIST; + return this; + } + + public DSLRangeFilter lte(Number value) { + type = Type.NUMBER; + ltNumberValue = value; + ltComparator = Comparator.LAXIST; + return this; + } + + public DSLRangeFilter gt(Number value) { + type = Type.NUMBER; + gtNumberValue = value; + gtComparator = Comparator.STRICT; + return this; + } + + public DSLRangeFilter lt(Number value) { + type = Type.NUMBER; + ltNumberValue = value; + ltComparator = Comparator.STRICT; + return this; + } + + public DSLRangeFilter gte(String string) { + type = Type.STRING; + gtString = string; + gtComparator = Comparator.LAXIST; + return this; + } + + public DSLRangeFilter lte(String string) { + type = Type.STRING; + ltString = string; + ltComparator = Comparator.LAXIST; + return this; + } + + public DSLRangeFilter gt(String string) { + type = Type.STRING; + gtString = string; + gtComparator = Comparator.STRICT; + return this; + } + + public DSLRangeFilter lt(String string) { + type = Type.STRING; + ltString = string; + ltComparator = Comparator.STRICT; + return this; + } + + public JsonObject getQueryAsJson() { + if( field == null ) { + return null; + } + JsonObject conditions = getRangeCondition(); + JsonObject rangeBody = new JsonObject(); + rangeBody.add(field, conditions); + JsonObject rangeObject = new JsonObject(); + rangeObject.add(RANGE, rangeBody); + return rangeObject; + } + + private JsonObject getRangeCondition() { + JsonObject conditions = new JsonObject(); + switch (type) { + case NUMBER: + return getNumericCondition(conditions); + case STRING: + return getStringCondition(conditions); + default: + throw new RuntimeException(); + } + + } + + private JsonObject getNumericCondition(JsonObject conditions) { + if( gtNumberValue == null && ltNumberValue == null ) { + return null; + } + if(gtNumberValue != null) { + if(gtComparator == Comparator.STRICT) { + conditions.add(GT, new JsonPrimitive(gtNumberValue)); + } else { + conditions.add(GTE, new JsonPrimitive(gtNumberValue)); + } + } + if(ltNumberValue != null) { + if(ltComparator == Comparator.STRICT) { + conditions.add(LT, new JsonPrimitive(ltNumberValue)); + } else { + conditions.add(LTE, new JsonPrimitive(ltNumberValue)); + } + } + return conditions; + } + + private JsonObject getStringCondition(JsonObject conditions) { + if( gtString == null && ltString == null ) { + return null; + } + if(gtString != null) { + if(gtComparator == Comparator.STRICT) { + conditions.add(GT, new JsonPrimitive(gtString)); + } else { + conditions.add(GTE, new JsonPrimitive(gtString)); + } + } + if(ltString != null) { + if(ltComparator == Comparator.STRICT) { + conditions.add(LT, new JsonPrimitive(ltString)); + } else { + conditions.add(LTE, new JsonPrimitive(ltString)); + } + } + return conditions; + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLTermFilter.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLTermFilter.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLTermFilter.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLTermFilter.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,103 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.filter; + +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +/** + * Represents a term filter + */ +public class DSLTermFilter implements DSLFilter { + private boolean cache; + private String field; + private String filterPattern; + private boolean booleanValue; + private Number number; + + enum Type { + BOOLEAN, + NUMBER, + STRING + } + + private Type type; + + public Type getType() { + return type; + } + + public DSLTermFilter(String field, boolean cache) { + this.cache = cache; + this.field = field; + } + + public DSLTermFilter(String field) { + this(field, true); + } + + public DSLTermFilter boolFilter(boolean booleanValue) { + this.booleanValue = booleanValue; + this.type = Type.BOOLEAN; + return this; + } + + public DSLTermFilter numberFilter(Number number) { + this.number = number; + this.type = Type.NUMBER; + return this; + } + + public DSLTermFilter stringFilter(String filterPattern) { + this.filterPattern = filterPattern; + this.type = Type.STRING; + return this; + } + + public JsonObject getQueryAsJson() { + if(field.isEmpty() ) { + return null; + } + JsonObject termField = new JsonObject(); + JsonPrimitive searchPrimitive = getPrimitive(); + if(searchPrimitive == null) { + return null; + } + termField.add(field, searchPrimitive); + JsonObject result = new JsonObject(); + result.add(TERM, termField); + if(!cache) { + result.add(CACHE, new JsonPrimitive(false)); + } + return result; + } + + private JsonPrimitive getPrimitive() { + switch (type) { + case BOOLEAN: + return new JsonPrimitive(booleanValue); + case STRING: + return new JsonPrimitive(filterPattern); + case NUMBER: + return new JsonPrimitive(number); + default: + return null; + } + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLBoolQuery.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLBoolQuery.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLBoolQuery.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLBoolQuery.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,95 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.query; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.util.ArrayList; +import java.util.List; + +/** + * Code factorisation for Boolean queries. + * + * Note : this bool builder is not as generic as the elastic search API, + * because we have a simple use case : our boolean queries will always have + * only one type of items : should, should not or must not. You can use a DSLAndQuery + * to combine these elements in a single query. + */ +public class DSLBoolQuery implements DSLQuery { + private List mustQueries; + private List shouldQueries; + private List must_notQueries; + + public DSLBoolQuery() { + this.mustQueries = new ArrayList(); + this.shouldQueries = new ArrayList(); + this.must_notQueries = new ArrayList(); + } + + /** + * @return The sub query count + */ + public int getQueryCount() { + return mustQueries.size() + shouldQueries.size() + must_notQueries.size(); + } + + public void must(DSLQuery query) { + mustQueries.add(query); + } + + public void should(DSLQuery query) { + shouldQueries.add(query); + } + + public void must_not(DSLQuery query) { + must_notQueries.add(query); + } + + protected JsonArray getQueriesJsonArray(List queries) { + if(queries.size() == 0) { + return null; + } + JsonArray result = new JsonArray(); + for(DSLQuery query : queries) { + result.add(query.getQueryAsJson()); + } + return result; + } + + public JsonObject getQueryAsJson() { + JsonObject boolBody = new JsonObject(); + if( mustQueries.size() > 0) { + JsonArray mustArray = getQueriesJsonArray(mustQueries); + boolBody.add(MUST, mustArray); + } + if(shouldQueries.size() > 0) { + JsonArray shouldArray = getQueriesJsonArray(shouldQueries); + boolBody.add(SHOULD, shouldArray); + } + if(must_notQueries.size() > 0) { + JsonArray must_notArray = getQueriesJsonArray(must_notQueries); + boolBody.add(MUST_NOT, must_notArray); + } + JsonObject boolQuery = new JsonObject(); + boolQuery.add(BOOL, boolBody); + return boolQuery; + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLFilteredQuery.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLFilteredQuery.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLFilteredQuery.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLFilteredQuery.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,76 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.query; + +import com.google.gson.JsonObject; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLFilter; + +/** + * Represents a DSL Filtered query + */ +public class DSLFilteredQuery implements DSLQuery { + public final String FILTER = "filter"; + public final String FILTERED = "filtered"; + + private DSLFilter filter; + private DSLQuery query; + + public DSLFilteredQuery(DSLQuery query, DSLFilter filter) { + this.filter = filter; + this.query = query; + } + + public DSLFilter getFilter() { + return filter; + } + + public void setFilter(DSLFilter filter) { + this.filter = filter; + } + + public DSLQuery getQuery() { + return query; + } + + public void setQuery(DSLQuery query) { + this.query = query; + } + + public JsonObject getQueryAsJson() { + if(filter == null) { + if(query == null) { + return null; + } + return query.getQueryAsJson(); + } + if(query == null) { + query = new DSLMatchAllQuery(); + } + return combineQueryAndFilter(); + } + + private JsonObject combineQueryAndFilter() { + JsonObject body = new JsonObject(); + body.add(QUERY, query.getQueryAsJson()); + body.add(FILTER, filter.getQueryAsJson()); + JsonObject filteredObject = new JsonObject(); + filteredObject.add(FILTERED, body); + return filteredObject; + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMatchAllQuery.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMatchAllQuery.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMatchAllQuery.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMatchAllQuery.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,32 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.query; + +import com.google.gson.JsonObject; + +/** + * Return a query that match any document. + */ +public class DSLMatchAllQuery implements DSLQuery { + public JsonObject getQueryAsJson() { + JsonObject result = new JsonObject(); + result.add(MATCH_ALL, new JsonObject()); + return result; + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMatchQuery.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMatchQuery.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMatchQuery.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMatchQuery.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,51 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.query; + +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +/** + * Represents a query getting documents matching a search criterion in a given field. + */ +public class DSLMatchQuery implements DSLQuery{ + private String field; + private String searchCriterion; + + /** + * @param field The document field you want to search into + * @param searchCriterion The pattern you want to search + */ + public DSLMatchQuery(String field, String searchCriterion) { + this.field = field; + this.searchCriterion = searchCriterion; + } + + public JsonObject getQueryAsJson() { + if(field.isEmpty() || searchCriterion.isEmpty()) { + return null; + } + JsonObject matchField = new JsonObject(); + JsonPrimitive searchPrimitive = new JsonPrimitive(searchCriterion); + matchField.add(field, searchPrimitive); + JsonObject result = new JsonObject(); + result.add(MATCH, matchField); + return result; + } +} \ No newline at end of file diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMustNotQuery.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMustNotQuery.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMustNotQuery.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMustNotQuery.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,28 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.query; + +/** + * Represents a should not query, combining sub queries + */ +public class DSLMustNotQuery extends DSLBoolQuery implements DSLSpecializedBoolQuery { + public void addQuery(DSLQuery query) { + must_not(query); + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMustQuery.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMustQuery.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMustQuery.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLMustQuery.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,28 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.query; + +/** + * Represents a must query, combining sub queries + */ +public class DSLMustQuery extends DSLBoolQuery implements DSLSpecializedBoolQuery { + public void addQuery(DSLQuery query) { + must(query); + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLNestedQuery.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLNestedQuery.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLNestedQuery.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLNestedQuery.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,91 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.query; + +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +/** + * DSL nested Query implementation + */ +public class DSLNestedQuery implements DSLQuery { + private String path; + private Score_mode scoreMode; + private DSLQuery subQuery; + + public enum Score_mode { + AVG, + SUM, + MAX, + NONE + } + + public JsonObject getQueryAsJson() { + JsonObject nestedContent = new JsonObject(); + nestedContent.add(PATH, new JsonPrimitive(path)); + switch (scoreMode) { + case AVG: + nestedContent.add(SCORE_MODE, new JsonPrimitive(AVG)); + break; + case MAX: + nestedContent.add(SCORE_MODE, new JsonPrimitive(MAX)); + break; + case NONE: + nestedContent.add(SCORE_MODE, new JsonPrimitive(NONE)); + break; + case SUM: + nestedContent.add(SCORE_MODE, new JsonPrimitive(SUM)); + break; + } + nestedContent.add(QUERY, subQuery.getQueryAsJson()); + JsonObject finalQuery = new JsonObject(); + finalQuery.add(NESTED, nestedContent); + return finalQuery; + } + + public DSLNestedQuery(String path, Score_mode scoreMode, DSLQuery subQuery) { + this.path = path; + this.scoreMode = scoreMode; + this.subQuery = subQuery; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public Score_mode getScoreMode() { + return scoreMode; + } + + public void setScoreMode(Score_mode scoreMode) { + this.scoreMode = scoreMode; + } + + public DSLQuery getSubQuery() { + return subQuery; + } + + public void setSubQuery(DSLQuery subQuery) { + this.subQuery = subQuery; + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLQuery.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLQuery.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLQuery.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLQuery.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,49 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.query; + +import com.google.gson.JsonObject; + +/** + * A DSLQuery is an object that represent a DSL query. + * + * The basic operation is to get it as a JSON. + */ +public interface DSLQuery { + public final String MATCH_ALL = "match_all"; + public final String MUST = "must"; + public final String SHOULD = "should"; + public final String MUST_NOT = "must_not"; + public final String MATCH = "match"; + public final String NESTED = "nested"; + public final String SCORE_MODE = "score_mode"; + public final String PATH = "path"; + public final String BOOL = "bool"; + public final String AVG = "avg"; + public final String SUM = "sum"; + public final String MAX = "max"; + public final String NONE = "none"; + public final String QUERY = "query"; + + /** + * + * @return JsonObject that will perform this query + */ + public JsonObject getQueryAsJson(); +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLShouldQuery.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLShouldQuery.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLShouldQuery.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLShouldQuery.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,28 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.query; + +/** + * Represents a should query, combining sub queries. + */ +public class DSLShouldQuery extends DSLBoolQuery implements DSLSpecializedBoolQuery { + public void addQuery(DSLQuery query) { + should(query); + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLSpecializedBoolQuery.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLSpecializedBoolQuery.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLSpecializedBoolQuery.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLSpecializedBoolQuery.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,23 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.dsl.query; + +public interface DSLSpecializedBoolQuery extends DSLQuery { + public void addQuery(DSLQuery query); +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/AbstractElasticsearchMessageSearchIndex.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/AbstractElasticsearchMessageSearchIndex.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/AbstractElasticsearchMessageSearchIndex.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/AbstractElasticsearchMessageSearchIndex.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,252 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.index; + +import io.searchbox.client.JestClient; +import io.searchbox.core.Bulk; +import org.apache.james.mailbox.MailboxSession; +import org.apache.james.mailbox.elasticsearch.search.bulk.MailboxBulkGenerator; +import org.apache.james.mailbox.elasticsearch.search.bulk.MessageBulkGenerator; +import org.apache.james.mailbox.elasticsearch.search.query.SearchHandler; +import org.apache.james.mailbox.elasticsearch.search.JestClientBuilder; +import org.apache.james.mailbox.store.json.model.MessageConstants; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLAndFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLRangeFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLTermFilter; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.exception.UnavailableIndexException; +import org.apache.james.mailbox.model.MessageRange; +import org.apache.james.mailbox.model.SearchQuery; +import org.apache.james.mailbox.store.MailboxEventDispatcher; +import org.apache.james.mailbox.store.mail.MessageMapperFactory; +import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.Message; +import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex; + +import org.apache.james.mailbox.store.search.extractor.TextExtractorFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; + +import javax.mail.Flags; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * This class has common stuff for all our MessageSearchIndex using ElasticSearch + */ +public abstract class AbstractElasticsearchMessageSearchIndex extends ListeningMessageSearchIndex implements InitializingBean { + + protected JestClient jestClient; + private static final Logger LOG = LoggerFactory.getLogger(JestClient.class); + protected TextExtractorFactory textExtractorFactory; + + protected MailboxBulkGenerator mailboxBulkGenerator; + protected MessageBulkGenerator messageBulkGenerator; + + public AbstractElasticsearchMessageSearchIndex(MessageMapperFactory messageMapperFactory) { + super(messageMapperFactory); + mailboxBulkGenerator = new MailboxBulkGenerator(); + messageBulkGenerator = new MessageBulkGenerator(messageMapperFactory); + } + + private String hostString; + private int shardsNb; + private int replicaNb; + private int batchSize; + private Long largeRangeLimit; + + /** + * You can set the content extractor factory used by this search index threw Spring configuration + * + * @param textExtractorFactory Text extractor factory + */ + public void setTextExtractorFactory(TextExtractorFactory textExtractorFactory) { + this.textExtractorFactory = textExtractorFactory; + messageBulkGenerator.setTextExtractorFactory(textExtractorFactory); + } + + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } + + public void setLargeRangeLimit(long largeRangeLimit) { + this.largeRangeLimit = largeRangeLimit; + } + + public void setHostString(String hostString) { + this.hostString = hostString; + } + + public void setShardsNb(int shardsNb) { + this.shardsNb = shardsNb; + } + + public void setReplicaNb(int replicaNb) { + this.replicaNb = replicaNb; + } + + public void afterPropertiesSet() { + JestClientBuilder builder; + builder = new JestClientBuilder(hostString, shardsNb, replicaNb); + try { + builder.ensureSearchIndexExists() + .ensureTypeMailboxsExists() + .ensureTypeMessagesExists(); + } catch (Exception e) { + LOG.error("Can not initialize properly search index : error while initializing jest client : " + e.getMessage(), e); + // We can continue + // This error simply implies that : + LOG.error("Because of this error we are unable to ensure index existence and mapping"); + // The jest client can be used as soon as ElasticSearch is back online. + // UnavailableIndexException will be thrown until it happens + } + jestClient = builder.getJestClient(); + } + + public void stop() throws IOException { + jestClient.shutdownClient(); + } + + /** + * Search in a given mailbox + * + * @param mailbox Mailbox to search into + * @param searchQuery Search to execute + * @return Iterator on message uid matching this request. + * @throws MailboxException + */ + @Override + public Iterator search(final MailboxSession session, final Mailbox mailbox, SearchQuery searchQuery) throws MailboxException { + return new SearchHandler(jestClient).search(mailbox, searchQuery); + } + + /** + * Plug on the event system so that we can manage Mailbox events + * + * @param event Event to be processed + */ + @Override + public void event(Event event) { + super.event(event); + + try { + // Plug on the event system to index mailbox related events + if (event instanceof MailboxEventDispatcher.MailboxDeletionImpl) { + addMailbox(((MailboxEventDispatcher.MailboxDeletionImpl) event).getMailbox()); + } else if (event instanceof MailboxEventDispatcher.MailboxAddedImpl) { + deleteMailbox(((MailboxEventDispatcher.MailboxAddedImpl) event).getMailbox()); + } else if (event instanceof MailboxEventDispatcher.MailboxRenamedEventImpl) { + renameMailbox(((MailboxEventDispatcher.MailboxRenamedEventImpl) event).getNewMailbox()); + } + } catch(MailboxException exception) { + LOG.error("Error while managing event", exception); + } + } + + @Override + public abstract void add(MailboxSession session, Mailbox mailbox, Message message) throws MailboxException; + + @Override + public void delete(MailboxSession session, Mailbox mailbox, MessageRange range) throws MailboxException { + if(range.getType() == MessageRange.Type.ALL) { + performDeleteByQuery(allFilter(mailbox)); + } else if(range.getType() == MessageRange.Type.FROM) { + performDeleteByQuery(fromFilter(mailbox, range)); + } else if(isRangeTooLarge(range)) { + performDeleteByQuery(rangeFilter(mailbox, range)); + } else { + performDeleteByBulk(session, mailbox, range); + } + } + + private boolean isRangeTooLarge(MessageRange range) { + return range.getType() == MessageRange.Type.RANGE + && largeRangeLimit != null + && range.getUidTo() - range.getUidFrom() > largeRangeLimit; + } + + private void performDeleteByBulk(MailboxSession session, Mailbox mailbox, MessageRange range) throws MailboxException { + List ranges = getMessageRanges(range, batchSize); + for (MessageRange working_range : ranges) { + concreteDelete(session, mailbox, working_range); + } + } + + @Override + public void update(MailboxSession session, Mailbox mailbox, MessageRange range, Flags flags) throws MailboxException { + // We can not perform Update by query because of the value of the MODSEQ that needs to be retrieved for each message + List ranges = getMessageRanges(range, batchSize); + for( MessageRange working_range : ranges) { + concreteUpdate(session, mailbox, working_range, flags); + } + } + + private List getMessageRanges(MessageRange range, int batchSize) { + List ranges = new ArrayList(); + if(batchSize > 0) { + ranges = range.split(batchSize); + } else { + ranges.add(range); + } + return ranges; + } + + public abstract void concreteDelete(MailboxSession session, Mailbox mailbox, MessageRange range) throws MailboxException; + + public abstract void concreteUpdate(MailboxSession session, Mailbox mailbox, MessageRange range, Flags flags) throws MailboxException; + + protected abstract void performDeleteByQuery(DSLFilter filter) throws UnavailableIndexException; + + protected abstract void performBulk(Bulk bulk) throws UnavailableIndexException; + + public abstract void addMailbox(Mailbox mailbox) throws MailboxException; + + public abstract void deleteMailbox(Mailbox mailbox) throws MailboxException; + + public abstract void renameMailbox(Mailbox mailbox) throws MailboxException; + + private DSLAndFilter rangeFilter(Mailbox mailbox, MessageRange range) { + DSLAndFilter filter = new DSLAndFilter(); + filter.addFilter(allFilter(mailbox)); + filter.addFilter(new DSLRangeFilter(MessageConstants.UID) + .gte(range.getUidFrom()) + .lte(range.getUidTo()) + ); + return filter; + } + + private DSLAndFilter fromFilter(Mailbox mailbox, MessageRange range) { + DSLAndFilter filter = new DSLAndFilter(); + filter.addFilter(allFilter(mailbox)); + filter.addFilter(new DSLRangeFilter(MessageConstants.UID) + .gte(range.getUidFrom()) + ); + return filter; + } + + private DSLFilter allFilter(Mailbox mailbox) { + return new DSLTermFilter(MessageConstants.MAILBOX_ID) + .stringFilter(mailbox.getMailboxId().toString()); + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/DirectElasticsearchMessageSearchIndex.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/DirectElasticsearchMessageSearchIndex.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/DirectElasticsearchMessageSearchIndex.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/DirectElasticsearchMessageSearchIndex.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,122 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.index; + +import io.searchbox.core.Bulk; +import io.searchbox.core.DeleteByQuery; +import org.apache.james.mailbox.MailboxSession; +import org.apache.james.mailbox.elasticsearch.search.bulk.MailboxBulkGenerator; +import org.apache.james.mailbox.elasticsearch.search.bulk.MessageBulkGenerator; +import org.apache.james.mailbox.elasticsearch.search.dsl.DSLBuilder; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLMatchAllQuery; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.exception.UnavailableIndexException; +import org.apache.james.mailbox.model.MessageRange; +import org.apache.james.mailbox.store.mail.MessageMapperFactory; +import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.mail.Flags; +import java.io.IOException; + +/** + * This MessageSearchIndex directly sends its bulk requests generated from events to ElasticSearch. + * + * Warning : If your ElasticSearch Cluster is too slow, using this messageSearchIndex : + * - can slow down James + * - can take a lot of RAM + */ +public class DirectElasticsearchMessageSearchIndex extends AbstractElasticsearchMessageSearchIndex { + + private static final Logger LOG = LoggerFactory.getLogger(DirectElasticsearchMessageSearchIndex.class); + + public DirectElasticsearchMessageSearchIndex(MessageMapperFactory messageMapperFactory) { + super(messageMapperFactory); + } + + @Override + public void add(MailboxSession session, Mailbox mailbox, Message message) throws MailboxException { + Bulk bulk; + try { + bulk = messageBulkGenerator.generateBulkForAdditions(message); + } catch(IOException ioException) { + throw new UnavailableIndexException("Failed retrieving message content while indexing message in ElasticSearch ", ioException); + } + performBulk(bulk); + } + + @Override + public void concreteDelete(MailboxSession session, Mailbox mailbox, MessageRange range) throws MailboxException { + Bulk bulk = messageBulkGenerator.generateBulkForDeletions(mailbox, range); + performBulk(bulk); + } + + @Override + public void concreteUpdate(MailboxSession session, Mailbox mailbox, MessageRange range, Flags flags) throws MailboxException { + Bulk bulk = messageBulkGenerator.generateBulkForUpdates(session, mailbox, range); + performBulk(bulk); + } + + @Override + public void addMailbox(Mailbox mailbox) throws MailboxException { + Bulk bulk = mailboxBulkGenerator.generateBulkForMailboxAddition(mailbox); + performBulk(bulk); + } + + @Override + public void deleteMailbox(Mailbox mailbox) throws MailboxException { + Bulk bulk = mailboxBulkGenerator.generateBulkForMailboxDeletion(mailbox); + performBulk(bulk); + } + + @Override + public void renameMailbox(Mailbox mailbox) throws MailboxException { + Bulk bulk = mailboxBulkGenerator.generateBulkForMailboxRename(mailbox); + performBulk(bulk); + } + + @Override + protected void performDeleteByQuery(DSLFilter filter) throws UnavailableIndexException { + DeleteByQuery deleteByQuery = new DeleteByQuery.Builder( + new DSLBuilder() + .setQuery(new DSLMatchAllQuery()) + .setFilter(filter) + .build() + .toString() + ).build(); + try { + jestClient.execute(deleteByQuery); + } catch(Exception exception) { + throw new UnavailableIndexException("Index is not available during a DELETE by query", exception); + } + } + + @Override + protected void performBulk(Bulk bulk) throws UnavailableIndexException { + try { + jestClient.execute(bulk); + } catch(Exception exception) { + throw new UnavailableIndexException("Failed during a Bulk delete request for mailbox : ", exception); + } + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/EmbeddedKafkaElasticsearchMessageSearchIndex.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/EmbeddedKafkaElasticsearchMessageSearchIndex.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/EmbeddedKafkaElasticsearchMessageSearchIndex.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/EmbeddedKafkaElasticsearchMessageSearchIndex.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,100 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.index; + +import kafka.server.KafkaConfig; +import kafka.server.KafkaServerStartable; +import org.apache.curator.test.TestingServer; +import org.apache.james.mailbox.store.mail.MessageMapperFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Properties; + +/** + * This MessageSearchIndex relies on an embedded message broker for managing ElasticSearch + * in a differed way ( index, delete, update ), that does not impact James performances + */ +public class EmbeddedKafkaElasticsearchMessageSearchIndex extends KafkaElasticsearchMessageSearchIndex { + + private KafkaServerStartable kafkaServer; + private TestingServer zookeeper; + private int zookeeperPort; + private static final Logger LOG = LoggerFactory.getLogger(EmbeddedKafkaElasticsearchMessageSearchIndex.class); + private KafkaConfig config; + + public EmbeddedKafkaElasticsearchMessageSearchIndex(MessageMapperFactory messageMapperFactory, String topic, String kafkaHostIpString, int kafkaPort, int zookeeperPort, String logDir) { + super(messageMapperFactory, topic, kafkaHostIpString, kafkaPort); + this.zookeeperPort = zookeeperPort; + config = new KafkaConfig(createProperties(logDir, kafkaPort, 1)); + } + + protected void taskBeforeLaunchingProducer() { + try { + startZookeeper(); + kafkaServer = new KafkaServerStartable(config); + LOG.info("Starting local kafka broker..."); + kafkaServer.startup(); + LOG.info("Local Kafka server started"); + } catch(Exception e) { + LOG.error("Error while launching embedded Kafka : " + e.getMessage()); + LOG.error("Can not perform any search task until James restart"); + } + } + + /** + * Create properties used for embedded kafka configuration + * + * @param logDir directory to use for logging + * @param port Port used for zookeeper + * @param brokerId Id of your broker + * @return Properties to use to build configuration + */ + private static Properties createProperties(String logDir, int port, int brokerId) { + Properties properties = new Properties(); + properties.put("port", port+""); + properties.put("broker.id", brokerId+""); + properties.put("log.dir", logDir); + properties.put("zookeeper.connect", "localhost:2181"); + return properties; + } + + /** + * Starts embedded zookeeper + * @throws Exception + */ + private void startZookeeper() throws Exception{ + int clientPort = 2181; + zookeeper = new TestingServer(clientPort); + + } + + /** + * Stops our kafka server + * + * @throws IOException + */ + @Override + public void stop() throws IOException { + super.stop(); + zookeeper.close(); + kafkaServer.shutdown(); + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/KafkaElasticsearchMessageSearchIndex.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/KafkaElasticsearchMessageSearchIndex.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/KafkaElasticsearchMessageSearchIndex.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/index/KafkaElasticsearchMessageSearchIndex.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,179 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.index; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import io.searchbox.core.Bulk; +import kafka.javaapi.producer.Producer; +import kafka.producer.KeyedMessage; +import kafka.producer.ProducerConfig; +import org.apache.james.mailbox.MailboxSession; +import org.apache.james.mailbox.elasticsearch.search.bulk.MailboxBulkGenerator; +import org.apache.james.mailbox.elasticsearch.search.bulk.MessageBulkGenerator; +import org.apache.james.mailbox.elasticsearch.search.dsl.DSLBuilder; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLMatchAllQuery; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.exception.UnavailableIndexException; +import org.apache.james.mailbox.model.MessageRange; +import org.apache.james.mailbox.store.mail.MessageMapperFactory; +import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.mail.Flags; +import java.io.IOException; +import java.util.Properties; + +/** + * This MessageSearchIndex relies on a message broker for managing ElasticSearch + * in a differed way ( index, delete, update ), that does not impact James performances + */ +public class KafkaElasticsearchMessageSearchIndex extends AbstractElasticsearchMessageSearchIndex { + + public static final String TYPE = "type"; + public final static JsonPrimitive BULK = new JsonPrimitive("bulk"); + public final static JsonPrimitive DELETE_BY_QUERY = new JsonPrimitive("delete_by_query"); + public final static String CONTENT = "content"; + public final static String QUERY = "query"; + + private Producer producer; + + private String topic; + private int kafka_port; + private String kafka_ip; + private static final Logger LOG = LoggerFactory.getLogger(KafkaElasticsearchMessageSearchIndex.class); + private boolean producerLaunched; + + private Gson gson; + + public KafkaElasticsearchMessageSearchIndex(MessageMapperFactory messageMapperFactory, String topic, String kafkaHostIpString, int kafka_port) { + super(messageMapperFactory); + this.topic = topic; + this.kafka_ip = kafkaHostIpString; + this.kafka_port = kafka_port; + this.gson = new Gson(); + producerLaunched = false; + } + + protected void taskBeforeLaunchingProducer() { + + } + + public void afterPropertiesSet() { + super.afterPropertiesSet(); + taskBeforeLaunchingProducer(); + // launch producer need not to be called outside the constructor because of the Embedded Kafka Search Index + launchProducer(); + } + + /** + * Launch Kafka producer that sends Bulk request to Kafka. You need to do that after instantiation + * as embedded subclass will have to perform a few other tasks before launching the producer. + */ + public KafkaElasticsearchMessageSearchIndex launchProducer() { + if( !producerLaunched) { + Properties props = new Properties(); + props.put("metadata.broker.list", kafka_ip + ":" + kafka_port); + props.put("serializer.class", "kafka.serializer.StringEncoder"); + props.put("request.required.acks", "1"); + ProducerConfig config = new ProducerConfig(props); + producer = new Producer(config); + producerLaunched = true; + } else { + LOG.warn("Kafka producer was already instantiated"); + } + return this; + } + + /** + * Stops Kafka producer and Jest client + * + * @throws IOException + */ + @Override + public void stop() throws IOException { + producer.close(); + super.stop(); + } + + @Override + public void add(MailboxSession session, Mailbox mailbox, Message message) throws MailboxException { + try { + Bulk bulk = messageBulkGenerator.generateBulkForAdditions(message); + performBulk(bulk); + } catch(IOException ioException) { + throw new UnavailableIndexException("Failed retrieving message content while indexing message in elasticSearch", ioException); + } + } + + @Override + public void concreteDelete(MailboxSession session, Mailbox mailbox, MessageRange range) throws MailboxException { + Bulk bulk = messageBulkGenerator.generateBulkForDeletions(mailbox, range); + performBulk(bulk); + } + + @Override + public void concreteUpdate(MailboxSession session, Mailbox mailbox, MessageRange range, Flags flags) throws MailboxException { + Bulk bulk = messageBulkGenerator.generateBulkForUpdates(session, mailbox, range); + performBulk(bulk); + } + + @Override + public void addMailbox(Mailbox mailbox) { + Bulk bulk = mailboxBulkGenerator.generateBulkForMailboxAddition(mailbox); + performBulk(bulk); + } + + @Override + public void deleteMailbox(Mailbox mailbox) { + Bulk bulk = mailboxBulkGenerator.generateBulkForMailboxDeletion(mailbox); + performBulk(bulk); + } + + @Override + public void renameMailbox(Mailbox mailbox) { + Bulk bulk = mailboxBulkGenerator.generateBulkForMailboxRename(mailbox); + performBulk(bulk); + } + + @Override + protected void performDeleteByQuery(DSLFilter filter) { + JsonObject queryAsJson = new DSLBuilder() + .setQuery(new DSLMatchAllQuery()) + .setFilter(filter) + .build(); + JsonObject sentObject = new JsonObject(); + sentObject.add(TYPE, DELETE_BY_QUERY); + sentObject.add(QUERY, new JsonPrimitive(queryAsJson.toString())); + producer.send( new KeyedMessage(topic, sentObject.toString() )); + } + + @Override + protected void performBulk(Bulk bulk) { + JsonObject sentObject = new JsonObject(); + sentObject.add(TYPE, BULK); + sentObject.add(CONTENT, new JsonPrimitive((String) bulk.getData(gson))); + producer.send( new KeyedMessage(topic, sentObject.toString() )); + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/DateResolutionFormater.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/DateResolutionFormater.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/DateResolutionFormater.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/DateResolutionFormater.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,113 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.query; + +import org.apache.james.mailbox.model.SearchQuery; + +import java.util.Calendar; +import java.util.Date; + +/** + * This class gives us the date rounded using a given date resolution + */ +public class DateResolutionFormater { + enum Mode{ + Round, + Up, + Down + } + + public static Date calculateUpperDate(Date date, SearchQuery.DateResolution resolution) { + return calculateRoundedTime(date, resolution, Mode.Up); + } + + public static Date calculateLowerDate(Date date, SearchQuery.DateResolution resolution) { + return calculateRoundedTime(date, resolution, Mode.Down); + } + + public static Date calculateRoundedDate(Date date, SearchQuery.DateResolution resolution) { + return calculateRoundedTime(date, resolution, Mode.Round); + } + + /** + * Round the date given a mode and a time resolution + * + * @param date Date we want to add delta to + * @param resolution Resolution to use + * @param mode Mode + * @return Your new date, with the resolution unit modified and sub resolution units set to 0 + */ + private static Date calculateRoundedTime(Date date, SearchQuery.DateResolution resolution, Mode mode) { + int delta = getTimeDifferenceFromMode(mode); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + boolean exactResolution = true; + switch(resolution) { + case Year: + calendar.set(Calendar.MONTH, 0); + exactResolution = false; + calendar.add(Calendar.YEAR, delta); + case Month: + if(exactResolution) { + exactResolution = false; + calendar.add(Calendar.MONTH, delta); + } + calendar.set(Calendar.DAY_OF_MONTH, 1); + case Day: + calendar.set(Calendar.HOUR_OF_DAY, 0); + if(exactResolution) { + exactResolution = false; + calendar.add(Calendar.DAY_OF_WEEK, delta); + } + case Hour: + calendar.set(Calendar.MINUTE, 0); + if(exactResolution) { + exactResolution = false; + calendar.add(Calendar.HOUR_OF_DAY, delta); + } + case Minute: + calendar.set(Calendar.SECOND, 0); + if(exactResolution) { + calendar.add(Calendar.MINUTE, delta); + } + return calendar.getTime(); + default: + return date; + } + } + + /** + * Generate the delta associated to this mode + * + * @param mode Mode + * @return add that to the date to specifically work on one mode + */ + private static int getTimeDifferenceFromMode(Mode mode) { + switch (mode) { + case Round: + return 0; + case Down: + return -1; + case Up: + return 1; + default: + return 0; + } + } +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/QueryConverter.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/QueryConverter.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/QueryConverter.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/QueryConverter.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,529 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.query; + +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLAndFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLTermFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLOrFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLNotFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLBooleanOperationFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLRangeFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLQuery; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLMatchQuery; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLMatchAllQuery; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLMustNotQuery; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLMustQuery; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLShouldQuery; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLSpecializedBoolQuery; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLNestedQuery; + +import org.apache.james.mailbox.exception.UnsupportedCriteriaException; +import org.apache.james.mailbox.store.json.JsonHeaderBuilder; +import org.apache.james.mailbox.store.json.model.MessageConstants; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.exception.UnsupportedSearchException; +import org.apache.james.mailbox.model.SearchQuery; + +import javax.mail.Flags; +import java.text.SimpleDateFormat; +import java.util.List; +import java.util.ArrayList; +import java.util.Date; + +/** + * This class is used for generating both a query builder and a filter builder from internal James SearchQuery. + */ +public class QueryConverter { + + /** + * Structure used to return the query builder and the filter to the caller. + * + * Note : both fields can be null : it means that this field was not specified in the caller SearchQuery. + * You mau want to use the default match all query , or no fields at all. + */ + public static class ElasticsearchQueryRepresentation{ + public DSLQuery query; + public DSLFilter filter; + + public ElasticsearchQueryRepresentation(DSLQuery query, DSLFilter filter) { + this.query = query; + this.filter = filter; + } + } + + /** + * Build ElasticSearch query builder and filterBuilder you should use with your request. + * All messages from all mailboxes are searched + * + * @param searchQuery Search query to execute + * @return The {@link ElasticsearchQueryRepresentation} Built from your search + */ + public static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery searchQuery) throws MailboxException { + List criteria = searchQuery.getCriterias(); + SearchQuery.Criterion criterion = new SearchQuery.ConjunctionCriterion(SearchQuery.Conjunction.AND, criteria); + return generateQueryBuilder(criterion); + } + + /** + * Build ElasticSearch query builder and filterBuilder you should use with your request. + * Only the mailbox designed by the mailboxUUID is searched + * + * @param searchQuery Search query to execute + * @param mailboxUUID Mailbox UUID to search in + * @return A json builder The {@link ElasticsearchQueryRepresentation} Built from your search + * @throws MailboxException + */ + public static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery searchQuery, String mailboxUUID) throws MailboxException { + List criteria = searchQuery.getCriterias(); + SearchQuery.Criterion criterion = new SearchQuery.ConjunctionCriterion(SearchQuery.Conjunction.AND, criteria); + ElasticsearchQueryRepresentation elasticsearchQueryRepresentation = generateQueryBuilder(criterion); + addMailboxFilters(elasticsearchQueryRepresentation, mailboxUUID); + return elasticsearchQueryRepresentation; + } + + /** + * Add filters to select only messages from a given mailbox + * + * @param elasticsearchQueryRepresentation result we want to add filter to. Will be modified + * @param mailboxUUID UUID of the mailbox + */ + private static void addMailboxFilters(ElasticsearchQueryRepresentation elasticsearchQueryRepresentation, String mailboxUUID) { + DSLAndFilter dslAndFilter; + if(elasticsearchQueryRepresentation.filter instanceof DSLAndFilter) { + dslAndFilter = (DSLAndFilter) elasticsearchQueryRepresentation.filter; + } else { + dslAndFilter = new DSLAndFilter(); + if(elasticsearchQueryRepresentation.filter != null) { + dslAndFilter.addFilter(elasticsearchQueryRepresentation.filter); + } + } + dslAndFilter.addFilter(new DSLTermFilter(MessageConstants.MAILBOX_ID).stringFilter(mailboxUUID)); + elasticsearchQueryRepresentation.filter = dslAndFilter; + } + + /** + * Build ElasticSearch query builder and filterBuilder you should use with your request. + * + * Note : this method will call the appropriate method for its subclass. If none can be find, its that there is no + * implementation for this subclass. + * + * @param criterion criterion we should parse. + * @return The {@link ElasticsearchQueryRepresentation} Built from your criterion + */ + private static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery.Criterion criterion) throws MailboxException { + if(criterion instanceof SearchQuery.ConjunctionCriterion) { + return generateQueryBuilder((SearchQuery.ConjunctionCriterion) criterion); + } else if(criterion instanceof SearchQuery.TextCriterion) { + return generateQueryBuilder((SearchQuery.TextCriterion) criterion); + } else if(criterion instanceof SearchQuery.ModSeqCriterion) { + return generateQueryBuilder((SearchQuery.ModSeqCriterion) criterion); + } else if(criterion instanceof SearchQuery.SizeCriterion) { + return generateQueryBuilder((SearchQuery.SizeCriterion) criterion); + } else if(criterion instanceof SearchQuery.FlagCriterion) { + return generateQueryBuilder((SearchQuery.FlagCriterion) criterion); + } else if(criterion instanceof SearchQuery.CustomFlagCriterion) { + return generateQueryBuilder((SearchQuery.CustomFlagCriterion) criterion); + } else if(criterion instanceof SearchQuery.InternalDateCriterion) { + return generateQueryBuilder((SearchQuery.InternalDateCriterion) criterion); + } else if(criterion instanceof SearchQuery.UidCriterion) { + return generateQueryBuilder((SearchQuery.UidCriterion) criterion); + } else if(criterion instanceof SearchQuery.HeaderCriterion) { + return generateQueryBuilder((SearchQuery.HeaderCriterion) criterion); + } else if(criterion instanceof SearchQuery.AllCriterion) { + return generateQueryBuilder((SearchQuery.AllCriterion) criterion); + } else { + throw new UnsupportedCriteriaException(); + } + } + + private static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery.AllCriterion criterion) throws MailboxException { + return new ElasticsearchQueryRepresentation( + new DSLMatchAllQuery(), null + ); + } + + private static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery.ConjunctionCriterion criterion) throws MailboxException { + List subCriteria = criterion.getCriteria(); + switch (criterion.getType()) { + case AND : + return convertAnd(subCriteria); + case OR : + return convertOr(subCriteria); + case NOR : + return convertNor(subCriteria); + default: + throw new RuntimeException("Can not find SearchCriterion.Conjonction used : " + criterion.getType()); + } + } + + /** + * Make DSL filters and DSL queries from a list of criteria and add it to a NOR query + * + * @param subCriteria Sub criteria + * @return DSL filters and DSL queries from a list of criteria and add it to a NOR query + * @throws MailboxException + */ + private static ElasticsearchQueryRepresentation convertNor(List subCriteria) throws MailboxException { + List dslQueries = new ArrayList(); + List dslFilters = new ArrayList(); + buildListsFromCriteria(subCriteria, dslQueries, dslFilters); + DSLQuery finalQuery = buildQueryFromListForNor(dslQueries); + DSLFilter finalFilter = getFilterFromListForNor(dslFilters); + return new ElasticsearchQueryRepresentation(finalQuery, finalFilter); + } + + /** + * Make DSL filters and DSL queries from a list of criteria and add it to a OR query + * + * @param subCriteria Sub criteria + * @return DSL filters and DSL queries from a list of criteria and add it to a OR query + * @throws MailboxException + */ + private static ElasticsearchQueryRepresentation convertOr(List subCriteria) throws MailboxException { + List dslQueries = new ArrayList(); + List dslFilters = new ArrayList(); + buildListsFromCriteria(subCriteria, dslQueries, dslFilters); + DSLQuery finalQuery = buildQueryFromList(dslQueries, new DSLShouldQuery()); + DSLFilter finalFilter = getFilterFromList(dslFilters, new DSLOrFilter()); + return new ElasticsearchQueryRepresentation(finalQuery, finalFilter); + } + + /** + * Make DSL filters and DSL queries from a list of criteria and add it to a AND query + * + * @param subCriteria Sub criteria + * @return DSL filters and DSL queries from a list of criteria and add it to a AND query + * @throws MailboxException + */ + private static ElasticsearchQueryRepresentation convertAnd(List subCriteria) throws MailboxException { + List dslQueries = new ArrayList(); + List dslFilters = new ArrayList(); + buildListsFromCriteria(subCriteria, dslQueries, dslFilters); + DSLQuery finalQuery = buildQueryFromList(dslQueries, new DSLMustQuery()); + DSLFilter finalFilter = getFilterFromList(dslFilters, new DSLAndFilter()); + return new ElasticsearchQueryRepresentation(finalQuery, finalFilter); + } + + /** + * Add filters to a not if needed + * + * @param dslFilters filters to add + * @return Resulting filter + */ + private static DSLFilter getFilterFromListForNor(List dslFilters) { + if(dslFilters.size() == 0) { + return null; + } else if(dslFilters.size() == 1) { + return new DSLNotFilter( dslFilters.get(0) ); + } else { + DSLBooleanOperationFilter andFilter = new DSLAndFilter(); + for(DSLFilter dslFilter : dslFilters) { + andFilter.addFilter(new DSLNotFilter(dslFilter)); + } + return andFilter; + } + } + + /** + * Add filters to the DSLBoolFilter if needed + * + * @param dslFilters filters to add + * @param boolFilter DSLBooleanOperationFilter + * @return Resulting filter + */ + private static DSLFilter getFilterFromList(List dslFilters, DSLBooleanOperationFilter boolFilter) { + if(dslFilters.size() == 0) { + return null; + } else if(dslFilters.size() == 1) { + return dslFilters.get(0); + } else { + for(DSLFilter dslFilter : dslFilters) { + boolFilter.addFilter(new DSLNotFilter(dslFilter)); + } + return boolFilter; + } + } + + /** + * Add filters to the DSLBoolFilter if needed + * + * @param dslQueries queries to add + * @param boolQuery DSLSpecializedBoolQuery + * @return Resulting query + */ + private static DSLQuery buildQueryFromList(List dslQueries, DSLSpecializedBoolQuery boolQuery) { + if(dslQueries.size() == 0) { + return null; + } else if(dslQueries.size() == 1) { + return dslQueries.get(0); + } else { + for(DSLQuery dslQuery : dslQueries) { + boolQuery.addQuery(dslQuery); + } + return boolQuery; + } + } + + /** + * Add queries to a not if needed + * + * @param dslQueries filters to add + * @return Resulting filter + */ + private static DSLQuery buildQueryFromListForNor(List dslQueries) { + if(dslQueries.size() == 0) { + return null; + } else if(dslQueries.size() == 1) { + DSLMustNotQuery dslMustNotQuery = new DSLMustNotQuery(); + dslMustNotQuery.addQuery(dslQueries.get(0)); + return dslMustNotQuery; + } else { + DSLMustNotQuery dslMustNotQuery = new DSLMustNotQuery(); + for(DSLQuery dslQuery : dslQueries) { + dslMustNotQuery.addQuery(dslQuery); + } + return dslMustNotQuery; + } + } + + + /** + * It converts sub criteria to Filters and Queries and add these Filters and Queries to the appropriate list + * + * @param subCriteria Subcriteria to convert + * @param dslQueries Queries to fill + * @param dslFilters Filters to fill + * @throws MailboxException + */ + private static void buildListsFromCriteria(List subCriteria, List dslQueries, List dslFilters) throws MailboxException { + for(SearchQuery.Criterion subCriterion :subCriteria) { + ElasticsearchQueryRepresentation tempRepresentation = generateQueryBuilder(subCriterion); + if(tempRepresentation.query != null) { + dslQueries.add(tempRepresentation.query); + } + if(tempRepresentation.filter != null) { + dslFilters.add(tempRepresentation.filter); + } + } + } + + private static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery.TextCriterion textCriterion) { + switch (textCriterion.getType()) { + case BODY: + return new ElasticsearchQueryRepresentation( + new DSLMatchQuery(MessageConstants.BODY_CONTENT, textCriterion.getOperator().getValue()), + null); + case FULL: + return new ElasticsearchQueryRepresentation( + new DSLMatchQuery(MessageConstants.FULL_CONTENT, textCriterion.getOperator().getValue()), + null); + default: + throw new RuntimeException("Text criterion Type used was not defined"); + } + } + + private static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery.ModSeqCriterion modSeqCriterion) { + return manageNumericOperator(MessageConstants.MODSEQ, modSeqCriterion.getOperator()); + } + + private static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery.SizeCriterion sizeCriterion) { + return manageNumericOperator(MessageConstants.FULL_CONTENT_OCTET, sizeCriterion.getOperator()); + } + + /** + * Return the most suited filter for a numeric operator + * + * @param fieldName field we want to apply the operator to + * @param numericOperator Operator + * @return representation associated with this filter + */ + private static ElasticsearchQueryRepresentation manageNumericOperator(String fieldName, SearchQuery.NumericOperator numericOperator) { + long value = numericOperator.getValue(); + SearchQuery.NumericComparator comparator = numericOperator.getType(); + return applyComparator(fieldName, value, comparator); + } + + /** + * Generate the appropriated DSL filter given an comparator + * + * If the comparator is an equality, then a term filter is what we need. In other cases we need a range filter + * + * @param fieldName Field we want to filter + * @param value value used by the numeric comparator + * @param comparator The numeric comparator + * @return representation associated with this filter + */ + private static ElasticsearchQueryRepresentation applyComparator(String fieldName, long value, SearchQuery.NumericComparator comparator) { + switch (comparator) { + case EQUALS: + return new ElasticsearchQueryRepresentation( null, + new DSLTermFilter(fieldName).numberFilter(value) + ); + case GREATER_THAN: + return new ElasticsearchQueryRepresentation( null, + new DSLRangeFilter(fieldName).gte(value) + ); + case LESS_THAN: + return new ElasticsearchQueryRepresentation(null, + new DSLRangeFilter(fieldName).lte(value) + ); + default: + throw new RuntimeException("A non existing numeric operator was triggered"); + } + } + + private static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery.CustomFlagCriterion customFlagCriterion) throws MailboxException { + String customFlag = customFlagCriterion.getFlag(); + return new ElasticsearchQueryRepresentation(null, + new DSLTermFilter(MessageConstants.USER_FLAGS).stringFilter(customFlag)); + } + + private static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery.FlagCriterion flagCriterion) throws MailboxException{ + SearchQuery.BooleanOperator operator = flagCriterion.getOperator(); + Flags.Flag flag = flagCriterion.getFlag(); + if(flag.equals(Flags.Flag.DELETED) ) { + return new ElasticsearchQueryRepresentation(null, + new DSLTermFilter(MessageConstants.FLAG_DELETED).boolFilter(operator.isSet())); + } + if(flag.equals(Flags.Flag.ANSWERED) ) { + return new ElasticsearchQueryRepresentation(null, + new DSLTermFilter(MessageConstants.FLAG_ANSWERED).boolFilter(operator.isSet())); + } + if(flag.equals(Flags.Flag.DRAFT) ) { + return new ElasticsearchQueryRepresentation(null, + new DSLTermFilter(MessageConstants.FLAG_DRAFT).boolFilter(operator.isSet())); + } + if(flag.equals(Flags.Flag.SEEN) ) { + return new ElasticsearchQueryRepresentation(null, + new DSLTermFilter(MessageConstants.FLAG_SEEN).boolFilter(operator.isSet())); + } + if(flag.equals(Flags.Flag.RECENT) ) { + return new ElasticsearchQueryRepresentation(null, + new DSLTermFilter(MessageConstants.FLAG_RECENT).boolFilter(operator.isSet())); + } + if(flag.equals(Flags.Flag.FLAGGED) ) { + return new ElasticsearchQueryRepresentation(null, + new DSLTermFilter(MessageConstants.FLAG_FLAGGED).boolFilter(operator.isSet())); + } + throw new UnsupportedSearchException(); + } + + private static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery.HeaderCriterion headerCriterion) throws MailboxException { + SearchQuery.HeaderOperator operator = headerCriterion.getOperator(); + String headerName = headerCriterion.getHeaderName(); + if(SearchQuery.ExistsOperator.exists().equals(operator)) { + return new ElasticsearchQueryRepresentation(null, new DSLTermFilter(MessageConstants.HEADERS_NAME).stringFilter(headerName)); + } + if(operator instanceof SearchQuery.ContainsOperator) { + String value = ((SearchQuery.ContainsOperator) operator).getValue(); + if(JsonHeaderBuilder.isAddressField(headerName)) { + return manageAddressFields(headerName, value); + } else if(headerName.equalsIgnoreCase("subject")) { + return new ElasticsearchQueryRepresentation(new DSLMatchQuery(MessageConstants.SUBJECT, value), null); + } else if (headerName.equalsIgnoreCase("date")) { + return new ElasticsearchQueryRepresentation(new DSLMatchQuery(MessageConstants.SENT_DATE, value), null); + } else { + DSLMustQuery dslMustQuery = new DSLMustQuery(); + DSLQuery dslQuery1 = new DSLMatchQuery(MessageConstants.HEADERS+"."+ MessageConstants.HEADERS_NESTED_VALUE, value); + DSLQuery dslQuery2 = new DSLMatchQuery(MessageConstants.HEADERS+"."+ MessageConstants.HEADERS_NESTED_NAME, headerName); + + dslMustQuery.addQuery(dslQuery1); + dslMustQuery.addQuery(dslQuery2); + DSLNestedQuery dslNestedQuery = new DSLNestedQuery(MessageConstants.HEADERS, DSLNestedQuery.Score_mode.MAX, dslMustQuery); + + return new ElasticsearchQueryRepresentation(dslNestedQuery, null); + } + } + if(operator instanceof SearchQuery.AddressOperator) { + String value = ((SearchQuery.AddressOperator) operator).getAddress(); + return manageAddressFields(headerName, value); + } + throw new UnsupportedSearchException(); + } + + private static ElasticsearchQueryRepresentation manageAddressFields(String headerName, String value) { + if(headerName.equalsIgnoreCase("to")) { + DSLShouldQuery shouldQuery = new DSLShouldQuery(); + shouldQuery.addQuery(new DSLMatchQuery(MessageConstants.TO, value)); + shouldQuery.addQuery(new DSLMatchQuery(MessageConstants.DISPLAY_TO, value)); + return new ElasticsearchQueryRepresentation(shouldQuery, null); + } else if(headerName.equalsIgnoreCase("cc")) { + return new ElasticsearchQueryRepresentation(new DSLMatchQuery(MessageConstants.CC, value),null); + } else if(headerName.equalsIgnoreCase("bcc")) { + return new ElasticsearchQueryRepresentation(new DSLMatchQuery(MessageConstants.BCC,value), null); + } else if(headerName.equalsIgnoreCase("from")) { + DSLShouldQuery shouldQuery = new DSLShouldQuery(); + shouldQuery.addQuery(new DSLMatchQuery(MessageConstants.FROM, value)); + shouldQuery.addQuery(new DSLMatchQuery(MessageConstants.DISPLAY_FROM, value)); + return new ElasticsearchQueryRepresentation(shouldQuery, null); + } + return new ElasticsearchQueryRepresentation(null, null); + } + + private static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery.UidCriterion uidCriterion) throws MailboxException { + SearchQuery.InOperator inOperator= uidCriterion.getOperator(); + SearchQuery.NumericRange[] numericRanges = inOperator.getRange(); + int nbRanges = numericRanges.length; + if(nbRanges < 1 ) { + return new ElasticsearchQueryRepresentation(null, null); + } + if(nbRanges == 1) { + return new ElasticsearchQueryRepresentation( + null, + new DSLRangeFilter(MessageConstants.UID) + .lte(numericRanges[0].getHighValue()) + .gte(numericRanges[0].getLowValue()) + ); + } + DSLAndFilter andFilter = new DSLAndFilter(); + for(SearchQuery.NumericRange numericRange : numericRanges) { + andFilter.addFilter( + new DSLRangeFilter(MessageConstants.UID) + .lte(numericRange.getHighValue()) + .gte(numericRange.getLowValue()) + ); + } + return new ElasticsearchQueryRepresentation(null, andFilter); + } + + private static ElasticsearchQueryRepresentation generateQueryBuilder(SearchQuery.InternalDateCriterion dateCriterion) throws MailboxException { + SearchQuery.DateOperator dateOperator = dateCriterion.getOperator(); + SearchQuery.DateResolution dateResolution = dateOperator.getDateResultion(); + Date date = dateOperator.getDate(); + SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + String dateString = format.format(date); + String lowDateString = format.format( DateResolutionFormater.calculateLowerDate(date, dateResolution) ); + String upDateString = format.format( DateResolutionFormater.calculateUpperDate(date, dateResolution)); + String roundDateString = format.format(DateResolutionFormater.calculateRoundedDate(date, dateResolution)); + switch (dateOperator.getType()) { + case BEFORE: + return new ElasticsearchQueryRepresentation(null, new DSLRangeFilter(MessageConstants.DATE).lte(upDateString)); + case AFTER: + return new ElasticsearchQueryRepresentation(null, new DSLRangeFilter(MessageConstants.DATE).gte(lowDateString)); + case ON: + if(dateResolution == SearchQuery.DateResolution.Second) { + return new ElasticsearchQueryRepresentation(null, new DSLTermFilter(MessageConstants.DATE).stringFilter(dateString)); + } + return new ElasticsearchQueryRepresentation(null, new DSLRangeFilter(MessageConstants.DATE).lte(upDateString).gte(roundDateString)); + } + return new ElasticsearchQueryRepresentation(null, null); + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/SearchHandler.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/SearchHandler.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/SearchHandler.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/SearchHandler.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,128 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.query; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import io.searchbox.client.JestClient; +import io.searchbox.core.Search; +import io.searchbox.core.SearchResult; +import io.searchbox.core.search.sort.Sort; +import org.apache.james.mailbox.elasticsearch.search.dsl.DSLBuilder; +import org.apache.james.mailbox.store.json.model.MessageConstants; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.exception.UnavailableIndexException; +import org.apache.james.mailbox.model.SearchQuery; +import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Object that handles a search query + */ +public class SearchHandler { + + private JestClient jestClient; + + private static final Logger LOG = LoggerFactory.getLogger(SearchHandler.class); + + + public SearchHandler(JestClient jestClient) { + this.jestClient = jestClient; + } + + public Iterator search( final Mailbox mailbox, SearchQuery searchQuery) throws MailboxException { + QueryConverter.ElasticsearchQueryRepresentation representation = QueryConverter + .generateQueryBuilder(searchQuery, mailbox.getMailboxId().toString()); + List sorts = SortConverter.generateSorts(searchQuery); + Search search = buildSearch(representation, sorts); + JsonObject searchResult = getSearchResult(search).getJsonObject(); + return getIdsFromSearchResult(searchResult).iterator(); + } + + private Search buildSearch(QueryConverter.ElasticsearchQueryRepresentation representation, List sorts) { + String searchQueryString = new DSLBuilder() + .setQuery(representation.query) + .setFilter(representation.filter) + .build() + .toString(); + return new Search.Builder(searchQueryString) + .addIndex(MessageConstants.SEARCH_INDEX_NAME) + .addType(MessageConstants.SEARCH_TYPE_NAME) + .addSort(sorts) + .build(); + } + + private SearchResult getSearchResult(Search search) throws UnavailableIndexException { + try { + return jestClient.execute(search); + } catch(Exception exception) { + throw new UnavailableIndexException("Error execution search on elastic search", exception); + } + } + + private List getIdsFromSearchResult(JsonObject searchResult) throws MailboxException { + List result = new ArrayList(); + JsonArray hits = getHits(searchResult); + for (JsonElement searchResultEntry : hits) { + try { + result.add(getIdFromHit(searchResultEntry)); + } catch (Exception exception) { + LOG.error("Malformed JSON", exception); + } + } + return result; + } + + private JsonArray getHits(JsonObject searchResult) throws UnavailableIndexException { + JsonElement hitsElement = searchResult.get("hits"); + if (!hitsElement.isJsonObject()) { + throw new UnavailableIndexException("hits is not a json object while parsing search result"); + } + JsonObject hitsObject = hitsElement.getAsJsonObject(); + hitsElement = hitsObject.get("hits"); + if (!hitsElement.isJsonArray()) { + throw new UnavailableIndexException("Can not get hits array while parsing search result"); + } + return hitsElement.getAsJsonArray(); + } + + private Long getIdFromHit(JsonElement searchResultEntry) throws Exception { + if (!searchResultEntry.isJsonObject()) { + throw new Exception("Can not get search entry as json object. Skipping search result"); + } + JsonObject searchResultEntryObject = searchResultEntry.getAsJsonObject(); + JsonElement sources = searchResultEntryObject.get("_source"); + if (sources == null || !sources.isJsonObject()) { + throw new Exception("Can not retrieve sources from search result. Skipping this search result"); + } + JsonObject sourcesObject = sources.getAsJsonObject(); + JsonElement idElement = sourcesObject.get(MessageConstants.UID); + if (idElement == null || !idElement.isJsonPrimitive()) { + throw new Exception("This search entry does not conform to the mapping. Skipping."); + } + return idElement.getAsLong(); + } + +} diff -uNr james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/SortConverter.java james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/SortConverter.java --- james-mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/SortConverter.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/query/SortConverter.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,105 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.query; + +import io.searchbox.core.search.sort.Sort; +import org.apache.james.mailbox.store.json.model.MessageConstants; +import org.apache.james.mailbox.model.SearchQuery; + +import java.util.ArrayList; +import java.util.List; + +/** + * Convert James sort into Jest sort + */ +public class SortConverter { + + /** + * Convert a search query sort list into a Jest sort list + * + * @param searchQuery Search query search list we want to convert + * @return Jest sort list + */ + public static List generateSorts(SearchQuery searchQuery) { + List originSorts = searchQuery.getSorts(); + List result = new ArrayList(); + for(SearchQuery.Sort sort : originSorts) { + result.add( generateSort(sort) ); + } + return result; + } + + /** + * Convert a search query sort into a Jest sort + * + * @param sort Search query sort + * @return Jest sort + */ + private static Sort generateSort(SearchQuery.Sort sort) { + return new Sort(getFieldFromClause(sort.getSortClause()), getOrder(sort)); + } + + /** + * Get the document field to sort on, given a searchQuery sort clause + * + * @param clause Clause from the sort of the search query + * @return Document field we shall use for sorting + */ + private static String getFieldFromClause(SearchQuery.Sort.SortClause clause) { + switch (clause) { + case Arrival : + return MessageConstants.ARRIVAL; + case MailboxCc : + return MessageConstants.CC; + case MailboxFrom : + return MessageConstants.FROM; + case MailboxTo : + return MessageConstants.TO; + case BaseSubject : + return MessageConstants.SUBJECT; + case Size : + return MessageConstants.FULL_CONTENT_OCTET; + case SentDate : + return MessageConstants.SENT_DATE; + case DisplayFrom : + return MessageConstants.DISPLAY_FROM; + case DisplayTo : + return MessageConstants.DISPLAY_TO; + case Uid : + return MessageConstants.UID; + default: + return null; + } + } + + /** + * Get order associated with the Sort + * + * @param sort Search query sort we want to convert order + * @return Ascendant or descendant sort + */ + private static Sort.Sorting getOrder(SearchQuery.Sort sort) { + if( sort.isReverse() ) { + return Sort.Sorting.DESC; + } else { + return Sort.Sorting.ASC; + } + } + +} diff -uNr james-mailbox/elasticsearch/src/main/resources/META-INF/spring/mailbox-index-elasticsearch.xml james-mailbox-elastic-search/elasticsearch/src/main/resources/META-INF/spring/mailbox-index-elasticsearch.xml --- james-mailbox/elasticsearch/src/main/resources/META-INF/spring/mailbox-index-elasticsearch.xml 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/resources/META-INF/spring/mailbox-index-elasticsearch.xml 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff -uNr james-mailbox/elasticsearch/src/main/resources/log4j.properties james-mailbox-elastic-search/elasticsearch/src/main/resources/log4j.properties --- james-mailbox/elasticsearch/src/main/resources/log4j.properties 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/main/resources/log4j.properties 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,8 @@ +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff -uNr james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/DateResolutionFormaterTest.java james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/DateResolutionFormaterTest.java --- james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/DateResolutionFormaterTest.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/DateResolutionFormaterTest.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,92 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search; + +import org.apache.james.mailbox.elasticsearch.search.query.DateResolutionFormater; +import org.apache.james.mailbox.model.SearchQuery; +import org.junit.Test; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +import static org.junit.Assert.assertEquals; + +/** + * Test for DateFormater + */ +public class DateResolutionFormaterTest { + + private final String dateString = "2014/01/02 15:15:15"; + private final SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + + @Test + public void testSecondResolution() throws ParseException{ + Date date = format.parse(dateString); + Date newUpperDate = DateResolutionFormater.calculateUpperDate(date, SearchQuery.DateResolution.Second); + assertEquals(dateString, format.format(newUpperDate)); + Date newLowerDate = DateResolutionFormater.calculateLowerDate(date, SearchQuery.DateResolution.Second); + assertEquals(dateString, format.format(newLowerDate)); + } + + @Test + public void testMinuteResolution() throws ParseException{ + Date date = format.parse(dateString); + Date newUpperDate = DateResolutionFormater.calculateUpperDate(date, SearchQuery.DateResolution.Minute); + assertEquals("2014/01/02 15:16:00", format.format(newUpperDate)); + Date newLowerDate = DateResolutionFormater.calculateLowerDate(date, SearchQuery.DateResolution.Minute); + assertEquals("2014/01/02 15:14:00", format.format(newLowerDate)); + } + + @Test + public void testHourResolution() throws ParseException{ + Date date = format.parse(dateString); + Date newUpperDate = DateResolutionFormater.calculateUpperDate(date, SearchQuery.DateResolution.Hour); + assertEquals("2014/01/02 16:00:00", format.format(newUpperDate)); + Date newLowerDate = DateResolutionFormater.calculateLowerDate(date, SearchQuery.DateResolution.Hour); + assertEquals("2014/01/02 14:00:00", format.format(newLowerDate)); + } + + @Test + public void testDayResolution() throws ParseException{ + Date date = format.parse(dateString); + Date newUpperDate = DateResolutionFormater.calculateUpperDate(date, SearchQuery.DateResolution.Day); + assertEquals("2014/01/03 00:00:00", format.format(newUpperDate)); + Date newLowerDate = DateResolutionFormater.calculateLowerDate(date, SearchQuery.DateResolution.Day); + assertEquals("2014/01/01 00:00:00", format.format(newLowerDate)); + } + + @Test + public void testMonthResolution() throws ParseException{ + Date date = format.parse(dateString); + Date newUpperDate = DateResolutionFormater.calculateUpperDate(date, SearchQuery.DateResolution.Month); + assertEquals("2014/02/01 00:00:00", format.format(newUpperDate)); + Date newLowerDate = DateResolutionFormater.calculateLowerDate(date, SearchQuery.DateResolution.Month); + assertEquals("2013/12/01 00:00:00", format.format(newLowerDate)); + } + + @Test + public void testYearResolution() throws ParseException{ + Date date = format.parse(dateString); + Date newUpperDate = DateResolutionFormater.calculateUpperDate(date, SearchQuery.DateResolution.Year); + assertEquals("2015/01/01 00:00:00", format.format(newUpperDate)); + Date newLowerDate = DateResolutionFormater.calculateLowerDate(date, SearchQuery.DateResolution.Year); + assertEquals("2013/01/01 00:00:00", format.format(newLowerDate)); + } +} diff -uNr james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/QueryConverterTest.java james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/QueryConverterTest.java --- james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/QueryConverterTest.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/QueryConverterTest.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,194 @@ +package org.apache.james.mailbox.elasticsearch.search; + +import org.apache.james.mailbox.elasticsearch.search.dsl.DSLBuilder; +import org.apache.james.mailbox.elasticsearch.search.query.QueryConverter; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.model.SearchQuery; +import org.junit.Test; + +import javax.mail.Flags; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Locale; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; + +/** + * Tests for query builder + */ +public class QueryConverterTest { + + private final String dateString = "2014-11-26 23"; + + @Test + public void testTextCriterion() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.bodyContains("Hello world")); + searchQuery.andCriteria(SearchQuery.mailContains("some value")); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"bool\":{\"must\":[{\"match\":{\"body-content\":\"Hello world\"}},{\"match\":{\"full-content\":\"some value\"}}]}}}", + queryString); + } + + @Test + public void testAllCriterion() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.all()); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"match_all\":{}}}", queryString); + } + + @Test + public void testOrCriterion() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.or(SearchQuery.bodyContains("Hello world"), SearchQuery.mailContains("some value"))); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"bool\":{\"should\":[{\"match\":{\"body-content\":\"Hello world\"}},{\"match\":{\"full-content\":\"some value\"}}]}}}", queryString); + } + + @Test + public void testNorCriterion() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.not(SearchQuery.bodyContains("Hello world"))); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"bool\":{\"must_not\":[{\"match\":{\"body-content\":\"Hello world\"}}]}}}", queryString); + } + + @Test + public void testModSeqEqualsCriterion() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.modSeqEquals(2048)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"term\":{\"modseq\":2048}}}}}", queryString); + } + + @Test + public void testModSeqGTCriterion() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.modSeqGreaterThan(2048)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"range\":{\"modseq\":{\"gte\":2048}}}}}}", queryString); + } + + @Test + public void testModSeqLTCriterion() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.modSeqLessThan(2048)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"range\":{\"modseq\":{\"lte\":2048}}}}}}", queryString); + } + + @Test + public void testInternalDateCriterionAfter() throws MailboxException, ParseException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.internalDateAfter(new SimpleDateFormat("yyyy-MM-dd HH", Locale.ENGLISH).parse(dateString), SearchQuery.DateResolution.Hour)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"range\":{\"date\":{\"gte\":\"2014/11/26 22:00:00\"}}}}}}", queryString); + } + + @Test + public void testInternalDateCriterionBefore() throws MailboxException, ParseException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.internalDateBefore(new SimpleDateFormat("yyyy-MM-dd HH", Locale.ENGLISH).parse(dateString), SearchQuery.DateResolution.Hour)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"range\":{\"date\":{\"lte\":\"2014/11/27 00:00:00\"}}}}}}", queryString); + } + + @Test + public void testInternalDateCriterionOn() throws MailboxException, ParseException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.internalDateOn(new SimpleDateFormat("yyyy-MM-dd HH", Locale.ENGLISH).parse(dateString), SearchQuery.DateResolution.Hour)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"range\":{\"date\":{\"gte\":\"2014/11/26 23:00:00\",\"lte\":\"2014/11/27 00:00:00\"}}}}}}", queryString); + } + + @Test + public void testUid() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + SearchQuery.NumericRange[] numericRanges = new SearchQuery.NumericRange[1]; + numericRanges[0] = new SearchQuery.NumericRange(42, 84); + searchQuery.andCriteria(SearchQuery.uid(numericRanges)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"range\":{\"uid\":{\"gte\":42,\"lte\":84}}}}}}", queryString); + } + + @Test + public void testAndFilter() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.or(SearchQuery.bodyContains("Hello world"), SearchQuery.modSeqEquals(2048))); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match\":{\"body-content\":\"Hello world\"}},\"filter\":{\"term\":{\"modseq\":2048}}}}}", queryString); + } + + @Test + public void testAnsweredFlag() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.flagIsSet(Flags.Flag.ANSWERED)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"term\":{\"flagAnswered\":true}}}}}", queryString); + } + + @Test + public void testDeletedFlag() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.flagIsSet(Flags.Flag.DELETED)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"term\":{\"flagDeleted\":true}}}}}", queryString); + } + + @Test + public void testDraftFlag() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.flagIsSet(Flags.Flag.DRAFT)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"term\":{\"flagDraft\":true}}}}}", queryString); + } + + @Test + public void testSeenFlag() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.flagIsUnSet(Flags.Flag.SEEN)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"term\":{\"flagSeen\":false}}}}}", queryString); + } + + @Test + public void testRecentFlag() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.flagIsSet(Flags.Flag.RECENT)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"term\":{\"flagRecent\":true}}}}}", queryString); + } + + @Test + public void testFlaggedFlag() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.flagIsUnSet(Flags.Flag.FLAGGED)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"term\":{\"flagFlagged\":false}}}}}", queryString); + } + + @Test + public void testSizeCriterion() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.sizeLessThan(2048)); + String queryString = buildQueryString(searchQuery); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"range\":{\"full-content-octet\":{\"lte\":2048}}}}}}", queryString); + } + + @Test + public void testMailboxCriterion() throws MailboxException { + UUID uuid = new UUID(42,13); + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.all()); + QueryConverter.ElasticsearchQueryRepresentation representation = QueryConverter.generateQueryBuilder(searchQuery, uuid.toString()); + String queryString = new DSLBuilder().setQuery(representation.query).setFilter(representation.filter).build().toString(); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"and\":[{\"term\":{\"mailbox-id\":\"00000000-0000-002a-0000-00000000000d\"}}]}}}}", queryString); + } + + private String buildQueryString(SearchQuery searchQuery) throws MailboxException { + QueryConverter.ElasticsearchQueryRepresentation representation = QueryConverter.generateQueryBuilder(searchQuery); + return new DSLBuilder().setQuery(representation.query).setFilter(representation.filter).build().toString(); + } +} diff -uNr james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/dsl/DSLBuilderTest.java james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/dsl/DSLBuilderTest.java --- james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/dsl/DSLBuilderTest.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/dsl/DSLBuilderTest.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,38 @@ +package org.apache.james.mailbox.elasticsearch.search.dsl; + +import org.apache.james.mailbox.elasticsearch.search.dsl.DSLBuilder; +import org.apache.james.mailbox.elasticsearch.search.dsl.filter.DSLRangeFilter; +import org.apache.james.mailbox.elasticsearch.search.dsl.query.DSLMatchQuery; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Some basic tests for DSLBuilder + */ +public class DSLBuilderTest { + + @Test + public void testDefaultBuilder() { + DSLBuilder dslBuilder = new DSLBuilder(); + assertEquals(null, dslBuilder.build()); + } + + @Test + public void testFilterOnly() { + DSLBuilder dslBuilder = new DSLBuilder().setFilter(new DSLRangeFilter("visitors").lt(72)); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"range\":{\"visitors\":{\"lt\":72}}}}}}", dslBuilder.build().toString()); + } + + @Test + public void testQueryOnly() { + DSLBuilder dslBuilder = new DSLBuilder().setQuery(new DSLMatchQuery("user","benwa")); + assertEquals("{\"query\":{\"match\":{\"user\":\"benwa\"}}}", dslBuilder.build().toString()); + } + + @Test + public void testBuilder() { + DSLBuilder dslBuilder = new DSLBuilder().setQuery(new DSLMatchQuery("user","benwa")).setFilter(new DSLRangeFilter("visitors").lt(72)); + assertEquals("{\"query\":{\"filtered\":{\"query\":{\"match\":{\"user\":\"benwa\"}},\"filter\":{\"range\":{\"visitors\":{\"lt\":72}}}}}}", dslBuilder.build().toString()); + } +} diff -uNr james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLFilterTest.java james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLFilterTest.java --- james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLFilterTest.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/dsl/filter/DSLFilterTest.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,131 @@ +package org.apache.james.mailbox.elasticsearch.search.dsl.filter; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Basic test on DSLFilters + */ +public class DSLFilterTest { + + + @Test + public void TestRangeFilterLte() { + DSLFilter filter = new DSLRangeFilter("visitors").lte(1000); + assertEquals("{\"range\":{\"visitors\":{\"lte\":1000}}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestRangeFilterGte() { + DSLFilter filter = new DSLRangeFilter("visitors").gte(1000); + assertEquals("{\"range\":{\"visitors\":{\"gte\":1000}}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestRangeFilterStringLteGte() { + DSLFilter filter = new DSLRangeFilter("visitors").lte(1000).gte(50); + assertEquals("{\"range\":{\"visitors\":{\"gte\":50,\"lte\":1000}}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestRangeFilterStringLt() { + DSLFilter filter = new DSLRangeFilter("date").lt("2014-01-02"); + assertEquals("{\"range\":{\"date\":{\"lt\":\"2014-01-02\"}}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestRangeFilterStringGt() { + DSLFilter filter = new DSLRangeFilter("date").gt("2014-01-02"); + assertEquals("{\"range\":{\"date\":{\"gt\":\"2014-01-02\"}}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestRangeFilterStringLte() { + DSLFilter filter = new DSLRangeFilter("date").lte("2014-01-02"); + assertEquals("{\"range\":{\"date\":{\"lte\":\"2014-01-02\"}}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestRangeFilterStringGte() { + DSLFilter filter = new DSLRangeFilter("date").gte("2014-01-02"); + assertEquals("{\"range\":{\"date\":{\"gte\":\"2014-01-02\"}}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestRangeFilterLteGte() { + DSLFilter filter = new DSLRangeFilter("visitors").lte(1000).gte(50); + assertEquals("{\"range\":{\"visitors\":{\"gte\":50,\"lte\":1000}}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestRangeFilterLt() { + DSLFilter filter = new DSLRangeFilter("visitors").lt(1000); + assertEquals("{\"range\":{\"visitors\":{\"lt\":1000}}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestRangeFilterGt() { + DSLFilter filter = new DSLRangeFilter("visitors").gt(1000); + assertEquals("{\"range\":{\"visitors\":{\"gt\":1000}}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestTermFilterString() { + DSLFilter filter = new DSLTermFilter("user").stringFilter("benwa"); + assertEquals("{\"term\":{\"user\":\"benwa\"}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestTermFilterNumber() { + DSLFilter filter = new DSLTermFilter("id").numberFilter(1024); + assertEquals("{\"term\":{\"id\":1024}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestTermFilterBoolean() { + DSLFilter filter = new DSLTermFilter("flag").boolFilter(false); + assertEquals("{\"term\":{\"flag\":false}}", filter.getQueryAsJson().toString()); + } + + @Test + public void TestTermFilterStringWithoutCache() { + DSLFilter filter = new DSLTermFilter("user", false).stringFilter("benwa"); + assertEquals("{\"term\":{\"user\":\"benwa\"},\"_cache\":false}", filter.getQueryAsJson().toString()); + } + + @Test + public void testAndFilter() { + DSLFilter subFilter1 = new DSLTermFilter("flag").boolFilter(false); + DSLFilter subFilter2 = new DSLRangeFilter("visitors").gte(50); + DSLAndFilter filter = new DSLAndFilter(); + filter.addFilter(subFilter1); + filter.addFilter(subFilter2); + assertEquals("{\"and\":[{\"term\":{\"flag\":false}},{\"range\":{\"visitors\":{\"gte\":50}}}]}", filter.getQueryAsJson().toString()); + } + + @Test + public void testOrFilter() { + DSLFilter subFilter1 = new DSLTermFilter("flag").boolFilter(false); + DSLFilter subFilter2 = new DSLRangeFilter("visitors").gte(50); + DSLOrFilter filter = new DSLOrFilter(); + filter.addFilter(subFilter1); + filter.addFilter(subFilter2); + assertEquals("{\"or\":[{\"term\":{\"flag\":false}},{\"range\":{\"visitors\":{\"gte\":50}}}]}", filter.getQueryAsJson().toString()); + } + + @Test + public void testNotFilter() { + DSLFilter subFilter = new DSLTermFilter("flag").boolFilter(false); + DSLFilter filter = new DSLNotFilter(subFilter); + assertEquals("{\"not\":{\"term\":{\"flag\":false}}}", filter.getQueryAsJson().toString()); + } + + @Test + public void testNestedFilter() { + DSLFilter subFilter = new DSLTermFilter("blog.user").stringFilter("benwa"); + DSLNestedFilter nestedFilter = new DSLNestedFilter("blog", subFilter); + assertEquals("{\"nested\":{\"path\":\"blog\",\"filter\":{\"term\":{\"blog.user\":\"benwa\"}}}}", nestedFilter.getQueryAsJson().toString()); + } + +} diff -uNr james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLQueriesTest.java james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLQueriesTest.java --- james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLQueriesTest.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/dsl/query/DSLQueriesTest.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,61 @@ +package org.apache.james.mailbox.elasticsearch.search.dsl.query; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Basic tests on DSLQueries + */ +public class DSLQueriesTest { + + @Test + public void testMatchAll() { + DSLQuery query = new DSLMatchAllQuery(); + assertEquals( "{\"match_all\":{}}", query.getQueryAsJson().toString() ); + } + + @Test + public void testMatch() { + DSLQuery query = new DSLMatchQuery("user", "kimchi"); + assertEquals("{\"match\":{\"user\":\"kimchi\"}}", query.getQueryAsJson().toString() ); + } + + @Test + public void testMust() { + DSLQuery subQuery1 = new DSLMatchQuery("user", "kimchi"); + DSLQuery subQuery2 = new DSLMatchQuery("job", "intern"); + DSLMustQuery query = new DSLMustQuery(); + query.addQuery(subQuery1); + query.addQuery(subQuery2); + assertEquals("{\"bool\":{\"must\":[{\"match\":{\"user\":\"kimchi\"}},{\"match\":{\"job\":\"intern\"}}]}}", query.getQueryAsJson().toString()); + } + + @Test + public void testShould() { + DSLQuery subQuery1 = new DSLMatchQuery("user", "kimchi"); + DSLQuery subQuery2 = new DSLMatchQuery("job", "intern"); + DSLShouldQuery query = new DSLShouldQuery(); + query.addQuery(subQuery1); + query.addQuery(subQuery2); + assertEquals("{\"bool\":{\"should\":[{\"match\":{\"user\":\"kimchi\"}},{\"match\":{\"job\":\"intern\"}}]}}", query.getQueryAsJson().toString()); + } + + @Test + public void testMustNot() { + DSLQuery subQuery1 = new DSLMatchQuery("user", "kimchi"); + DSLQuery subQuery2 = new DSLMatchQuery("job", "intern"); + DSLMustNotQuery query = new DSLMustNotQuery(); + query.addQuery(subQuery1); + query.addQuery(subQuery2); + assertEquals("{\"bool\":{\"must_not\":[{\"match\":{\"user\":\"kimchi\"}},{\"match\":{\"job\":\"intern\"}}]}}", query.getQueryAsJson().toString()); + } + + @Test + public void testNestedQuery() { + DSLQuery subQuery = new DSLMatchQuery("nested.user", "kimchi"); + DSLQuery query = new DSLNestedQuery("nested", DSLNestedQuery.Score_mode.SUM, subQuery); + assertEquals("{\"nested\":{\"path\":\"nested\",\"score_mode\":\"sum\",\"query\":{\"match\":{\"nested.user\":\"kimchi\"}}}}", query.getQueryAsJson().toString()); + } + +} diff -uNr james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/DirectMessageSearchIndexTest.java james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/DirectMessageSearchIndexTest.java --- james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/DirectMessageSearchIndexTest.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/DirectMessageSearchIndexTest.java 2015-04-13 20:20:46.057783371 +0200 @@ -0,0 +1,48 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + + + + // We can not launch an embedded elastic search because of a conflicting Lucene version used in lucene sub project. + // Uncomment if you have a standalone ElasticSearch + +/* + +package org.apache.james.mailbox.elasticsearch.search.index; + +import org.apache.james.mailbox.store.MailboxSessionMapperFactory; +import org.apache.james.mailbox.store.search.extractor.DefaultTextExtractorFactory; + +public class DirectMessageSearchIndexTest extends ElasticSearchMessageSearchIndexTest { + + protected void indexSetUp(MailboxSessionMapperFactory factory) throws Exception { + searchIndex = new DirectElasticsearchMessageSearchIndex(factory); + searchIndex.setHostString("http://127.0.0.1:9200"); + searchIndex.setTextExtractorFactory(new DefaultTextExtractorFactory("UTF-8")); + searchIndex.setShardsNb(1); + searchIndex.setReplicaNb(0); + searchIndex.afterPropertiesSet(); + } + + protected void sleepTimeAfterIndexSetUp() throws Exception { + sleepTimeAfterIndex = 1000; + } +} + +*/ diff -uNr james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/ElasticSearchMessageSearchIndexTest.java james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/ElasticSearchMessageSearchIndexTest.java --- james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/ElasticSearchMessageSearchIndexTest.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/ElasticSearchMessageSearchIndexTest.java 2015-04-13 20:20:18.247782168 +0200 @@ -0,0 +1,596 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.elasticsearch.search.index; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import io.searchbox.client.JestClient; +import io.searchbox.client.JestResult; +import io.searchbox.core.Get; +import org.apache.james.mailbox.MailboxSession; +import org.apache.james.mailbox.elasticsearch.search.JestClientBuilder; +import org.apache.james.mailbox.store.json.model.MailboxConstants; +import org.apache.james.mailbox.store.json.model.MessageConstants; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.inmemory.InMemoryMailboxSessionMapperFactory; +import org.apache.james.mailbox.model.MailboxPath; +import org.apache.james.mailbox.model.MessageRange; +import org.apache.james.mailbox.model.SearchQuery; +import org.apache.james.mailbox.store.MailboxSessionMapperFactory; +import org.apache.james.mailbox.store.SimpleMailboxSession; +import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder; +import org.apache.james.mailbox.store.mail.model.impl.SimpleMailbox; +import org.apache.james.mailbox.store.mail.model.impl.SimpleMessage; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.mail.Flags; +import javax.mail.internet.SharedInputStream; +import javax.mail.util.SharedByteArrayInputStream; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Iterator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * + * Tests for elastic search direct client + * While running this test, you should provide an elastic search instance. + * + * We can not run an embedded ElasticSearch because of a difference in Apache Lucene version, so that's why you should provide a standalone version. + */ +public abstract class ElasticSearchMessageSearchIndexTest { + + protected JestClient jestClient; + protected AbstractElasticsearchMessageSearchIndex searchIndex; + private List> messages; + private List> mailboxes; + protected long sleepTimeAfterIndex; + + MailboxSessionMapperFactory factory; + MailboxSession mailboxSession; + private static final Logger LOG = LoggerFactory.getLogger(ElasticSearchMessageSearchIndexTest.class); + + + @Before + public void setUp() throws Exception { + messages = new ArrayList>(); + mailboxes = new ArrayList>(); + factory = new InMemoryMailboxSessionMapperFactory(); + mailboxSession = new SimpleMailboxSession(42, "benwa", "toto", LOG, new ArrayList(), '.', MailboxSession.SessionType.User); + messages = new ArrayList>(); + mailboxes = new ArrayList>(); + indexSetUp(factory); + jestClient = searchIndex.jestClient; + sleepTimeAfterIndexSetUp(); + initializeMailboxes(); + sleepTimeAfterIndexSetUp(); + initializeMessages(); + putMessages(); + } + + protected abstract void indexSetUp(MailboxSessionMapperFactory factory) throws Exception; + + protected abstract void sleepTimeAfterIndexSetUp() throws Exception; + + private void putMessages() throws Exception { + searchIndex.add(null, mailboxes.get(0), messages.get(0)); + searchIndex.add(null, mailboxes.get(0), messages.get(1)); + searchIndex.add(null, mailboxes.get(0), messages.get(2)); + searchIndex.add(null, mailboxes.get(1), messages.get(3)); + sleepAfterIndex(); + } + + private void initializeMailboxes() throws MailboxException { + addMailbox(new MailboxPath("test", "test_user", "name"), 43, 42); + addMailbox(new MailboxPath("test2", "test_user2", "name2"), 430, 420); + } + + private void addMailbox(MailboxPath mailboxPath, long uid, long uidValidity) throws MailboxException { + SimpleMailbox mailbox = new SimpleMailbox(mailboxPath, uidValidity); + mailbox.setMailboxId(uid); + mailboxes.add(mailbox); + factory.createMailboxMapper(mailboxSession).save(mailbox); + searchIndex.addMailbox(mailbox); + } + + private void initializeMessages() throws ParseException, MailboxException { + initializeMessage("To: my*address@mydomain.com\nSubject: ***Renew Your Records***\nFrom: \"renew@USBank.com\"\nContent-Type: text/html\n\n", + "Hello world! \n", mailboxes.get(0), 0, 1, "2014/11/27 22:24:25"); + initializeMessage("To: benwa@mydomain.com\nSubject: ***MiNET Powaa***\nFrom: \"MiNET\"\nContent-Type: text/html\n\n", + "We are legion!\n", mailboxes.get(0), 1, 2, "2014/11/27 22:24:24"); + initializeMessage("Return-Path: \n" + + "Received: from mx1.minet.net (mx1.minet.net [192.168.102.25])\n" + + "\t by imap (Cyrus v2.4.16-Debian-2.4.16-4+deb7u1) with LMTPA;\n" + + "\t Thu, 27 Nov 2014 20:44:20 +0100\n" + + "X-Sieve: CMU Sieve 2.4\n" + + "Received: from localhost (spam.minet.net [192.168.102.97])\n" + + "\tby mx1.minet.net (Postfix) with ESMTP id 84BAD3270E0\n" + + "\tfor ; Thu, 27 Nov 2014 20:44:21 +0100 (CET)\n" + + "X-Virus-Scanned: by amavisd-new using ClamAV at minet.net\n" + + "X-Spam-Flag: NO\n" + + "X-Spam-Score: -1.5\n" + + "X-Spam-Level:\n" + + "X-Spam-Status: No, score=-1.5 required=1 tests=[BAYES_00=-1.5] autolearn=ham\n" + + "Received: from mx2.minet.net ([IPv6:::ffff:192.168.102.26])\n" + + "\tby localhost (spam.minet.net [::ffff:192.168.102.97]) (amavisd-new, port 10024)\n" + + "\twith ESMTP id F4LTgGMPhjD3 for ;\n" + + "\tThu, 27 Nov 2014 19:44:20 +0000 (UTC)\n" + + "Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=217.24.82.4; helo=cabale.usenet-fr.net; envelope-from=frnog-owner@frnog.org; receiver=benwa@minet.net \n" + + "Received: from cabale.usenet-fr.net (cabale.usenet-fr.net [217.24.82.4])\n" + + "\tby mx2.minet.net (Postfix) with ESMTP id 6AF179A8710\n" + + "\tfor ; Thu, 27 Nov 2014 20:44:18 +0100 (CET)\n" + + "Received: by cabale.usenet-fr.net (Postfix, from userid 90)\n" + + "\tid C480598A5E1F; Thu, 27 Nov 2014 20:44:18 +0100 (CET)\n" + + "X-Original-To: frnog-tech@frnog.org\n" + + "Delivered-To: frnog-tech@frnog.org\n" + + "Received: from newserver.arneill-py.local (arneill-py.sacramento.ca.us [50.1.8.254])\n" + + "\tby cabale.usenet-fr.net (Postfix) with ESMTP id DC7CE98A5AB9\n" + + "\tfor ; Thu, 27 Nov 2014 20:44:03 +0100 (CET)\n" + + "Received: from NEWSERVER.arneill-py.local ([fe80::55b0:14b8:c8bb:8cb3]) by\n" + + " newserver.arneill-py.local ([fe80::55b0:14b8:c8bb:8cb3%11]) with mapi id\n" + + " 14.03.0210.002; Thu, 27 Nov 2014 11:44:01 -0800\n" + + "From: Michel Py \n" + + "To: 'Stephane Bortzmeyer' , \"frnog-tech@frnog.org\"\n" + + "\t\n" + + "Thread-Topic: [FRnOG] [TECH] Cherche adresse IP trou noir public\n" + + "Thread-Index: AQHQCYNv2NKyXWrt70uw0CjxZBQx7Jx04QEA\n" + + "Date: Thu, 27 Nov 2014 19:44:00 +0000\n" + + "Message-ID: <58BE41EEDD170C47A3202FF6E169960B02C301CF@newserver.arneill-py.local>\n" + + "References: <20141126141413.GA18179@nic.fr>\n" + + "In-Reply-To: <20141126141413.GA18179@nic.fr>\n" + + "Accept-Language: en-US\n" + + "Content-Language: en-US\n" + + "X-MS-Has-Attach:\n" + + "X-MS-TNEF-Correlator:\n" + + "x-originating-ip: [fe80::55b0:14b8:c8bb:8cb3]\n" + + "Content-Type: text/plain; charset=\"utf-8\"\n" + + "Content-Transfer-Encoding: base64\n" + + "MIME-Version: 1.0\n" + + "Subject: RE: [FRnOG] [TECH] Cherche adresse IP trou noir public\n" + + "X-Loop: frnog@frnog.org\n" + + "X-Sequence: 2994\n" + + "Errors-to: frnog-owner@frnog.org\n" + + "Precedence: list\n" + + "Precedence: bulk\n" + + "Sender: frnog-request@frnog.org\n" + + "X-mailing-list: frnog@frnog.org\n" + + "List-Id: \n" + + "List-Archive: \n" + + "List-Help: \n" + + "List-Owner: \n" + + "List-Post: \n" + + "List-Subscribe: \n" + + "List-Unsubscribe: \n\n", + "Will you attend the meeting tomorrow? \n", mailboxes.get(0), 2, 3, "2014/11/27 22:23:25"); + initializeMessage("To: jean@dujardin.fr\nSubject: *I love your latest movie*\nFrom: \"Your better groupie\"\nContent-Type: text/html\n\n", + "I love you so much <3 \n", mailboxes.get(1), 3, 4, "2014/11/27 21:23:25"); + } + + private void initializeMessage(String headerContent, String bodyContent, Mailbox mailbox, long uid, long modseq, String date) throws ParseException, MailboxException { + int bodyStartOctet = headerContent.length(); + String messageContent = headerContent + bodyContent; + SharedInputStream sharedInputStream = new SharedByteArrayInputStream(messageContent.getBytes()); + SimpleMessage message = new SimpleMessage(new SimpleDateFormat("yyyy/MM/dd HH:mm:ss").parse(date), messageContent.length(), bodyStartOctet, + sharedInputStream, new Flags(), new PropertyBuilder(), mailbox.getMailboxId()); + message.setUid(uid); + message.setModSeq(modseq); + messages.add(message); + } + + @Test + public void testUpdate() throws Exception { + searchIndex.delete(mailboxSession, mailboxes.get(0), MessageRange.range(0, 2)); + factory.createMessageMapper(mailboxSession).add(mailboxes.get(0), messages.get(0)); + searchIndex.add(mailboxSession, mailboxes.get(0), messages.get(0)); + String messageId = calculateElasticSearchId(messages.get(0).getMailboxId(), messages.get(0).getUid()); + Flags flags = new Flags(); + flags.add(Flags.Flag.ANSWERED); + flags.add(Flags.Flag.DELETED); + flags.add(Flags.Flag.DRAFT); + flags.add(Flags.Flag.RECENT); + flags.add(Flags.Flag.SEEN); + flags.add(Flags.Flag.FLAGGED); + factory.createMessageMapper(mailboxSession).updateFlags(mailboxes.get(0), flags, true, true, MessageRange.one(messages.get(0).getUid())); + searchIndex.update(mailboxSession, mailboxes.get(0), MessageRange.one(messages.get(0).getUid()), flags); + sleepAfterIndex(); + JsonObject jsonResult = getJsonEntry(messageId, mailboxes.get(0)); + JsonElement sourceElement = jsonResult.get("_source"); + assertTrue(sourceElement.isJsonObject()); + JsonObject source = sourceElement.getAsJsonObject(); + assertTrue(source.get(MessageConstants.FLAG_ANSWERED).isJsonPrimitive()); + assertTrue(source.get(MessageConstants.FLAG_DELETED).isJsonPrimitive()); + assertTrue(source.get(MessageConstants.FLAG_DRAFT).isJsonPrimitive()); + assertTrue(source.get(MessageConstants.FLAG_FLAGGED).isJsonPrimitive()); + assertTrue(source.get(MessageConstants.FLAG_SEEN).isJsonPrimitive()); + assertTrue(source.get(MessageConstants.FLAG_RECENT).isJsonPrimitive()); + assertTrue(source.get(MessageConstants.FLAG_DRAFT).getAsBoolean()); + assertTrue(source.get(MessageConstants.FLAG_DELETED).getAsBoolean()); + assertTrue(source.get(MessageConstants.FLAG_SEEN).getAsBoolean()); + assertTrue(source.get(MessageConstants.FLAG_RECENT).getAsBoolean()); + assertTrue(source.get(MessageConstants.FLAG_ANSWERED).getAsBoolean()); + assertTrue(source.get(MessageConstants.FLAG_FLAGGED).getAsBoolean()); + } + + @Test + public void testSearchMatchAll() throws Exception { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.all()); + Iterator result = searchIndex.search(null, mailboxes.get(1), searchQuery ); + List resultList = getLongs(result); + assertFalse(resultList.contains(new Long(0))); + assertFalse(resultList.contains(new Long(1))); + assertFalse(resultList.contains(new Long(2))); + assertTrue(resultList.contains(new Long(3))); + } + + @Test + public void testSearchMatch() throws Exception { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.bodyContains("hello")); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery ); + List resultList = getLongs(result); + assertTrue(resultList.contains(new Long(0))); + assertFalse(resultList.contains(new Long(1))); + assertFalse(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + } + + @Test + public void testSearchMatchAndModSeq() throws Exception { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.and(SearchQuery.or(SearchQuery.bodyContains("hello"), SearchQuery.bodyContains("legion")), SearchQuery.sizeGreaterThan(42))); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery ); + List resultList = getLongs(result); + assertTrue(resultList.contains(new Long(0))); + assertTrue(resultList.contains(new Long(1))); + assertFalse(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + } + + @Test + public void testSearchSize() throws Exception { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.sizeLessThan(140)); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery ); + List resultList = getLongs(result); + assertFalse(resultList.contains(new Long(0))); + assertTrue(resultList.contains(new Long(1))); + assertFalse(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + } + + @Test + public void testSearchUid() throws Exception { + SearchQuery searchQuery = new SearchQuery(); + SearchQuery.NumericRange[] ranges = new SearchQuery.NumericRange[1]; + ranges[0] = new SearchQuery.NumericRange(0,1); + searchQuery.andCriteria(SearchQuery.uid(ranges)); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery ); + List resultList = getLongs(result); + assertTrue(resultList.contains(new Long(0))); + assertTrue(resultList.contains(new Long(1))); + assertFalse(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + + } + + @Test + public void testSearchNor() throws Exception { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.not(SearchQuery.sizeLessThan(140))); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery); + List resultList = getLongs(result); + assertTrue(resultList.contains(new Long(0))); + assertFalse(resultList.contains(new Long(1))); + assertTrue(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + + } + + @Test + public void testModSeq() throws Exception { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.modSeqLessThan(2)); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery ); + List resultList = getLongs(result); + assertTrue(resultList.contains(new Long(0))); + assertTrue(resultList.contains(new Long(1))); + assertFalse(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + } + + @Test + public void testDateSecond() throws Exception { + String dateString = "2014/11/27 22:24:25"; + SearchQuery.DateResolution dateResolution = SearchQuery.DateResolution.Second; + List resultList = getDateSearchResult(dateString, dateResolution); + assertTrue(resultList.contains(new Long(0))); + assertFalse(resultList.contains(new Long(1))); + assertFalse(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + } + + @Test + public void testDateMinute() throws Exception { + String dateString = "2014/11/27 22:24:25"; + SearchQuery.DateResolution dateResolution = SearchQuery.DateResolution.Minute; + List resultList = getDateSearchResult(dateString, dateResolution); + assertTrue(resultList.contains(new Long(0))); + assertTrue(resultList.contains(new Long(1))); + assertFalse(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + } + + @Test + public void testDateHour() throws Exception { + String dateString = "2014/11/27 22:24:25"; + SearchQuery.DateResolution dateResolution = SearchQuery.DateResolution.Hour; + List resultList = getDateSearchResult(dateString, dateResolution); + assertTrue(resultList.contains(new Long(0))); + assertTrue(resultList.contains(new Long(1))); + assertTrue(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + } + + private List getDateSearchResult(String dateString, SearchQuery.DateResolution resolution) throws ParseException, org.apache.james.mailbox.exception.MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.internalDateOn(new SimpleDateFormat("yyyy/MM/dd HH:mm:ss").parse(dateString), resolution)); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery ); + return getLongs(result); + } + + + @Test + public void testSort() throws MailboxException { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.all()); + List sorts = new ArrayList(); + sorts.add(new SearchQuery.Sort(SearchQuery.Sort.SortClause.Uid, true)); + searchQuery.setSorts(sorts); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery); + assertEquals(2, (long) result.next()); + assertEquals(1, (long) result.next()); + assertEquals(0, (long) result.next()); + assertFalse(result.hasNext()); + } + + @Test + public void testDeletion() throws Exception { + sleepAfterIndex(); + String messageId = calculateElasticSearchId(messages.get(0).getMailboxId(), messages.get(0).getUid()); + searchIndex.delete(null, mailboxes.get(0), MessageRange.one(0)); + sleepAfterIndex(); + ensureIdAbsent(messageId, mailboxes.get(0)); + } + + @Test + public void testAllDeletion() throws Exception { + sleepAfterIndex(); + String messageId = calculateElasticSearchId(messages.get(0).getMailboxId(), messages.get(0).getUid()); + searchIndex.delete(null, mailboxes.get(0), MessageRange.all()); + sleepAfterIndex(); + ensureIdAbsent(messageId, mailboxes.get(0)); + } + + @Test + public void testPartialDeletion() throws Exception { + sleepAfterIndex(); + searchIndex.setLargeRangeLimit(1); + searchIndex.delete(null, mailboxes.get(0), MessageRange.range(0, 1)); + sleepAfterIndex(); + ensureIdAbsent(calculateElasticSearchId(messages.get(0).getMailboxId(), messages.get(0).getUid()), mailboxes.get(0)); + ensureIdAbsent(calculateElasticSearchId(messages.get(1).getMailboxId(), messages.get(1).getUid()), mailboxes.get(0)); + ensureIdPresent(calculateElasticSearchId(messages.get(2).getMailboxId(), messages.get(2).getUid()), mailboxes.get(0)); + ensureIdPresent(calculateElasticSearchId(messages.get(3).getMailboxId(), messages.get(3).getUid()), mailboxes.get(1)); + } + + private void ensureIdPresent(String messageId, Mailbox mailbox) throws Exception { + JsonObject jsonResult = getJsonEntry(messageId, mailbox); + JsonElement sourceElement = jsonResult.get("_source"); + assertTrue(sourceElement.isJsonObject()); + JsonObject source = sourceElement.getAsJsonObject(); + assertTrue(source.get(MessageConstants.FLAG_ANSWERED).isJsonPrimitive()); + } + + private void ensureIdAbsent(String messageId, Mailbox mailbox) throws Exception { + JsonObject jsonResult = getJsonEntry(messageId, mailbox); + JsonElement statusElement = jsonResult.get("found"); + assertTrue(statusElement.isJsonPrimitive()); + assertFalse(statusElement.getAsBoolean()); + } + + @Test + public void testAddition() throws Exception { + searchIndex.add(null, mailboxes.get(0), messages.get(0)); + sleepAfterIndex(); + String messageId = calculateElasticSearchId(messages.get(0).getMailboxId(), messages.get(0).getUid()); + JsonObject jsonResult = getJsonEntry(messageId, mailboxes.get(0)); + sleepAfterIndex(); + JsonElement sourceElement = jsonResult.get("_source"); + assertTrue(sourceElement.isJsonObject()); + JsonObject source = sourceElement.getAsJsonObject(); + assertEquals("Hello world! \n", source.get(MessageConstants.BODY_CONTENT).getAsString()); + assertEquals("To: my*address@mydomain.com\nSubject: ***Renew Your Records***\nFrom: \"renew@USBank.com\"\nContent-Type: text/html\n\n", source.get(MessageConstants.HEADER_CONTENT).getAsString()); + } + + @After + public void cleanUp() throws Exception { + searchIndex.delete(null, mailboxes.get(0), MessageRange.one(0)); + searchIndex.delete(null, mailboxes.get(0), MessageRange.one(1)); + searchIndex.delete(null, mailboxes.get(0), MessageRange.one(2)); + searchIndex.delete(null, mailboxes.get(1), MessageRange.one(3)); + searchIndex.deleteMailbox(mailboxes.get(0)); + searchIndex.deleteMailbox(mailboxes.get(1)); + sleepAfterIndex(); + searchIndex.stop(); + } + + @Test + public void testHeaderTo() throws Exception { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.address(SearchQuery.AddressType.To, "Stephane Bortzmeyer")); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery ); + List resultList = getLongs(result); + assertFalse(resultList.contains(new Long(0))); + assertFalse(resultList.contains(new Long(1))); + assertTrue(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + } + + @Test + public void testHeaderFrom() throws Exception { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.address(SearchQuery.AddressType.From, "MiNET")); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery); + List resultList = getLongs(result); + assertFalse(resultList.contains(new Long(0))); + assertTrue(resultList.contains(new Long(1))); + assertFalse(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + } + + @Test + public void testHeaderSubject() throws Exception { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.headerContains("subject", "records")); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery ); + List resultList = getLongs(result); + assertTrue(resultList.contains(new Long(0))); + assertFalse(resultList.contains(new Long(1))); + assertFalse(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + } + + @Test + public void testHeaderNested() throws Exception { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.headerContains("Received", "NEWSERVER.arneill-py.local")); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery ); + List resultList = getLongs(result); + assertFalse(resultList.contains(new Long(0))); + assertFalse(resultList.contains(new Long(1))); + assertTrue(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + } + + @Test + public void testHeaderNested2() throws Exception { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.andCriteria(SearchQuery.headerContains("Delivered-To", "NEWSERVER.arneill-py.local")); + Iterator result = searchIndex.search(null, mailboxes.get(0), searchQuery ); + List resultList = getLongs(result); + assertFalse(resultList.contains(new Long(0))); + assertFalse(resultList.contains(new Long(1))); + assertFalse(resultList.contains(new Long(2))); + assertFalse(resultList.contains(new Long(3))); + } + + @Test + public void testMailboxCreation() throws Exception { + SimpleMailbox mailbox = new SimpleMailbox(new MailboxPath("test_create", "test_user_create", "name_test_create"), 42); + mailbox.setMailboxId(new Long(142)); + searchIndex.addMailbox(mailbox); + Get get = new Get.Builder(MailboxConstants.SEARCH_INDEX_NAME, mailbox.getMailboxId().toString()).type(MailboxConstants.SEARCH_TYPE_NAME).build(); + JestResult result = jestClient.execute(get); + JsonObject resultAsJson = result.getJsonObject(); + JsonElement sourceElement = resultAsJson.get("_source"); + assertTrue(sourceElement.isJsonObject()); + JsonObject source = sourceElement.getAsJsonObject(); + assertEquals(source.get(MailboxConstants.ID).getAsString(), "142"); + searchIndex.deleteMailbox(mailbox); + } + + @Test + public void testMailboxDeletion() throws Exception { + SimpleMailbox mailbox = new SimpleMailbox(new MailboxPath("test_create", "test_user_create", "name_test_create"), 42); + mailbox.setMailboxId(new Long(142)); + searchIndex.addMailbox(mailbox); + sleepAfterIndex(); + searchIndex.deleteMailbox(mailbox); + sleepAfterIndex(); + Get get = new Get.Builder(MailboxConstants.SEARCH_INDEX_NAME, mailbox.getMailboxId().toString()).type(MailboxConstants.SEARCH_TYPE_NAME).build(); + JestResult result = jestClient.execute(get); + JsonObject resultAsJson = result.getJsonObject(); + assertFalse(resultAsJson.get("found").getAsJsonPrimitive().getAsBoolean()); + } + + @Test + public void testMailboxRename() throws Exception { + SimpleMailbox mailbox = new SimpleMailbox(new MailboxPath("test_create", "test_user_create", "name_test_create"), 42); + mailbox.setMailboxId(new Long(142)); + searchIndex.addMailbox(mailbox); + sleepAfterIndex(); + mailbox.setName("new_name"); + mailbox.setUser("new_user"); + mailbox.setNamespace("new_namespace"); + searchIndex.renameMailbox(mailbox); + sleepAfterIndex(); + Get get = new Get.Builder(MailboxConstants.SEARCH_INDEX_NAME, mailbox.getMailboxId().toString()).type(MailboxConstants.SEARCH_TYPE_NAME).build(); + JestResult result = jestClient.execute(get); + JsonObject resultAsJson = result.getJsonObject(); + JsonElement sourceElement = resultAsJson.get("_source"); + assertTrue(sourceElement.isJsonObject()); + JsonObject source = sourceElement.getAsJsonObject(); + assertEquals(source.get(MailboxConstants.ID).getAsString(), "142"); + assertEquals(source.get(MailboxConstants.NAME).getAsString(), "new_name"); + assertEquals(source.get(MailboxConstants.NAMESPACE).getAsString(), "new_namespace"); + assertEquals(source.get(MailboxConstants.USER).getAsString(), "new_user"); + searchIndex.deleteMailbox(mailbox); + sleepAfterIndex(); + } + + private List getLongs(Iterator result) { + List resultList = new ArrayList(); + while(result.hasNext()) { + resultList.add(result.next()); + } + return resultList; + } + + private String calculateElasticSearchId(long uuid, long uid) { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(uuid); + stringBuilder.append("-"); + stringBuilder.append(uid); + return stringBuilder.toString(); + } + + private JsonObject getJsonEntry(String messageId, Mailbox mailbox) throws Exception { + Get get = new Get.Builder(MessageConstants.SEARCH_INDEX_NAME, messageId).type(MessageConstants.SEARCH_TYPE_NAME).setParameter("routing", mailbox.getMailboxId().toString()).build(); + JestResult jestResult = jestClient.execute(get); + return jestResult.getJsonObject(); + } + + private void sleepAfterIndex() throws InterruptedException { + if( sleepTimeAfterIndex > 0) { + Thread.sleep(sleepTimeAfterIndex); + } + } +} diff -uNr james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/EmbeddedKafkaSearchIndexTest.java james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/EmbeddedKafkaSearchIndexTest.java --- james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/EmbeddedKafkaSearchIndexTest.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/EmbeddedKafkaSearchIndexTest.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,56 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +/* + + // This test is commented as you need have a configured kafka river on a standalone ElaticSearch to run it. We can not do that in embedded. + // Moreover we can not launch an embedded ElasticSearch because of a conflicting Lucene version used in lucene sub project. + // Uncomment if you have a configured Kafka river on a standalone ElasticSearch + + +package org.apache.james.mailbox.elasticsearch.search; + +import org.apache.james.mailbox.store.MailboxSessionMapperFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.james.mailbox.store.search.extractor.TikaTextExtractorFactory; + + +public class EmbeddedKafkaSearchIndexTest extends ElasticSearchMessageSearchIndexTest { + + private final static Logger LOG = LoggerFactory.getLogger(EmbeddedKafkaSearchIndexTest.class); + + + protected void indexSetUp(MailboxSessionMapperFactory factory) throws Exception { + searchIndex = new EmbeddedKafkaElasticsearchMessageSearchIndex(factory, "messages-river", "127.0.0.1", 9092, 2181, "/tmp/james-embedded-kafka-logs"); + searchIndex.setHostString("127.0.0.1"); + searchIndex.setShardsNb(1); + searchIndex.setReplicaNb(2); + searchIndex.setTextExtractorFactory(new TikaTextExtractorFactory()); + searchIndex.afterPropertiesSet(); + + LOG.info("search index created"); + } + + protected void sleepTimeAfterIndexSetUp() throws Exception { + sleepTimeAfterIndex = 1000; + } +} + +*/ \ No newline at end of file diff -uNr james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/KafkaMessageSearchIndexTest.java james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/KafkaMessageSearchIndexTest.java --- james-mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/KafkaMessageSearchIndexTest.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/search/index/KafkaMessageSearchIndexTest.java 2015-04-13 19:38:03.001005836 +0200 @@ -0,0 +1,53 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + + +/* + + + // This test is commented as you need have a configured kafka river on a standalone ElaticSearch to run it. We can not do that in embedded. + // Moreover we can not launch an embedded ElasticSearch because of a conflicting Lucene version used in lucene sub project. + // You have to use a Kafka river that execute bulk requests sent other kafka. + // Uncomment if you have a configured Kafka river on a standalone ElasticSearch + + +package org.apache.james.mailbox.elasticsearch.search; + +import org.apache.james.mailbox.store.MailboxSessionMapperFactory; +import org.apache.james.mailbox.store.search.extractor.TikaTextExtractorFactory; + +public class KafkaMessageSearchIndexTest extends ElasticSearchMessageSearchIndexTest { + + + protected void indexSetUp(MailboxSessionMapperFactory mapperFactory) throws Exception { + searchIndex = new KafkaElasticsearchMessageSearchIndex(mapperFactory, "messages-river", "127.0.0.1", 9092); + searchIndex.setHostString("http://127.0.0.1:9200"); + searchIndex.setShardsNb(1); + searchIndex.setReplicaNb(2); + searchIndex.setTextExtractorFactory(new TikaTextExtractorFactory()); + searchIndex.afterPropertiesSet(); + } + + protected void sleepTimeAfterIndexSetUp() throws Exception { + sleepTimeAfterIndex = 2000; + } + +} + +*/ \ No newline at end of file diff -uNr james-mailbox/pom.xml james-mailbox-elastic-search/pom.xml --- james-mailbox/pom.xml 2015-04-13 19:38:21.307673294 +0200 +++ james-mailbox-elastic-search/pom.xml 2015-04-13 20:06:52.271080638 +0200 @@ -53,6 +53,7 @@ caching hbase cassandra + elasticsearch jcr jpa lucene @@ -121,11 +122,20 @@ 13.0 2.0.1 2.0.2.1 + 2.3.3 + 2.3.3 + 2.7.0 + 2.3 + org.apache.james + apache-mailet-base + ${mailet.version} + + com.google.guava guava ${guava.version} @@ -198,8 +208,12 @@ org.apache.james apache-james-mailbox-cassandra ${project.version} + + + org.apache.james + apache-james-mailbox-elasticsearch + ${project.version} - org.apache.james apache-james-mailbox-tool @@ -362,6 +376,13 @@ commons-beanutils-core ${commons-beanutils-core.version} + + com.google.code.gson + gson + ${gson.version} + + + @@ -409,6 +430,7 @@ ${commons-io.version} + diff -uNr james-mailbox/store/pom.xml james-mailbox-elastic-search/store/pom.xml --- james-mailbox/store/pom.xml 2015-04-13 19:38:21.314339961 +0200 +++ james-mailbox-elastic-search/store/pom.xml 2015-04-13 20:07:13.164414875 +0200 @@ -86,5 +86,10 @@ test-jar test + + com.google.code.gson + gson + + diff -uNr james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/JsonHeaderBuilder.java james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/JsonHeaderBuilder.java --- james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/JsonHeaderBuilder.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/JsonHeaderBuilder.java 2015-04-13 19:55:12.000000000 +0200 @@ -0,0 +1,217 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.store.json; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import org.apache.james.mailbox.store.json.model.MessageConstants; +import org.apache.james.mailbox.store.mail.model.Message; +import org.apache.james.mailbox.store.search.SearchUtil; +import org.apache.james.mime4j.dom.address.Address; +import org.apache.james.mime4j.dom.address.AddressList; +import org.apache.james.mime4j.dom.address.Group; +import org.apache.james.mime4j.dom.address.MailboxList; +import org.apache.james.mime4j.dom.field.DateTimeField; +import org.apache.james.mime4j.field.address.LenientAddressBuilder; +import org.apache.james.mime4j.stream.Field; +import org.apache.james.mime4j.util.MimeUtil; +import org.apache.james.mime4j.dom.address.Mailbox; + +import java.text.SimpleDateFormat; +import java.util.Set; +import java.util.HashSet; +import java.util.Date; + +/** + * A header builder that directly add header fields into a Json Object representing a Json message + */ +public class JsonHeaderBuilder { + + private Message message; + private MessageDocumentBuilder.HeaderMode headerMode; + + private class HeaderStrings { + String name; + String content; + + public HeaderStrings(String name, String content) { + this.name = name; + this.content = content; + } + } + + private Set headerNameSet = new HashSet(); + private Set toAddressSet = new HashSet(); + private Set fromAddressSet = new HashSet(); + private Set ccAddressSet = new HashSet(); + private Set bccAddressSet = new HashSet(); + private Set displayFromSet = new HashSet(); + private Set displayToSet = new HashSet(); + private Set subjectSet = new HashSet(); + private Set headerStringsSet = new HashSet(); + private Date sentDate; + + public JsonHeaderBuilder(Message message, MessageDocumentBuilder.HeaderMode headerMode) { + this.message = message; + this.headerMode = headerMode; + } + + /** + * Builder. Add Header information to message JSON + */ + public void build(JsonObject source) { + if(headerMode == MessageDocumentBuilder.HeaderMode.All) { + indexHeaders(source); + index(source, MessageConstants.HEADERS_NAME, headerNameSet); + } + index(source, MessageConstants.TO, toAddressSet); + index(source, MessageConstants.DISPLAY_TO, displayToSet); + index(source, MessageConstants.FROM, fromAddressSet); + index(source, MessageConstants.DISPLAY_FROM, displayFromSet); + index(source, MessageConstants.CC, ccAddressSet); + index(source, MessageConstants.BCC, bccAddressSet); + index(source, MessageConstants.SUBJECT, subjectSet); + indexSentDate(source); + } + + /** + * Manage sent date + */ + private void indexSentDate(JsonObject source) { + if(sentDate == null) { + sentDate = message.getInternalDate(); + } + SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + String dateFormated = format.format(sentDate); + source.add(MessageConstants.SENT_DATE, new JsonPrimitive(dateFormated)); + } + + /** + * Add a field holding information contained in set to the source object + * + * @param field name of the field to create + * @param values Set holding values for this field + */ + private void index(JsonObject source, String field, Set values) { + JsonArray array = new JsonArray(); + for(String value : values) { + array.add(new JsonPrimitive(value)); + } + source.add(field, array); + } + + /** + * We add information about this field in our headers + * + * @param field Mails header field + */ + public void addField(Field field) { + String headerName = field.getName(); + String headerValue = field.getBody(); + if(headerMode == MessageDocumentBuilder.HeaderMode.All) { + headerNameSet.add(headerName); + headerStringsSet.add(new HeaderStrings(headerName, headerValue)); + } + if (field instanceof DateTimeField) { + sentDate = ((DateTimeField) field).getDate(); + } else if(isAddressField(headerName)) { + manageAddressField(field, headerName); + } else if (headerName.equalsIgnoreCase("Subject")) { + subjectSet.add(SearchUtil.getBaseSubject(headerValue)); + } + + } + + /** + * Manage address header field. + * + * Deals with addresses and group of addresses + * + * @param headerField Header field + * @param headerName Header name + */ + private void manageAddressField(Field headerField, String headerName) { + AddressList aList = LenientAddressBuilder.DEFAULT.parseAddressList(MimeUtil.unfold(headerField.getBody())); + for (Address address : aList) { + if (address instanceof Mailbox) { + Mailbox mailbox = (Mailbox) address; + addToAddressSet(headerName, mailbox); + } else if (address instanceof Group) { + MailboxList mList = ((Group) address).getMailboxes(); + for (Mailbox mailbox : mList) { + addToAddressSet(headerName, mailbox); + } + } + } + } + + /** + * Tells us if this header is a To, CC, BCC, or from field. + * + * @param headerName Header name we want to check + * @return True if the field is a classic address field, false in other cases + */ + public static boolean isAddressField(String headerName) { + if ("To".equalsIgnoreCase(headerName)) { + return true; + } else if ("From".equalsIgnoreCase(headerName)) { + return true; + } else if ("Cc".equalsIgnoreCase(headerName)) { + return true; + } else if ("Bcc".equalsIgnoreCase(headerName)) { + return true; + } + return false; + } + + /** + * Add the address to the appropriate set + * + * @param headerName Header name used to identify set + * @param mailbox mailbox holding information on this address + */ + private void addToAddressSet(String headerName, org.apache.james.mime4j.dom.address.Mailbox mailbox) { + String mailboxAddress = mailbox.getAddress(); + String mailboxDisplay = SearchUtil.getDisplayAddress(mailbox); + if ("To".equalsIgnoreCase(headerName)) { + toAddressSet.add(mailboxAddress); + displayToSet.add(mailboxDisplay); + } else if ("From".equalsIgnoreCase(headerName)) { + fromAddressSet.add(mailboxAddress); + displayFromSet.add(mailboxDisplay); + } else if ("Cc".equalsIgnoreCase(headerName)) { + ccAddressSet.add(mailboxAddress); + } else if ("Bcc".equalsIgnoreCase(headerName)) { + bccAddressSet.add(mailboxAddress); + } + } + + private void indexHeaders(JsonObject source) { + JsonArray array = new JsonArray(); + for(HeaderStrings headerStrings : headerStringsSet) { + JsonObject headerJson = new JsonObject(); + headerJson.add(MessageConstants.HEADERS_NESTED_NAME, new JsonPrimitive(headerStrings.name) ); + headerJson.add(MessageConstants.HEADERS_NESTED_VALUE, new JsonPrimitive(headerStrings.content) ); + array.add(headerJson); + } + source.add(MessageConstants.HEADERS, array); + } + +} diff -uNr james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/JsonMessageContentParser.java james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/JsonMessageContentParser.java --- james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/JsonMessageContentParser.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/JsonMessageContentParser.java 2015-04-13 19:55:12.000000000 +0200 @@ -0,0 +1,178 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.store.json; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.apache.james.mailbox.store.json.model.MessageConstants; +import org.apache.james.mailbox.store.mail.model.Message; +import org.apache.james.mailbox.store.search.extractor.TextExtractor; +import org.apache.james.mailbox.store.search.extractor.TextExtractorFactory; +import org.apache.james.mime4j.MimeException; +import org.apache.james.mime4j.stream.EntityState; +import org.apache.james.mime4j.stream.Field; +import org.apache.james.mime4j.stream.MimeTokenStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Parser for Mime content. Extracts mail header, and attachments, and add these information to + * a provided JSON document. + */ +public class JsonMessageContentParser { + + private Message message; + + private boolean isInHeader; + private int multiPartDepth; + private boolean bodyLocated; + + private JsonHeaderBuilder headerBuilder; + private JsonMimePartBuilder currentJsonMimePartBuilder; + private String body; + + private MessageDocumentBuilder.ContentMode contentMode; + + private JsonArray attachmentJsonArray; + + private TextExtractorFactory textExtractorFactory; + + private static final Logger LOG = LoggerFactory.getLogger(JsonMessageContentParser.class); + + public JsonMessageContentParser(Message message, MessageDocumentBuilder.HeaderMode headerMode, MessageDocumentBuilder.ContentMode contentMode) { + this.message = message; + headerBuilder = new JsonHeaderBuilder(message, headerMode); + this.isInHeader = false; + this.multiPartDepth = 0; + attachmentJsonArray = new JsonArray(); + this.contentMode = contentMode; + } + + public void setTextExtractorFactory(TextExtractorFactory textExtractorFactory) { + this.textExtractorFactory = textExtractorFactory; + } + + public void parse(JsonObject messageObject) throws IOException, MimeException { + walkThrewMimeMessage(); + addInformationToMessageJson(messageObject); + } + + private void walkThrewMimeMessage() throws IOException, MimeException { + MimeTokenStream stream = new MimeTokenStream(); + stream.parse(message.getFullContent()); + for (EntityState state = stream.getState(); + state != EntityState.T_END_OF_STREAM; + state = stream.next()) { + processMimePart(stream, state); + } + } + + private void processMimePart(MimeTokenStream stream, EntityState state) throws IOException { + switch (state) { + case T_START_MESSAGE: + this.isInHeader = true; + case T_START_HEADER: + if(multiPartDepth>0) { + this.currentJsonMimePartBuilder = new JsonMimePartBuilder(); + } + break; + case T_FIELD: + manageHeaderField(stream.getField()); + break; + case T_END_HEADER: + this.isInHeader=false; + break; + case T_START_MULTIPART: + startMultipart(); + break; + case T_END_MULTIPART: + this.multiPartDepth--; + break; + case T_BODY: + handleBody(stream.getDecodedInputStream()); + break; + } + } + + private void addInformationToMessageJson(JsonObject messageObject) { + if(contentMode != MessageDocumentBuilder.ContentMode.None) { + if( attachmentJsonArray.size() > 0) { + messageObject.add(MessageConstants.ATTACHMENTS, attachmentJsonArray); + } + if( body != null) { + messageObject.addProperty(MessageConstants.BODY_CONTENT, body); + } + } + headerBuilder.build(messageObject); + } + + private void manageHeaderField(Field field) { + if (isInHeader) { + headerBuilder.addField(field); + } + if (multiPartDepth > 0) { + currentJsonMimePartBuilder.addField(field); + } + } + + private void handleBody(InputStream inputStream) throws IOException { + if (multiPartDepth<= 0) { + handleBodyRoot(inputStream); + } else { + if (currentJsonMimePartBuilder != null) { + currentJsonMimePartBuilder.processHeaderFields(); + if (currentJsonMimePartBuilder.isAnAttachment()) { + if(contentMode == MessageDocumentBuilder.ContentMode.Full || contentMode == MessageDocumentBuilder.ContentMode.BodyAndAttachment) { + if (textExtractorFactory != null) { + currentJsonMimePartBuilder.manageBodyReader(inputStream, textExtractorFactory.getTextExtractor()); + } + attachmentJsonArray.add(currentJsonMimePartBuilder.convertToJson()); + } + } else { + handleBodyRoot(inputStream); + } + } else { + LOG.error("Attachment is null..."); + } + } + } + + private void handleBodyRoot(InputStream inputStream) throws IOException { + if(!bodyLocated) { + bodyLocated = true; + TextExtractor textExtractor; + try { + textExtractor = textExtractorFactory.getTextExtractor(); + textExtractor.extractContent(inputStream, null, null); + } catch(Exception exception) { + LOG.warn("Can not retrieve BODY", exception); + return; + } + body = textExtractor.getTextualContent(); + } + } + + private void startMultipart() { + this.multiPartDepth++; + this.currentJsonMimePartBuilder = new JsonMimePartBuilder(); + } +} diff -uNr james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/JsonMimePartBuilder.java james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/JsonMimePartBuilder.java --- james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/JsonMimePartBuilder.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/JsonMimePartBuilder.java 2015-04-13 20:09:17.000000000 +0200 @@ -0,0 +1,255 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.store.json; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import org.apache.james.mailbox.store.json.model.AttachmentConstants; +import org.apache.james.mailbox.store.search.extractor.TextExtractor; +import org.apache.james.mime4j.stream.Field; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Parses Mime part, and gives the availability to get it as Json. + */ +public class JsonMimePartBuilder { + + private static final Logger LOG = LoggerFactory.getLogger(JsonMimePartBuilder.class); + + private static final String HEADER_CONTENT_TYPE = "Content-Type"; + private static final String HEADER_CONTENT_DISPOSITION = "Content-Disposition"; + + private final static String ATTACHMENT = "attachment"; + private final static String FILENAME_ = "filename="; + private final static String CHARSET_ = "charset="; + + private final static String FIELD_PART_SEPARATOR = ";"; + private final static String FILE_EXTENSION_SEPARATOR = "."; + private final static char QUOTE = '"'; + + private List headerFields = new ArrayList(); + private String charset; + private String contentType; + private String fileName; + private String fileExtension; + private boolean isAttachment = false; + private String textualContent; + private List metadata = new ArrayList(); + + /** + * Convert this Mime part into Json + * + * You must process the headers before calling this. Otherwise you will get a null result. + * To have a textual content you need : manageBodyReader + * + * @return Json for this attachment + */ + public JsonObject convertToJson() { + JsonObject attachmentObject = new JsonObject(); + if(fileName != null) { + attachmentObject.addProperty(AttachmentConstants.FILENAME, fileName); + } + if(fileExtension != null) { + attachmentObject.addProperty(AttachmentConstants.EXTENSION, fileExtension); + } + if(charset != null) { + attachmentObject.addProperty(AttachmentConstants.CHARSET, charset); + } + if(contentType != null) { + attachmentObject.addProperty(AttachmentConstants.CONTENT_TYPE, contentType); + } + if(textualContent !=null) { + attachmentObject.addProperty(AttachmentConstants.TEXTUAL_CONTENT, textualContent); + } + if(headerFields.size()>0) { + attachmentObject.add(AttachmentConstants.HEADERS, getHeaderArray() ); + } + if(metadata.size()>0) { + attachmentObject.add(AttachmentConstants.METADATA, getMetadataArray()); + } + return attachmentObject; + } + + private JsonArray getHeaderArray() { + JsonArray headerArray = new JsonArray(); + for(Field field : headerFields) { + JsonObject fieldObject = new JsonObject(); + fieldObject.addProperty(AttachmentConstants.HEADER_NAME, field.getName()); + fieldObject.addProperty(AttachmentConstants.HEADER_BODY, field.getBody()); + headerArray.add(fieldObject); + } + return headerArray; + } + + private JsonArray getMetadataArray() { + JsonArray metadataArray = new JsonArray(); + for(TextExtractor.JamesMetadata jamesMetadata : metadata) { + JsonObject metadataObject = new JsonObject(); + metadataObject.addProperty(AttachmentConstants.NAME, jamesMetadata.getName()); + JsonArray jsonArray = new JsonArray(); + String[] values = jamesMetadata.getValue(); + for(String value : values) { + jsonArray.add(new JsonPrimitive(value)); + } + metadataObject.add(AttachmentConstants.VALUES, jsonArray); + metadataArray.add(metadataObject); + } + return metadataArray; + } + + public void addField(Field field) { + headerFields.add(field); + } + + + /** + * Tells if this Mime part is an attachment + * + * You must process the headers before calling this. Otherwise you will get a wrong result. + * + * @return True if this is an attachment + */ + public boolean isAnAttachment() { + return isAttachment; + } + + public void processHeaderFields() { + for(Field field : headerFields) { + if(field.getName().equals(HEADER_CONTENT_DISPOSITION)) { + parseContentDisposition(field.getBody()); + } + if(field.getName().equals(HEADER_CONTENT_TYPE)) { + parseContentType(field.getBody()); + } + } + } + + public void manageBodyReader(InputStream inputStream, TextExtractor textExtractor) { + try { + textExtractor.extractContent(inputStream, contentType, fileName); + } catch(Exception exception) { + LOG.warn("Error while extracting attachment text : ", exception); + } + textualContent = textExtractor.getTextualContent(); + metadata = textExtractor.getMetadata(); + } + + private void parseContentType(String contentTypeString) { + String[] contentTypeSplit = contentTypeString.split(FIELD_PART_SEPARATOR); + if(contentTypeSplit.length < 1 ) { + LOG.warn("Nothing to parse for Content-Type Header"); + return; + } + contentType = suppressWhiteSpace(contentTypeSplit[0]); + for(int i = 1; i < contentTypeSplit.length; i++) { + String foundCharset = lookForCharset(contentTypeSplit[i]); + if(foundCharset != null) { + charset = foundCharset; + } + } + } + + private void parseContentDisposition(String contentDisposition) { + String[] contentDispositionSplit = contentDisposition.split(FIELD_PART_SEPARATOR); + if(contentDispositionSplit.length < 1 ) { + LOG.warn("Nothing to parse for Content-Disposition Header"); + return; + } + if(suppressWhiteSpace(contentDispositionSplit[0]).startsWith(ATTACHMENT)) { + isAttachment = true; + } + for(int i = 1; i < contentDispositionSplit.length; i++) { + String foundFileName = lookForFileName(contentDispositionSplit[i]); + if(foundFileName != null) { + fileName = foundFileName; + findFileExtension(); + } + } + } + + private void findFileExtension() { + if(fileName.contains(FILE_EXTENSION_SEPARATOR)) { + String[] fileNameSplit = fileName.split("\\."); + fileExtension = fileNameSplit[fileNameSplit.length-1]; + } + } + + private String lookForFileName(String input) { + return lookForQuotedValue(input, FILENAME_); + } + + private String lookForCharset(String input) { + return lookForUnQuotedValue(input, CHARSET_); + } + + /* + String parsing utilities + */ + + private String lookForQuotedValue(String input, String startSequence) { + String valueUnExtracted = suppressWhiteSpace(input); + if (valueUnExtracted.startsWith(startSequence)) { + String valueQuoted = suppressWhiteSpace(valueUnExtracted.substring(startSequence.length())); + if(valueQuoted.startsWith(Character.toString(QUOTE))) { + Integer closingQuotePosition = findCharFromPos(valueQuoted, 1, QUOTE); + if (closingQuotePosition == null) { + LOG.warn("Error : no closing quote for "+ startSequence); + return null; + } + return valueQuoted.substring(1, closingQuotePosition); + } + } + return null; + } + + private String lookForUnQuotedValue(String input, String startSequence) { + String valueUnExtracted = suppressWhiteSpace(input); + if (valueUnExtracted.startsWith(startSequence)) { + return suppressWhiteSpace(valueUnExtracted.substring(startSequence.length())); + } + return null; + } + + private String suppressWhiteSpace(String input) { + int pos = 0; + while(pos < input.length() && ( input.charAt(pos) == ' ' || input.charAt(pos) == '\t' ) ) { + pos++; + } + if(pos >= input.length()) { + return ""; + } + return input.substring(pos); + } + + private Integer findCharFromPos(String string, int pos, char c) { + for(int i = pos; i < string.length(); i++) { + if (string.charAt(i) == c) { + return i; + } + } + return null; + } + +} diff -uNr james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/MailboxDocumentBuilder.java james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/MailboxDocumentBuilder.java --- james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/MailboxDocumentBuilder.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/MailboxDocumentBuilder.java 2015-04-13 19:55:12.000000000 +0200 @@ -0,0 +1,56 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.mailbox.store.json; + +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import org.apache.james.mailbox.model.MailboxPath; +import org.apache.james.mailbox.store.json.model.MailboxConstants; +import org.apache.james.mailbox.store.mail.model.Mailbox; + +/** + * This class generates mailbox JSON documents + */ +public class MailboxDocumentBuilder { + + private MailboxPath mailboxPath; + private Id id; + + public MailboxDocumentBuilder(Mailbox mailbox) { + this.id = mailbox.getMailboxId(); + this.mailboxPath = new MailboxPath(mailbox.getNamespace(), mailbox.getUser(), mailbox.getName()); + } + + public MailboxDocumentBuilder(MailboxPath mailboxPath) { + this.mailboxPath = mailboxPath; + } + + public JsonObject build() { + JsonObject source = new JsonObject(); + if(id != null) { + source.add(MailboxConstants.ID, new JsonPrimitive(id.toString())); + } + source.add(MailboxConstants.USER, new JsonPrimitive(mailboxPath.getUser())); + source.add(MailboxConstants.NAME, new JsonPrimitive(mailboxPath.getName())); + source.add(MailboxConstants.NAMESPACE, new JsonPrimitive(mailboxPath.getNamespace())); + return source; + } + +} diff -uNr james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/MessageDocumentBuilder.java james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/MessageDocumentBuilder.java --- james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/MessageDocumentBuilder.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/MessageDocumentBuilder.java 2015-04-13 19:55:12.000000000 +0200 @@ -0,0 +1,170 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.store.json; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import org.apache.commons.io.IOUtils; +import org.apache.james.mailbox.model.MailboxPath; +import org.apache.james.mailbox.store.json.model.MessageConstants; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.store.mail.model.Message; +import org.apache.james.mailbox.store.search.extractor.TextExtractorFactory; +import org.apache.james.mime4j.MimeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * This class generates message JSON documents + */ +public class MessageDocumentBuilder { + + private static final Logger LOG = LoggerFactory.getLogger(MessageDocumentBuilder.class); + + private TextExtractorFactory textExtractorFactory; + + public MessageDocumentBuilder(Message message) { + this.source = new JsonObject(); + this.message = message; + } + + private static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + + private static synchronized String format(Date date) { + return simpleDateFormat.format(date); + } + + private JsonObject source; + private Message message; + + public enum ContentMode { + Full, + BodyAndAttachment, + Body, + None + } + + public enum MetadataMode { + All, + Limited + } + + public enum HeaderMode { + All, + Standard + } + + public JsonObject build() { + return source; + } + + public MessageDocumentBuilder setTextExtractorFactory(TextExtractorFactory textExtractorFactory) { + this.textExtractorFactory = textExtractorFactory; + return this; + } + + public MessageDocumentBuilder addMetadata(MetadataMode metadataMode) { + switch(metadataMode) { + case All: + source.addProperty(MessageConstants.MAILBOX_ID, message.getMailboxId().toString()); + source.addProperty(MessageConstants.FULL_CONTENT_OCTET, message.getFullContentOctets()); + source.addProperty(MessageConstants.BODY_OCTET, message.getBodyOctets()); + source.addProperty(MessageConstants.TEXTUAL_LINE_COUNT, getLineCount(message)); + source.addProperty(MessageConstants.MEDIA_TYPE, getMediaType(message)); + case Limited: + source.addProperty(MessageConstants.DATE, format(message.getInternalDate())); + source.addProperty(MessageConstants.UID, message.getUid()); + source.addProperty(MessageConstants.MODSEQ, message.getModSeq()); + source.addProperty(MessageConstants.FLAG_ANSWERED, message.isAnswered()); + source.addProperty(MessageConstants.FLAG_DELETED, message.isDeleted()); + source.addProperty(MessageConstants.FLAG_DRAFT, message.isDraft()); + source.addProperty(MessageConstants.FLAG_FLAGGED, message.isFlagged()); + source.addProperty(MessageConstants.FLAG_RECENT, message.isRecent()); + source.addProperty(MessageConstants.FLAG_SEEN, message.isSeen()); + source.add(MessageConstants.USER_FLAGS, createFlagsJsonArray()); + } + return this; + } + public MessageDocumentBuilder addContent(ContentMode contentMode) throws IOException { + switch(contentMode) { + case Full: + source.addProperty(MessageConstants.FULL_CONTENT, IOUtils.toString(message.getFullContent(), "UTF-8")); + case Body: + case BodyAndAttachment: + source.addProperty(MessageConstants.HEADER_CONTENT, IOUtils.toString(message.getHeaderContent(), "UTF-8")); + } + return this; + } + + public MessageDocumentBuilder addMailbox(MailboxPath mailboxPath) { + source.add(MessageConstants.MAILBOX, + new MailboxDocumentBuilder(mailboxPath).build()); + return this; + } + + public MessageDocumentBuilder parseMessage(HeaderMode headerMode, ContentMode contentMode) throws MailboxException { + try { + JsonMessageContentParser parser = new JsonMessageContentParser(message, headerMode, contentMode); + parser.setTextExtractorFactory(textExtractorFactory); + parser.parse(source); + } catch(IOException io) { + LOG.error("IO Exception",io); + } catch(MimeException mimeException) { + LOG.error("Mime exception", mimeException); + } + return this; + } + + /** + * Create custom user flag Json Array + * + * @return JsonArray that contains userFlags + */ + public JsonArray createFlagsJsonArray() { + String[] userFlags = message.createFlags().getUserFlags(); + JsonArray userFlagsArray = new JsonArray(); + for(String userFlag : userFlags) { + userFlagsArray.add(new JsonPrimitive(userFlag)); + } + return userFlagsArray; + } + + private String getMediaType(Message message) { + if(message.getMediaType() != null) { + return message.getMediaType(); + } else { + return ""; + } + } + + private long getLineCount(Message message) { + Long lineCount = message.getTextualLineCount(); + if(lineCount != null) { + return lineCount; + } else { + return 0; + } + } + +} diff -uNr james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/model/AttachmentConstants.java james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/model/AttachmentConstants.java --- james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/model/AttachmentConstants.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/model/AttachmentConstants.java 2015-04-13 19:55:12.000000000 +0200 @@ -0,0 +1,39 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.store.json.model; + +/** + * Constants for attachments + */ +public interface AttachmentConstants { + + String CHARSET = "charset"; + String EXTENSION = "extension"; + String FILENAME = "filename"; + String CONTENT_TYPE = "content-type"; + String TEXTUAL_CONTENT = "textualContent"; + + String HEADERS = "headers"; + String HEADER_NAME = "headers.name"; + String HEADER_BODY = "headers.body"; + + String METADATA = "metadata"; + String NAME = "metadata.name"; + String VALUES = "metadata.values"; +} diff -uNr james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/model/MailboxConstants.java james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/model/MailboxConstants.java --- james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/model/MailboxConstants.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/model/MailboxConstants.java 2015-04-13 19:55:12.000000000 +0200 @@ -0,0 +1,32 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.store.json.model; + +/** + * Constants used to generate JSON messages representing a mailbox + */ +public interface MailboxConstants { + String SEARCH_INDEX_NAME = "mailbox-search"; + String SEARCH_TYPE_NAME = "mailbox"; + + String ID = "id"; + String USER = "user"; + String NAMESPACE = "namespace"; + String NAME = "name"; +} diff -uNr james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/model/MessageConstants.java james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/model/MessageConstants.java --- james-mailbox/store/src/main/java/org/apache/james/mailbox/store/json/model/MessageConstants.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/json/model/MessageConstants.java 2015-04-13 19:55:12.000000000 +0200 @@ -0,0 +1,69 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.store.json.model; + +/** + * String constants used to generate JSON messages representing Messages + */ +public interface MessageConstants { + String SEARCH_INDEX_NAME = "mailbox-search"; + String SEARCH_TYPE_NAME = "messages"; + String DATE = "date"; + + // There is no 128 bit numeric value in cassandra. We are compelled to split the id in 2. + + // Data consolidation to increase performances : James will filter these field in every + // request, so to have good performances we need to have these field directly in the message + // and not in a parent object. + String MAILBOX_ID = "mailbox-id"; + + String UID = "uid"; + String MODSEQ = "modseq"; + String FLAG_ANSWERED = "flagAnswered"; + String FLAG_DELETED = "flagDeleted"; + String FLAG_DRAFT = "flagDraft"; + String FLAG_FLAGGED = "flagFlagged"; + String FLAG_RECENT = "flagRecent"; + String FLAG_SEEN = "flagSeen"; + String USER_FLAGS = "userFlag"; + String BODY_CONTENT = "body-content"; + String TEXTUAL_LINE_COUNT = "textual-line-count"; + String MEDIA_TYPE = "media-type"; + String HEADER_CONTENT = "header-content"; + String BODY_OCTET = "body-octet"; + String FULL_CONTENT_OCTET = "full-content-octet"; + String FULL_CONTENT = "full-content"; + String ARRIVAL = "arrival"; + String CC = "cc"; + String BCC = "bcc"; + String FROM = "from"; + String TO = "to"; + String SUBJECT = "subject"; + String SENT_DATE = "sent-date"; + String DISPLAY_FROM = "display-from"; + String DISPLAY_TO = "display-to"; + String HEADERS_NAME = "header-names"; + String HEADERS = "headers"; + String HEADERS_NESTED_NAME = "name"; + String HEADERS_NESTED_VALUE = "value"; + + String ATTACHMENTS = "attachments"; + + String MAILBOX = "mailbox"; +} diff -uNr james-mailbox/store/src/main/java/org/apache/james/mailbox/store/search/extractor/DefaultTextExtractor.java james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/search/extractor/DefaultTextExtractor.java --- james-mailbox/store/src/main/java/org/apache/james/mailbox/store/search/extractor/DefaultTextExtractor.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/search/extractor/DefaultTextExtractor.java 2015-04-13 19:56:38.000000000 +0200 @@ -0,0 +1,63 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.mailbox.store.search.extractor; + +import org.apache.commons.io.IOUtils; + +import java.io.InputStream; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; + +/** + * A default text extractor that is directly based on the input file provided. + * + * Costs less calculs that TikaTextExtractor, but result is not good at all. + */ +public class DefaultTextExtractor implements TextExtractor { + + private String textualContent; + private String encoding; + + public DefaultTextExtractor(String encoding) { + this.encoding = encoding; + } + + @Override + public void extractContent(InputStream inputStream, String contentType, String fileName) throws Exception { + if(contentType == null || contentType.startsWith("text/") ) { + StringWriter writer = new StringWriter(); + IOUtils.copy(inputStream, writer, encoding); + textualContent = writer.toString(); + } else { + textualContent = ""; + } + } + + @Override + public String getTextualContent() { + return textualContent; + } + + @Override + public List getMetadata() { + return new ArrayList(); + } +} diff -uNr james-mailbox/store/src/main/java/org/apache/james/mailbox/store/search/extractor/DefaultTextExtractorFactory.java james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/search/extractor/DefaultTextExtractorFactory.java --- james-mailbox/store/src/main/java/org/apache/james/mailbox/store/search/extractor/DefaultTextExtractorFactory.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/search/extractor/DefaultTextExtractorFactory.java 2015-04-13 19:56:38.000000000 +0200 @@ -0,0 +1,39 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.mailbox.store.search.extractor; + +/** + * Factory for DefaultTextExtractor + * + * Costs less calculs that TikaTextExtractor, but result is not good at all. + */ +public class DefaultTextExtractorFactory implements TextExtractorFactory { + + private String encoding; + + public DefaultTextExtractorFactory(String encoding) { + this.encoding = encoding; + } + + @Override + public TextExtractor getTextExtractor() { + return new DefaultTextExtractor(encoding); + } +} diff -uNr james-mailbox/store/src/main/java/org/apache/james/mailbox/store/search/extractor/TextExtractor.java james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/search/extractor/TextExtractor.java --- james-mailbox/store/src/main/java/org/apache/james/mailbox/store/search/extractor/TextExtractor.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/search/extractor/TextExtractor.java 2015-04-13 19:56:38.000000000 +0200 @@ -0,0 +1,75 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.mailbox.store.search.extractor; + +import java.io.InputStream; +import java.util.List; + +/** + * Extract textual content and metadata from files + */ +public interface TextExtractor { + + /** + * Parses the file to extract its content. + * + * @param inputStream File + * @param contentType Content type of the file + * @param fileName File name + * @throws Exception + */ + void extractContent(InputStream inputStream, String contentType, String fileName) throws Exception; + + /** + * Retrieve textual content. extractContent must be call before using this. + * + * @return Textual content of the file + */ + String getTextualContent(); + + /** + * Gets the file metadata + * + * @return List of file metadata + */ + List getMetadata(); + + /** + * Represents file metadata. + */ + public class JamesMetadata { + String name; + String[] values; + + public JamesMetadata(String name, String[] values) { + this.name = name; + this.values = values; + } + + public String getName() { + return name; + } + + public String[] getValue() { + return values; + } + } + +} diff -uNr james-mailbox/store/src/main/java/org/apache/james/mailbox/store/search/extractor/TextExtractorFactory.java james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/search/extractor/TextExtractorFactory.java --- james-mailbox/store/src/main/java/org/apache/james/mailbox/store/search/extractor/TextExtractorFactory.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/main/java/org/apache/james/mailbox/store/search/extractor/TextExtractorFactory.java 2015-04-13 19:56:38.000000000 +0200 @@ -0,0 +1,34 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.mailbox.store.search.extractor; + +/** + * Factory for TextExtractor + */ +public interface TextExtractorFactory { + + /** + * By calling this method, you get a TextExtractor you can work with. + * + * @return A TextExtractor you can work with. + */ + TextExtractor getTextExtractor(); + +} diff -uNr james-mailbox/store/src/test/java/org/apache/james/mailbox/store/json/MessageDocumentBuilderTest.java james-mailbox-elastic-search/store/src/test/java/org/apache/james/mailbox/store/json/MessageDocumentBuilderTest.java --- james-mailbox/store/src/test/java/org/apache/james/mailbox/store/json/MessageDocumentBuilderTest.java 1970-01-01 01:00:00.000000000 +0100 +++ james-mailbox-elastic-search/store/src/test/java/org/apache/james/mailbox/store/json/MessageDocumentBuilderTest.java 2015-04-13 20:13:57.000000000 +0200 @@ -0,0 +1,105 @@ +package org.apache.james.mailbox.store.json; + + +import com.google.gson.JsonObject; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.model.MailboxPath; +import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.Message; +import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder; +import org.apache.james.mailbox.store.mail.model.impl.SimpleMailbox; +import org.apache.james.mailbox.store.mail.model.impl.SimpleMessage; +import org.apache.james.mailbox.store.search.extractor.DefaultTextExtractorFactory; +import org.junit.Test; + +import javax.mail.Flags; +import javax.mail.internet.SharedInputStream; +import javax.mail.util.SharedByteArrayInputStream; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; + +import static org.junit.Assert.assertEquals; + +public class MessageDocumentBuilderTest { + + private static String ATTACHMENT_HEADERS = "From: John Doe \n" + + "MIME-Version: 1.0\n" + + "Content-Type: multipart/mixed;\n" + + " boundary=\"XXXXboundary text\"\n"; + + private static String BODY_ATTACHMENT = "\n" + + "This is a multipart message in MIME format.\n" + + "\n" + + "--XXXXboundary text \n" + + "Content-Type: text/plain\n" + + "\n" + + "this is the body text\n" + + "\n" + + "--XXXXboundary text \n" + + "Content-Type: text/plain;\n" + + "Content-Disposition: attachment;\n" + + " filename=\"test.txt\"\n" + + "\n" + + "this is the attachment text\n" + + "\n" + + "--XXXXboundary text--\n"; + + @Test + public void testAttachment() throws MailboxException, ParseException, IOException, InterruptedException { + SimpleMailbox mailbox = new SimpleMailbox(new MailboxPath("namespace", "user", "INBOX"), 45); + mailbox.setMailboxId(42l); + Message message = initializeMessage(ATTACHMENT_HEADERS, BODY_ATTACHMENT, mailbox, 3, 4, "2014/11/27 21:23:25"); + MessageDocumentBuilder messageDocumentBuilder = new MessageDocumentBuilder(message); + messageDocumentBuilder.setTextExtractorFactory(new DefaultTextExtractorFactory("UTF-8")); + JsonObject messageAsJson = messageDocumentBuilder + .parseMessage(MessageDocumentBuilder.HeaderMode.Standard, MessageDocumentBuilder.ContentMode.BodyAndAttachment) + .addContent(MessageDocumentBuilder.ContentMode.BodyAndAttachment).addMetadata(MessageDocumentBuilder.MetadataMode.All) + .build(); + assertEquals("{\"attachments\":[{\"filename\":\"test.txt\",\"extension\":\"txt\",\"content-type\":\"text/plain\",\"textualContent\":\"this is the attachment text\\n\",\"headers\":[{\"headers.name\":\"Content-Type\",\"headers.body\":\"text/plain;\"},{\"headers.name\":\"Content-Disposition\",\"headers.body\":\"attachment; filename=\\\"test.txt\\\"\"}]}],\"body-content\":\"this is the body text\\n\",\"to\":[],\"display-to\":[],\"from\":[\"example@example.com\"],\"display-from\":[\"John Doe\"],\"cc\":[],\"bcc\":[],\"subject\":[],\"sent-date\":\"2014/11/27 21:23:25\",\"header-content\":\"From: John Doe \\nMIME-Version: 1.0\\nContent-Type: multipart/mixed;\\n boundary=\\\"XXXXboundary text\\\"\\n\",\"mailbox-id\":\"42\",\"full-content-octet\":399,\"body-octet\":276,\"textual-line-count\":0,\"media-type\":\"\",\"date\":\"2014/11/27 21:23:25\",\"uid\":3,\"modseq\":4,\"flagAnswered\":false,\"flagDeleted\":false,\"flagDraft\":false,\"flagFlagged\":false,\"flagRecent\":false,\"flagSeen\":false,\"userFlag\":[]}", + messageAsJson.toString()); + } + + @Test + public void testMessageWithNoBody() throws MailboxException, ParseException, IOException, InterruptedException { + SimpleMailbox mailbox = new SimpleMailbox(new MailboxPath("namespace", "user", "INBOX"), 45); + mailbox.setMailboxId(42l); + Message message = initializeMessage(ATTACHMENT_HEADERS, BODY_ATTACHMENT, mailbox, 3, 4, "2014/11/27 21:23:25"); + MessageDocumentBuilder messageDocumentBuilder = new MessageDocumentBuilder(message); + messageDocumentBuilder.setTextExtractorFactory(new DefaultTextExtractorFactory("UTF-8")); + JsonObject messageAsJson = messageDocumentBuilder + .parseMessage(MessageDocumentBuilder.HeaderMode.Standard, MessageDocumentBuilder.ContentMode.None) + .addContent(MessageDocumentBuilder.ContentMode.None).addMetadata(MessageDocumentBuilder.MetadataMode.All) + .build(); + assertEquals("{\"to\":[],\"display-to\":[],\"from\":[\"example@example.com\"],\"display-from\":[\"John Doe\"],\"cc\":[],\"bcc\":[],\"subject\":[],\"sent-date\":\"2014/11/27 21:23:25\",\"mailbox-id\":\"42\",\"full-content-octet\":399,\"body-octet\":276,\"textual-line-count\":0,\"media-type\":\"\",\"date\":\"2014/11/27 21:23:25\",\"uid\":3,\"modseq\":4,\"flagAnswered\":false,\"flagDeleted\":false,\"flagDraft\":false,\"flagFlagged\":false,\"flagRecent\":false,\"flagSeen\":false,\"userFlag\":[]}", + messageAsJson.toString()); + } + + @Test + public void testSimpleMessage() throws MailboxException, ParseException, IOException { + SimpleMailbox mailbox = new SimpleMailbox(new MailboxPath("namespace", "user", "INBOX"), 45); + mailbox.setMailboxId(42l); + Message message = initializeMessage("To: jean@dujardin.fr\nSubject: *I love your latest movie*\nFrom: \"Your better groupie\"\nContent-Type: text/html\n\n", + "I love you so much <3 \n", mailbox, 3, 4, "2014/11/27 21:23:25"); + MessageDocumentBuilder messageDocumentBuilder = new MessageDocumentBuilder(message); + messageDocumentBuilder.setTextExtractorFactory(new DefaultTextExtractorFactory("UTF-8")); + JsonObject messageAsJson = messageDocumentBuilder + .parseMessage(MessageDocumentBuilder.HeaderMode.Standard, MessageDocumentBuilder.ContentMode.BodyAndAttachment) + .addContent(MessageDocumentBuilder.ContentMode.BodyAndAttachment).addMetadata(MessageDocumentBuilder.MetadataMode.All) + .build(); + assertEquals("{\"body-content\":\"I love you so much <3 \\n\",\"to\":[\"jean@dujardin.fr\"],\"display-to\":[\"jean@dujardin.fr\"],\"from\":[\"mad@fan.com\"],\"display-from\":[\"Your better groupie\"],\"cc\":[],\"bcc\":[],\"subject\":[\"*I love your latest movie*\"],\"sent-date\":\"2014/11/27 21:23:25\",\"header-content\":\"To: jean@dujardin.fr\\nSubject: *I love your latest movie*\\nFrom: \\\"Your better groupie\\\"\\nContent-Type: text/html\\n\\n\",\"mailbox-id\":\"42\",\"full-content-octet\":146,\"body-octet\":23,\"textual-line-count\":0,\"media-type\":\"\",\"date\":\"2014/11/27 21:23:25\",\"uid\":3,\"modseq\":4,\"flagAnswered\":false,\"flagDeleted\":false,\"flagDraft\":false,\"flagFlagged\":false,\"flagRecent\":false,\"flagSeen\":false,\"userFlag\":[]}", + messageAsJson.toString()); + } + + private Message initializeMessage(String headerContent, String bodyContent, Mailbox mailbox, long uid, long modseq, String date) throws ParseException, MailboxException { + int bodyStartOctet = headerContent.length(); + String messageContent = headerContent + bodyContent; + SharedInputStream sharedInputStream = new SharedByteArrayInputStream(messageContent.getBytes()); + SimpleMessage message = new SimpleMessage(new SimpleDateFormat("yyyy/MM/dd HH:mm:ss").parse(date), messageContent.length(), bodyStartOctet, + sharedInputStream, new Flags(), new PropertyBuilder(), mailbox.getMailboxId()); + message.setUid(uid); + message.setModSeq(modseq); + return message; + } + +} diff -uNr james-mailbox/tool/pom.xml james-mailbox-elastic-search/tool/pom.xml --- james-mailbox/tool/pom.xml 2015-04-13 19:38:21.091006618 +0200 +++ james-mailbox-elastic-search/tool/pom.xml 2015-04-13 19:38:02.434339145 +0200 @@ -39,10 +39,22 @@ org.apache.james apache-james-mailbox-store + + + org.apache.james + apache-james-mailbox-memory + + org.apache.james apache-james-mailbox-memory + + + org.apache.james + apache-james-mailbox-store + + org.slf4j