Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java (date 1438672868000) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java (revision ) @@ -605,12 +605,12 @@ } @Override - public boolean deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { - + public long deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { + long count = 0; // sanity check if (chunkIds.isEmpty()) { // sanity check, nothing to do - return true; + return count; } Connection con = this.ch.getRWConnection(); @@ -645,7 +645,7 @@ prepData.setString(idx + 1, chunkIds.get(idx)); } - prepMeta.execute(); + count = prepMeta.executeUpdate(); prepData.execute(); prepMeta.close(); prepMeta = null; @@ -658,7 +658,7 @@ this.ch.closeConnection(con); } - return true; + return count; } @Override Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/BlobCache.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/BlobCache.java (date 1438672868000) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/BlobCache.java (revision ) @@ -156,7 +156,7 @@ } @Override - public boolean deleteChunks(List arg0, long arg1) throws Exception { + public long deleteChunks(List arg0, long arg1) throws Exception { return base.deleteChunks(arg0, arg1); } Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java (date 1438672868000) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java (revision ) @@ -205,7 +205,7 @@ } @Override - public boolean deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { + public long deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { DBCollection collection = getBlobCollection(); QueryBuilder queryBuilder = new QueryBuilder(); if (chunkIds != null) { @@ -217,11 +217,7 @@ } WriteResult result = collection.remove(queryBuilder.get()); - if (result.getN() == chunkIds.size()) { - return true; - } - - return false; + return result.getN(); } @Override Index: oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/package-info.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/package-info.java (date 1438672868000) +++ oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/package-info.java (revision ) @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@Version("1.0") +@Version("2.0") @Export(optional = "provide:=true") package org.apache.jackrabbit.oak.spi.blob; import aQute.bnd.annotation.Version; -import aQute.bnd.annotation.Export; \ No newline at end of file +import aQute.bnd.annotation.Export; Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java (date 1438672868000) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java (revision ) @@ -221,19 +221,19 @@ } @Override - public boolean deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { + public long deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { Preconditions.checkNotNull(context); - + long count = 0; for (String chunkId : chunkIds) { final org.jclouds.blobstore.BlobStore blobStore = context.getBlobStore(); StorageMetadata metadata = blobStore.blobMetadata(cloudContainer, chunkId); if ((maxLastModifiedTime <= 0) || (metadata.getLastModified().getTime() <= maxLastModifiedTime)) { blobStore.removeBlob(cloudContainer, chunkId); - return true; + count++; } } - return true; + return count; } class CloudStoreIterator implements Iterator { Index: oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java (date 1438672868000) +++ oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java (revision ) @@ -220,7 +220,8 @@ } @Override - public boolean deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { + public long deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { + int count = 0; for (String chunkId : chunkIds) { byte[] digest = StringUtils.convertHexToBytes(chunkId); File f = getFile(digest, false); @@ -233,9 +234,10 @@ if ((maxLastModifiedTime <= 0) || FileUtils.isFileOlder(f, maxLastModifiedTime)) { f.delete(); + count++; } } - return true; + return count; } @Override @@ -270,4 +272,4 @@ public void clearCache() { // no cache } -} \ No newline at end of file +} Index: oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java (date 1438672868000) +++ oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java (revision ) @@ -85,16 +85,19 @@ * Ignores the maxlastModifiedTime */ @Override - public boolean deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { + public long deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { + int count = 0; for (String chunkId : chunkIds) { BlockId id = new BlockId(StringUtils.convertHexToBytes(chunkId), 0); if (map.containsKey(id)) { map.remove(id); + count++; } else if (old.containsKey(id)) { old.remove(id); + count++; } } - return true; + return count; } /** Index: oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java (date 1438672868000) +++ oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java (revision ) @@ -88,10 +88,10 @@ * * @param chunkIds the chunk ids * @param maxLastModifiedTime the max last modified time to consider for retrieval - * @return true, if successful + * @return long the count of successful deletions * @throws Exception the exception */ - boolean deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception; + long deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception; /** * Resolve chunks from the given Id. Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (date 1438672868000) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (revision ) @@ -183,10 +183,8 @@ int deleteCount = sweep(); threw = false; - LOG.info( - "Blob garbage collection completed in {}. Number of blobs identified for deletion [{}] (This " - + "includes blobs newer than configured interval [{}] which are ignored for deletion)", - sw.toString(), deleteCount, maxLastModifiedInterval); + LOG.info("Blob garbage collection completed in {}. Number of blobs deleted [{}]", sw.toString(), + deleteCount, maxLastModifiedInterval); } } finally { if (!LOG.isTraceEnabled()) { @@ -300,6 +298,8 @@ // Calculate the references not used difference(); int count = 0; + int deleted = 0; + state = State.SWEEPING; LOG.debug("Starting sweep phase of the garbage collector"); LOG.debug("Sweeping blobs with modified time > than the configured max deleted time ({}). ", @@ -314,18 +314,17 @@ while (iterator.hasNext()) { ids.add(iterator.next()); - if (ids.size() > getBatchCount()) { + if (ids.size() >= getBatchCount()) { count += ids.size(); - sweepInternal(ids, exceptionQueue, earliestRefAvailTime); + deleted += sweepInternal(ids, exceptionQueue, earliestRefAvailTime); ids = Lists.newArrayList(); } } if (!ids.isEmpty()) { count += ids.size(); - sweepInternal(ids, exceptionQueue, earliestRefAvailTime); + deleted += sweepInternal(ids, exceptionQueue, earliestRefAvailTime); } - count -= exceptionQueue.size(); BufferedWriter writer = null; try { if (!exceptionQueue.isEmpty()) { @@ -336,12 +335,17 @@ LineIterator.closeQuietly(iterator); IOUtils.closeQuietly(writer); } + if(!exceptionQueue.isEmpty()) { + LOG.warn("Unable to delete some blobs entries from the blob store. Details around such blob entries can be found in [{}]", fs.getGarbage().getAbsolutePath()); + } + if(count != deleted) { LOG.warn( - "Unable to delete some blobs entries from the blob store. This may happen if blob modified time is > " - + "than the max deleted time ({}). Details around such blob entries can be found in [{}]", - timestampToString(getLastMaxModifiedTime(earliestRefAvailTime)), fs.getGarbage().getAbsolutePath()); + "Deleted only [{}] blobs entries from the [{}] candidates identified. This may happen if blob modified time is > " + + "than the max deleted time ({})", + deleted, count, timestampToString(getLastMaxModifiedTime(earliestRefAvailTime))); } + // Remove all the merged marked references GarbageCollectionType.get(blobStore).removeAllMarkedReferences(blobStore); LOG.debug("Ending sweep phase of the garbage collector"); @@ -375,19 +379,21 @@ * @param exceptionQueue * @param maxModified */ - private void sweepInternal(List ids, ConcurrentLinkedQueue exceptionQueue, long maxModified) { + private long sweepInternal(List ids, ConcurrentLinkedQueue exceptionQueue, long maxModified) { + long deleted = 0; try { - LOG.debug("Blob ids to be deleted {}", ids); - boolean deleted = blobStore.deleteChunks(ids, getLastMaxModifiedTime(maxModified)); - if (!deleted) { + LOG.trace("Blob ids to be deleted {}", ids); + deleted = blobStore.deleteChunks(ids, getLastMaxModifiedTime(maxModified)); + if (deleted != ids.size()) { // Only log and do not add to exception queue since some blobs may not match the // lastMaxModifiedTime criteria. - LOG.debug("Some blobs were not deleted from the batch : [{}]", ids); + LOG.debug("Some [{}] blobs were not deleted from the batch : [{}]", ids.size() - deleted, ids); } } catch (Exception e) { LOG.warn("Error occurred while deleting blob with ids [{}]", ids, e); exceptionQueue.addAll(ids); } + return deleted; } /** Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java (date 1438672868000) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java (revision ) @@ -377,7 +377,8 @@ } @Override - public boolean deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { + public long deleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { + int count = 0; if (delegate instanceof MultiDataStoreAware) { for (String chunkId : chunkIds) { String blobId = extractBlobId(chunkId); @@ -390,10 +391,11 @@ if (success) { ((MultiDataStoreAware) delegate).deleteRecord(identifier); log.debug("Deleted blob [{}]", blobId); + count++; } } } - return true; + return count; } @Override Index: oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>windows-1252 =================================================================== --- oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java (date 1438672868000) +++ oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java (revision ) @@ -412,7 +412,7 @@ public void delete() throws Exception { Set ids = createArtifacts(); - store.deleteChunks(Lists.newArrayList(ids), 0); + long count = store.deleteChunks(Lists.newArrayList(ids), 0); Iterator iter = store.getAllChunkIds(0); Set ret = Sets.newHashSet(); @@ -421,6 +421,7 @@ } assertTrue(ret.toString(), ret.isEmpty()); + assertEquals(ids.size(), count); } private Set createArtifacts() throws Exception {