diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java index ef60308401..e9a65bef75 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java @@ -16,55 +16,95 @@ */ package org.apache.jackrabbit.oak.segment.file; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Maps.newHashMap; import static com.google.common.collect.Sets.newHashSet; import static org.apache.commons.io.FileUtils.listFiles; import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Set; import java.util.UUID; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.base.Predicate; import com.google.common.base.Supplier; +import com.google.common.collect.Iterators; import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector; import org.apache.jackrabbit.oak.segment.SegmentGraph.SegmentGraphVisitor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; class TarFiles implements Closeable { - private static class Node { + private static class Node implements Iterable { final TarReader reader; - final Node next; + private final Node next; Node(TarReader reader, Node next) { this.reader = reader; this.next = next; } + + Node reverse() { + Node reversed = null; + for (final TarReader r : this) { + reversed = new Node(r, reversed); + } + return reversed; + } + + @Override + public Iterator iterator() { + return new Iterator() { + + private Node next = Node.this; + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public TarReader next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + final Node current = next; + next = current.next; + return current.reader; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("not implemented"); + } + }; + } } static class CleanupResult { private boolean interrupted; private long reclaimedSize; @@ -158,23 +198,14 @@ class TarFiles implements Closeable { } private static final Logger log = LoggerFactory.getLogger(TarFiles.class); private static final Pattern FILE_NAME_PATTERN = Pattern.compile("(data)((0|[1-9][0-9]*)[0-9]{4})([a-z])?.tar"); - private static Node reverse(Node n) { - Node r = null; - while (n != null) { - r = new Node(n.reader, r); - n = n.next; - } - return r; - } - private static Map> collectFiles(File directory) { Map> dataFiles = newHashMap(); for (File file : listFiles(directory, null, false)) { Matcher matcher = FILE_NAME_PATTERN.matcher(file.getName()); if (matcher.matches()) { Integer index = Integer.parseInt(matcher.group(2)); Map files = dataFiles.get(index); @@ -192,21 +223,19 @@ class TarFiles implements Closeable { return dataFiles; } private static void includeForwardReferences(Node head, Set referencedIds) throws IOException { Set references = newHashSet(referencedIds); do { // Add direct forward references - Node n = head; - while (n != null) { - n.reader.calculateForwardReferences(references); + for (final TarReader reader : head) { + reader.calculateForwardReferences(references); if (references.isEmpty()) { break; // Optimisation: bail out if no references left } - n = n.next; } // ... as long as new forward references are found. } while (referencedIds.addAll(references)); } static Builder builder() { return new Builder(); @@ -225,15 +254,15 @@ class TarFiles implements Closeable { /** * Points to the first node of the linked list of TAR readers. Every node in * the linked list is immutable. Thus, you need to to hold {@link #lock} * while reading the value of the reference, but you can release it before * iterating through the list. */ - private Node readers; + private volatile Node readers; /** * The currently used TAR writer. Its access is protected by {@link #lock}. */ private TarWriter writer; /** @@ -276,171 +305,157 @@ class TarFiles implements Closeable { } @Override public void close() throws IOException { shutdown = true; TarWriter writer; - Node n; + Node head; lock.writeLock().lock(); try { writer = this.writer; - n = this.readers; + head = this.readers; } finally { lock.writeLock().unlock(); } IOException exception = null; if (writer != null) { try { writer.close(); } catch (IOException e) { exception = e; } } - while (n != null) { + for (final TarReader reader : head) { try { - n.reader.close(); + reader.close(); } catch (IOException e) { if (exception == null) { exception = e; } else { exception.addSuppressed(e); } } - n = n.next; } if (exception != null) { throw exception; } } @Override public String toString() { String w = null; - Node n; + Node head; lock.readLock().lock(); try { if (writer != null) { w = writer.toString(); } - n = readers; + head = readers; } finally { lock.readLock().unlock(); } - List rs = new ArrayList<>(); - while (n != null) { - rs.add(n.reader); - n = n.next; - } + List rs = newArrayList(head); return String.format("TarFiles{readers=%s,writer=%s", rs, w); } long size() { long size = 0; - Node readers; + Node head; lock.readLock().lock(); try { if (writer != null) { size = writer.fileLength(); } - readers = this.readers; + head = this.readers; } finally { lock.readLock().unlock(); } - Node n = readers; - while (n != null) { - size += n.reader.size(); - n = n.next; + for (final TarReader reader : head) { + size += reader.size(); } return size; } int readerCount() { - Node n; + Node head; lock.readLock().lock(); try { - n = readers; + head = readers; } finally { lock.readLock().unlock(); } - int count = 0; - while (n != null) { - count++; - n = n.next; - } - return count; + return Iterators.size(head.iterator()); } void flush() throws IOException { lock.readLock().lock(); try { writer.flush(); } finally { lock.readLock().unlock(); } } boolean containsSegment(long msb, long lsb) { - Node n; + Node head; lock.readLock().lock(); try { if (writer != null) { if (writer.containsEntry(msb, lsb)) { return true; } } - n = this.readers; + head = this.readers; } finally { lock.readLock().unlock(); } - while (n != null) { - if (n.reader.containsEntry(msb, lsb)) { + for (final TarReader reader : head) { + if (reader.containsEntry(msb, lsb)) { return true; } - n = n.next; } return false; } ByteBuffer readSegment(long msb, long lsb) { try { - Node n; + Node head; lock.readLock().lock(); try { if (writer != null) { ByteBuffer b = writer.readEntry(msb, lsb); if (b != null) { return b; } } - n = readers; + head = readers; } finally { lock.readLock().unlock(); } - while (n != null) { - ByteBuffer b = n.reader.readEntry(msb, lsb); + for (final TarReader reader : head) { + ByteBuffer b = reader.readEntry(msb, lsb); if (b != null) { return b; } - n = n.next; } } catch (IOException e) { log.warn("Unable to read from TAR file", e); } return null; } @@ -495,39 +510,37 @@ class TarFiles implements Closeable { CleanupResult cleanup(Supplier> referencesSupplier, Predicate reclaimPredicate) throws IOException { CleanupResult result = new CleanupResult(); result.removableFiles = new ArrayList<>(); result.reclaimedSegmentIds = new HashSet<>(); Set references; - Node readers; + Node head; lock.writeLock().lock(); lock.readLock().lock(); ; try { try { newWriter(); } finally { lock.writeLock().unlock(); } - readers = this.readers; + head = this.readers; references = referencesSupplier.get(); } finally { lock.readLock().unlock(); } Map cleaned = new LinkedHashMap<>(); { - Node n = readers; - while (n != null) { - cleaned.put(n.reader, n.reader); - result.reclaimedSize += n.reader.size(); - n = n.next; + for (final TarReader reader : head) { + cleaned.put(reader, reader); + result.reclaimedSize += reader.size(); } } Set reclaim = newHashSet(); for (TarReader reader : cleaned.keySet()) { if (shutdown) { @@ -549,164 +562,148 @@ class TarFiles implements Closeable { long reclaimed; while (true) { closeables = null; reclaimed = 0; Node swept = null; - Node n = readers; // The following loops creates a modified version of `readers` and // saves it into `swept`. Some TAR readers in `readers` have been // swept by the previous code and must be replaced with a slimmer // TAR reader with the same index but a higher generation. - while (n != null) { - if (cleaned.containsKey(n.reader)) { + for (final TarReader reader : head) { + if (cleaned.containsKey(reader)) { // We distinguish three cases. First, the original TAR // reader is unmodified. This happens with no content or not // enough content could be swept from the original TAR // reader. Second, some content could be swept from the // original TAR reader and a new TAR reader with the same // index and a higher generation was created. Third, all the // content from the original TAR reader could be swept. - TarReader r = cleaned.get(n.reader); - - if (r != null) { + TarReader cleandedReader = cleaned.get(reader); + if (cleandedReader != null) { // We are either in the first or in the second case. // Save the TAR reader (either the original or the one // with a higher generation) in the resulting linked list. - swept = new Node(r, swept); - reclaimed += r.size(); + swept = new Node(cleandedReader, swept); + reclaimed += cleandedReader.size(); } - - if (r != n.reader) { - - // We are either in the second or third case. Save the - // original TAR reader in a list of TAR readers that - // will be closed at the end of this methods. - - closeables = new Node(n.reader, closeables); + if (cleandedReader != reader) { + closeables = new Node(reader, closeables); } } else { - // This reader was not involved in the mark-and-sweep. This - // might happen in iterations of this loop successive to the - // first, when we re-read `readers` and recompute `swept` - // all over again. + // We are either in the second or third case. Save the + // original TAR reader in a list of TAR readers that + // will be closed at the end of this methods. - swept = new Node(n.reader, swept); + swept = new Node(reader, swept); } - n = n.next; } // `swept` is in the reverse order because we prepended new nodes // to it. We have to reverse it before we save it into `readers`. - swept = reverse(swept); + swept = swept == null ? null : swept.reverse(); // Following is a compare-and-set operation. We based the // computation of `swept` of a specific value of `readers`. If // `readers` is still the same as the one we started with, we just // update `readers` and exit from the loop. Otherwise, we read the // value of `readers` and recompute `swept` based on this value. lock.writeLock().lock(); try { - if (this.readers == readers) { + if (this.readers == head) { this.readers = swept; break; } else { - readers = this.readers; + head = this.readers; } } finally { lock.writeLock().unlock(); } } result.reclaimedSize -= reclaimed; - { - Node n = closeables; - while (n != null) { + if (closeables != null) { + for (final TarReader closeable : closeables) { try { - n.reader.close(); + closeable.close(); } catch (IOException e) { log.warn("Unable to close swept TAR reader", e); } - result.removableFiles.add(n.reader.getFile()); - n = n.next; + result.removableFiles.add(closeable.getFile()); } } return result; } void collectBlobReferences(ReferenceCollector collector, int minGeneration) throws IOException { - Node n; + Node head; lock.writeLock().lock(); try { if (writer != null) { newWriter(); } - n = readers; + head = readers; } finally { lock.writeLock().unlock(); } - while (n != null) { - n.reader.collectBlobReferences(collector, minGeneration); - n = n.next; + for (final TarReader reader : head) { + reader.collectBlobReferences(collector, minGeneration); } } Iterable getSegmentIds() { - Node n; + Node head; lock.readLock().lock(); try { - n = readers; + head = readers; } finally { lock.readLock().unlock(); } List ids = new ArrayList<>(); - while (n != null) { - ids.addAll(n.reader.getUUIDs()); - n = n.next; + for (final TarReader reader : head) { + ids.addAll(reader.getUUIDs()); } return ids; } Map> getGraph(String fileName) throws IOException { - Node n; + Node head; lock.readLock().lock(); try { - n = readers; + head = readers; } finally { lock.readLock().unlock(); } Set index = null; Map> graph = null; - while (n != null) { - TarReader r = n.reader; - if (fileName.equals(r.getFile().getName())) { - index = r.getUUIDs(); - graph = r.getGraph(false); + for (final TarReader reader : head) { + if (fileName.equals(reader.getFile().getName())) { + index = reader.getUUIDs(); + graph = reader.getGraph(false); break; } - n = n.next; } Map> result = new HashMap<>(); if (index != null) { for (UUID uuid : index) { result.put(uuid, null); } @@ -714,42 +711,40 @@ class TarFiles implements Closeable { if (graph != null) { result.putAll(graph); } return result; } Map> getIndices() { - Node n; + Node head; lock.readLock().lock(); try { - n = readers; + head = readers; } finally { lock.readLock().unlock(); } Map> index = new HashMap<>(); - while (n != null) { - index.put(n.reader.getFile().getAbsolutePath(), n.reader.getUUIDs()); - n = n.next; + for (final TarReader reader : head) { + index.put(reader.getFile().getAbsolutePath(), reader.getUUIDs()); } return index; } void traverseSegmentGraph(Set roots, SegmentGraphVisitor visitor) throws IOException { - Node n; + Node head; lock.readLock().lock(); try { - n = readers; + head = readers; } finally { lock.readLock().unlock(); } - includeForwardReferences(n, roots); - while (n != null) { - n.reader.traverseSegmentGraph(roots, visitor); - n = n.next; + includeForwardReferences(head, roots); + for (final TarReader reader : head) { + reader.traverseSegmentGraph(roots, visitor); } } }