Index: main/java/org/apache/jackrabbit/core/data/CachingFDS.java =================================================================== --- main/java/org/apache/jackrabbit/core/data/CachingFDS.java (revision 0) +++ main/java/org/apache/jackrabbit/core/data/CachingFDS.java (working copy) @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * {@link CachingDataStore} with {@link FSBackend}. It is performant + * {@link DataStore} when {@link FSBackend} is hosted on network storage + * (SAN or NAS). It leverages all caching capabilites of + * {@link CachingDataStore}. + */ +package org.apache.jackrabbit.core.data; + +import java.util.Properties; + +public class CachingFDS extends CachingDataStore { + private Properties properties; + + @Override + protected Backend createBackend() { + FSBackend backend = new FSBackend(); + if (properties != null) { + backend.setProperties(properties); + } + return backend; + } + + @Override + protected String getMarkerFile() { + return "fs.init.done"; + } + + /** + * Properties required to configure the S3Backend + */ + public void setProperties(Properties properties) { + this.properties = properties; + } +} Property changes on: main/java/org/apache/jackrabbit/core/data/CachingFDS.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: main/java/org/apache/jackrabbit/core/data/FSBackend.java =================================================================== --- main/java/org/apache/jackrabbit/core/data/FSBackend.java (revision 0) +++ main/java/org/apache/jackrabbit/core/data/FSBackend.java (working copy) @@ -0,0 +1,492 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * File system {@link Backend} used with {@link CachingDataStore}. + * The file system can be network storage. + */ +package org.apache.jackrabbit.core.data; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.jackrabbit.core.data.util.NamedThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FSBackend implements Backend { + + private Properties properties; + + private String path; + + private CachingDataStore store; + + private String homeDir; + + private String config; + + File pathDir; + + private ThreadPoolExecutor asyncWriteExecuter; + + public static final String PATH = "path"; + + /** + * Logger instance. + */ + private static final Logger LOG = LoggerFactory.getLogger(FSBackend.class); + + /** + * The maximum last modified time resolution of the file system. + */ + private static final int ACCESS_TIME_RESOLUTION = 2000; + + @Override + public void init(CachingDataStore store, String homeDir, String config) + throws DataStoreException { + Properties initProps = null; + // Check is configuration is already provided. That takes precedence + // over config provided via file based config + this.config = config; + if (this.properties != null) { + initProps = this.properties; + } else { + initProps = new Properties(); + InputStream in = null; + try { + in = new FileInputStream(config); + initProps.load(in); + } catch (IOException e) { + throw new DataStoreException( + "Could not initialize FSBackend from " + config, e); + } finally { + IOUtils.closeQuietly(in); + } + this.properties = initProps; + } + init(store, homeDir, initProps); + + } + + public void init(CachingDataStore store, String homeDir, Properties prop) + throws DataStoreException { + this.store = store; + this.homeDir = homeDir; + this.path = prop.getProperty(PATH); + if (this.path == null || "".equals(this.path)) { + throw new DataStoreException("Could not initialize FSBackend from " + + config + ". [" + PATH + "] property not found."); + } + pathDir = new File(this.path); + if (pathDir.exists() && pathDir.isFile()) { + throw new DataStoreException("Can not create a directory " + + "because a file exists with the same name: " + this.path); + } + boolean created = pathDir.mkdirs(); + if (!created) { + throw new DataStoreException("Could not create directory: " + + pathDir.getAbsolutePath()); + } + asyncWriteExecuter = (ThreadPoolExecutor) Executors.newFixedThreadPool( + 10, new NamedThreadFactory("fs-write-worker")); + + } + + @Override + public InputStream read(DataIdentifier identifier) + throws DataStoreException { + File file = getFile(identifier); + try { + return new LazyFileInputStream(file); + } catch (IOException e) { + throw new DataStoreException("Error opening input stream of " + + file.getAbsolutePath(), e); + } + } + + @Override + public long getLength(DataIdentifier identifier) throws DataStoreException { + File file = getFile(identifier); + if (file.isFile()) { + return file.length(); + } + throw new DataStoreException("Could not length of dataIdentifier [" + + identifier + "]"); + } + + @Override + public long getLastModified(DataIdentifier identifier) + throws DataStoreException { + long start = System.currentTimeMillis(); + File f = getFile(identifier); + if (f.isFile()) { + return getLastModified(f); + } + LOG.info("getLastModified:Identifier [{}] not found. Took [{}] ms.", + identifier, (System.currentTimeMillis() - start)); + throw new DataStoreException("Identifier [" + identifier + + "] not found."); + } + + @Override + public void write(DataIdentifier identifier, File src) + throws DataStoreException { + File dest = getFile(identifier); + synchronized (this) { + if (dest.exists()) { + long now = System.currentTimeMillis(); + if (getLastModified(dest) < now + ACCESS_TIME_RESOLUTION) { + setLastModified(dest, now + ACCESS_TIME_RESOLUTION); + } + } else { + try { + FileUtils.copyFile(src, dest); + } catch (IOException ioe) { + throw new DataStoreException("Not able to write file [" + + identifier + "]"); + } + } + } + + } + + @Override + public void writeAsync(final DataIdentifier identifier, final File src, + final AsyncUploadCallback callback) + throws DataStoreException { + if (callback == null) { + throw new IllegalArgumentException( + "callback parameter cannot be null in asyncUpload"); + } + asyncWriteExecuter.execute(new Runnable() { + @Override + public void run() { + try { + write(identifier, src); + callback.onSuccess(new AsyncUploadResult(identifier, src)); + } catch (DataStoreException dse) { + AsyncUploadResult res = new AsyncUploadResult(identifier, + src); + res.setException(dse); + callback.onFailure(res); + } + + } + }); + } + + @Override + public Iterator getAllIdentifiers() + throws DataStoreException { + ArrayList files = new ArrayList(); + for (File file : pathDir.listFiles()) { + if (file.isDirectory()) { // skip top-level files + listRecursive(files, file); + } + } + + ArrayList identifiers = new ArrayList(); + for (File f : files) { + String name = f.getName(); + identifiers.add(new DataIdentifier(name)); + } + LOG.debug("Found " + identifiers.size() + " identifiers."); + return identifiers.iterator(); + } + + @Override + public boolean exists(DataIdentifier identifier, boolean touch) + throws DataStoreException { + File file = getFile(identifier); + if (file.isFile()) { + if (touch) { + long now = System.currentTimeMillis(); + setLastModified(file, now + ACCESS_TIME_RESOLUTION); + } + return true; + } + return false; + } + + @Override + public boolean exists(DataIdentifier identifier) throws DataStoreException { + return exists(identifier, false); + } + + @Override + public void touch(DataIdentifier identifier, long minModifiedDate) + throws DataStoreException { + File file = getFile(identifier); + long now = System.currentTimeMillis(); + if (minModifiedDate > 0 && minModifiedDate > getLastModified(file)) { + setLastModified(file, now + ACCESS_TIME_RESOLUTION); + } + } + + @Override + public void touchAsync(final DataIdentifier identifier, + final long minModifiedDate, + final AsyncTouchCallback callback) + throws DataStoreException { + try { + if (callback == null) { + throw new IllegalArgumentException( + "callback parameter cannot be null in touchAsync"); + } + Thread.currentThread().setContextClassLoader( + getClass().getClassLoader()); + + asyncWriteExecuter.execute(new Runnable() { + @Override + public void run() { + try { + touch(identifier, minModifiedDate); + callback.onSuccess(new AsyncTouchResult(identifier)); + } catch (DataStoreException e) { + AsyncTouchResult result = new AsyncTouchResult( + identifier); + result.setException(e); + callback.onFailure(result); + } + } + }); + } catch (Exception e) { + callback.onAbort(new AsyncTouchResult(identifier)); + throw new DataStoreException("Cannot touch the record " + + identifier.toString(), e); + } + + } + + @Override + public void close() throws DataStoreException { + asyncWriteExecuter.shutdownNow(); + + } + + @Override + public Set deleteAllOlderThan(long min) + throws DataStoreException { + Set deleteIdSet = new HashSet(30); + for (File file : pathDir.listFiles()) { + if (file.isDirectory()) { // skip top-level files + deleteOlderRecursive(file, min, deleteIdSet); + } + } + return deleteIdSet; + } + + @Override + public void deleteRecord(DataIdentifier identifier) + throws DataStoreException { + File file = getFile(identifier); + synchronized (this) { + if (file.exists()) { + if (file.delete()) { + deleteEmptyParentDirs(file); + } else { + LOG.warn("Failed to delete file " + file.getAbsolutePath()); + } + } + } + } + + /** + * Properties used to configure the backend. If provided explicitly before + * init is invoked then these take precedence + * @param properties to configure S3Backend + */ + public void setProperties(Properties properties) { + this.properties = properties; + } + + /** + * Returns the identified file. This method implements the pattern used to + * avoid problems with too many files in a single directory. + *

+ * No sanity checks are performed on the given identifier. + * @param identifier data identifier + * @return identified file + */ + private File getFile(DataIdentifier identifier) { + String string = identifier.toString(); + File file = this.pathDir; + file = new File(file, string.substring(0, 2)); + file = new File(file, string.substring(2, 4)); + file = new File(file, string.substring(4, 6)); + return new File(file, string); + } + + /** + * Set the last modified date of a file, if the file is writable. + * @param file the file + * @param time the new last modified date + * @throws DataStoreException if the file is writable but modifying the date + * fails + */ + private static void setLastModified(File file, long time) + throws DataStoreException { + if (!file.setLastModified(time)) { + if (!file.canWrite()) { + // if we can't write to the file, so garbage collection will + // also not delete it + // (read only files or file systems) + return; + } + try { + // workaround for Windows: if the file is already open for + // reading + // (in this or another process), then setting the last modified + // date + // doesn't work - see also JCR-2872 + RandomAccessFile r = new RandomAccessFile(file, "rw"); + try { + r.setLength(r.length()); + } finally { + r.close(); + } + } catch (IOException e) { + throw new DataStoreException( + "An IO Exception occurred while trying to set the last modified date: " + + file.getAbsolutePath(), e); + } + } + } + + /** + * Get the last modified date of a file. + * @param file the file + * @return the last modified date + * @throws DataStoreException if reading fails + */ + private static long getLastModified(File file) throws DataStoreException { + long lastModified = file.lastModified(); + if (lastModified == 0) { + throw new DataStoreException( + "Failed to read record modified date: " + + file.getAbsolutePath()); + } + return lastModified; + } + + private void listRecursive(List list, File file) { + File[] files = file.listFiles(); + if (files != null) { + for (File f : files) { + if (f.isDirectory()) { + listRecursive(list, f); + } else { + list.add(f); + } + } + } + } + + private void deleteEmptyParentDirs(File file) { + File parent = file.getParentFile(); + try { + // Only iterate & delete if parent directory of the blob file is + // child + // of the base directory and if it is empty + while (FileUtils.directoryContains(pathDir, parent)) { + String[] entries = parent.list(); + if (entries == null) { + LOG.warn("Failed to list directory {}", + parent.getAbsolutePath()); + break; + } + if (entries.length > 0) { + break; + } + boolean deleted = parent.delete(); + LOG.debug("Deleted parent [{}] of file [{}]: {}", new Object[] { + parent, file.getAbsolutePath(), deleted }); + parent = parent.getParentFile(); + } + } catch (IOException e) { + LOG.warn("Error in parents deletion for " + file.getAbsoluteFile(), + e); + } + } + + private void deleteOlderRecursive(File file, long min, + Set deleteIdSet) throws DataStoreException { + if (file.isFile() && file.exists() && file.canWrite()) { + synchronized (this) { + long lastModified; + try { + lastModified = getLastModified(file); + } catch (DataStoreException e) { + LOG.warn( + "Failed to read modification date; file not deleted", e); + // don't delete the file, since the lastModified date is + // uncertain + lastModified = min; + } + if (lastModified < min) { + DataIdentifier id = new DataIdentifier(file.getName()); + if (store.confirmDelete(id)) { + store.deleteFromCache(id); + if (LOG.isInfoEnabled()) { + LOG.info("Deleting old file " + + file.getAbsolutePath() + " modified: " + + new Timestamp(lastModified).toString() + + " length: " + file.length()); + } + if (file.delete()) { + deleteIdSet.add(id); + } else { + LOG.warn("Failed to delete old file " + + file.getAbsolutePath()); + } + } + } + } + } else if (file.isDirectory()) { + File[] list = file.listFiles(); + if (list != null) { + for (File f : list) { + deleteOlderRecursive(f, min, deleteIdSet); + } + } + + // JCR-1396: FileDataStore Garbage Collector and empty directories + // Automatic removal of empty directories (but not the root!) + synchronized (this) { + list = file.listFiles(); + if (list != null && list.length == 0) { + file.delete(); + } + } + } + } + +} Property changes on: main/java/org/apache/jackrabbit/core/data/FSBackend.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: test/java/org/apache/jackrabbit/core/data/TestCachingFDS.java =================================================================== --- test/java/org/apache/jackrabbit/core/data/TestCachingFDS.java (revision 0) +++ test/java/org/apache/jackrabbit/core/data/TestCachingFDS.java (working copy) @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.core.data; + +import java.util.Properties; + +import javax.jcr.RepositoryException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestCachingFDS extends TestFileDataStore { + + protected static final Logger LOG = LoggerFactory.getLogger(TestCachingFDS.class); + + protected DataStore createDataStore() throws RepositoryException { + CachingFDS cacheFDS = new CachingFDS(); + Properties props = loadProperties("/fs.properties"); + String pathValue = props.getProperty("path"); + if (props != null && !"".equals(pathValue.trim())) { + path = pathValue + "/cachingFds" + "-" + + String.valueOf(randomGen.nextInt(100000)) + "-" + + String.valueOf(randomGen.nextInt(100000)); + } else { + path = dataStoreDir + "/cachingFds"; + } + props.setProperty("path", path); + LOG.info("path [{}] set.", path); + cacheFDS.setProperties(props); + cacheFDS.setSecret("12345"); + cacheFDS.init(dataStoreDir); + return cacheFDS; + } + +} Property changes on: test/java/org/apache/jackrabbit/core/data/TestCachingFDS.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: test/java/org/apache/jackrabbit/core/data/TestCachingFDSCacheOff.java =================================================================== --- test/java/org/apache/jackrabbit/core/data/TestCachingFDSCacheOff.java (revision 0) +++ test/java/org/apache/jackrabbit/core/data/TestCachingFDSCacheOff.java (working copy) @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jackrabbit.core.data; + +import java.util.Properties; + +import javax.jcr.RepositoryException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestCachingFDSCacheOff extends TestFileDataStore { + + protected static final Logger LOG = LoggerFactory.getLogger(TestCachingFDS.class); + + protected DataStore createDataStore() throws RepositoryException { + CachingFDS cacheFDS = new CachingFDS(); + Properties props = loadProperties("/fs.properties"); + String pathValue = props.getProperty("path"); + if (props != null && !"".equals(pathValue.trim())) { + path = pathValue + "/cachingFds" + "-" + + String.valueOf(randomGen.nextInt(100000)) + "-" + + String.valueOf(randomGen.nextInt(100000)); + } else { + path = dataStoreDir + "/cachingFDS"; + } + props.setProperty("path", path); + cacheFDS.setProperties(props); + cacheFDS.setSecret("12345"); + cacheFDS.setCacheSize(0); + cacheFDS.init(dataStoreDir); + return cacheFDS; + } +} Property changes on: test/java/org/apache/jackrabbit/core/data/TestCachingFDSCacheOff.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: test/java/org/apache/jackrabbit/core/data/TestCaseBase.java =================================================================== --- test/java/org/apache/jackrabbit/core/data/TestCaseBase.java (revision 1671263) +++ test/java/org/apache/jackrabbit/core/data/TestCaseBase.java (working copy) @@ -22,10 +22,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.SequenceInputStream; +import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Properties; import java.util.Random; import javax.jcr.RepositoryException; @@ -663,4 +665,18 @@ } } } + + /** + * Return {@link Properties} from class resource. Return empty + * {@link Properties} if not found. + */ + protected Properties loadProperties(String resource) { + Properties configProp = new Properties(); + try { + configProp.load(this.getClass().getResourceAsStream(resource)); + } catch (Exception ignore) { + + } + return configProp; + } } Index: test/java/org/apache/jackrabbit/core/data/TestFileDataStore.java =================================================================== --- test/java/org/apache/jackrabbit/core/data/TestFileDataStore.java (revision 1671263) +++ test/java/org/apache/jackrabbit/core/data/TestFileDataStore.java (working copy) @@ -18,6 +18,7 @@ package org.apache.jackrabbit.core.data; import java.io.File; +import java.util.Properties; import javax.jcr.RepositoryException; @@ -32,12 +33,21 @@ protected static final Logger LOG = LoggerFactory.getLogger(TestFileDataStore.class); - String path; + protected String path; @Override protected DataStore createDataStore() throws RepositoryException { FileDataStore fds = new FileDataStore(); - path = dataStoreDir + "/repository/datastore"; + Properties props = loadProperties("/fs.properties"); + String pathValue = props.getProperty("path"); + if (props != null && !"".equals(pathValue.trim())) { + path = pathValue + "/fds" + "-" + + String.valueOf(randomGen.nextInt(100000)) + "-" + + String.valueOf(randomGen.nextInt(100000)); + } else { + path = dataStoreDir + "/repository/datastore"; + } + LOG.info("path [{}] set.", path); fds.setPath(path); fds.init(dataStoreDir); return fds; @@ -45,6 +55,7 @@ @Override protected void tearDown() { + LOG.info("cleaning path [{}]", path); File f = new File(path); try { for (int i = 0; i < 4 && f.exists(); i++) { @@ -54,6 +65,7 @@ } catch (Exception ignore) { } + super.tearDown(); } } Index: test/java/org/apache/jackrabbit/core/data/TestInMemDs.java =================================================================== --- test/java/org/apache/jackrabbit/core/data/TestInMemDs.java (revision 1671263) +++ test/java/org/apache/jackrabbit/core/data/TestInMemDs.java (working copy) @@ -28,7 +28,7 @@ protected static final Logger LOG = LoggerFactory.getLogger(TestInMemDs.class); - + @Override protected DataStore createDataStore() throws RepositoryException { InMemoryDataStore inMemDS = new InMemoryDataStore(); inMemDS.setProperties(null); Index: test/resources/fs.properties =================================================================== --- test/resources/fs.properties (revision 0) +++ test/resources/fs.properties (working copy) @@ -0,0 +1 @@ +path= Property changes on: test/resources/fs.properties ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property