Uploaded image for project: 'Commons VFS'
  1. Commons VFS
  2. VFS-62

[vfs] [PATCH] Added modulus based delay to DefaultFileMonitor for better performance when monitoring large number of files

    XMLWordPrintableJSON

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Major
    • Resolution: Fixed
    • Affects Version/s: None
    • Fix Version/s: None
    • Labels:
      None
    • Environment:

      Operating System: other
      Platform: Other

    • Bugzilla Id:
      34735

      Description

      /*

      • Copyright 2002, 2003,2004 The Apache Software Foundation.
        *
      • Licensed under the Apache License, Version 2.0 (the "License");
      • you may not use this file except in compliance with the License.
      • You may obtain a copy of the License at
        *
      • http://www.apache.org/licenses/LICENSE-2.0
        *
      • Unless required by applicable law or agreed to in writing, software
      • distributed under the License is distributed on an "AS IS" BASIS,
      • WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      • See the License for the specific language governing permissions and
      • limitations under the License.
        */
        package org.apache.commons.vfs.impl;

      import org.apache.commons.logging.Log;
      import org.apache.commons.logging.LogFactory;
      import org.apache.commons.vfs.FileListener;
      import org.apache.commons.vfs.FileMonitor;
      import org.apache.commons.vfs.FileName;
      import org.apache.commons.vfs.FileObject;
      import org.apache.commons.vfs.FileSystemException;
      import org.apache.commons.vfs.FileType;
      import org.apache.commons.vfs.provider.AbstractFileSystem;

      import java.util.HashMap;
      import java.util.Map;
      import java.util.Stack;

      /**

      • A polling {@link FileMonitor}

        implementation.<br />

      • <br />
      • The DefaultFileMonitor is a Thread based polling file system monitor with a 1
        second delay.<br />
      • <br />
      • <b>Design:</b>
      • <p/>
      • There is a Map of monitors known as FileMonitorAgents. With the thread running,
      • each FileMonitorAgent object is asked to "check" on the file it is
        responsible for.
      • To do this check, the cache is cleared.
      • </p>
      • <ul>
      • <li>If the file existed before the refresh and it no longer exists, a delete
        event is fired.</li>
      • <li>If the file existed before the refresh and it still exists, check the
        last modified timestamp to see if that has changed.</li>
      • <li>If it has, fire a change event.</li>
      • </ul>
      • <p/>
      • With each file delete, the FileMonitorAgent of the parent is asked to
        re-build its
      • list of children, so that they can be accurately checked when there are new
        children.<br/>
      • New files are detected during each "check" as each file does a check for new
        children.
      • If new children are found, create events are fired recursively if recursive
        descent is
      • enabled.
      • </p>
      • <p>
      • For performance reasons, added a delay that increases as the number of files
        monitored
      • increases. The default is a delay of 1 second for every 1000 files processed.
      • </p>
      • <p/>
      • <br /><b>Example usage:</b><pre>
      • FileSystemManager fsManager = VFS.getManager();
      • FileObject listendir = fsManager.resolveFile("/home/username/monitored/");
      • <p/>
      • DefaultFileMonitor fm = new DefaultFileMonitor(new CustomFileListener());
      • fm.setRecursive(true);
      • fm.addFile(listendir);
      • fm.start();
      • </pre>
      • <i>(where CustomFileListener is a class that implements the FileListener
        interface.)</i>
        *
      • @author <a href="mailto:xknight@users.sourceforge.net">Christopher Ottley</a>
      • @version $Revision: 1.4 $ $Date: 2004/12/29 19:47:34 $
        */
        public class DefaultFileMonitor implements Runnable, FileMonitor
        {
        private final static Log log = LogFactory.getLog(DefaultFileMonitor.class);

      /**

      • Map from FileName to FileObject being monitored.
        */
        private final Map monitorMap = new HashMap();

      /**

      • The low priority thread used for checking the files being monitored.
        */
        private Thread monitorThread;

      /**

      • File objects to be removed from the monitor map.
        */
        private Stack deleteStack = new Stack();

      /**

      • File objects to be added to the monitor map.
        */
        private Stack addStack = new Stack();

      /**

      • A flag used to determine if the monitor thread should be running.
        */
        private boolean shouldRun = true;

      /**

      • A flag used to determine if adding files to be monitored should be recursive.
        */
        private boolean recursive = false;

      /**

      • Set the delay between checks
        */
        private long delay = 1000;

      /**

      • A listener object that if set, is notified on file creation and deletion.
        */
        private final FileListener listener;

      public DefaultFileMonitor(final FileListener listener)

      { this.listener = listener; }

      /**

      • Access method to get the recursive setting when adding files for monitoring.
        */
        public boolean isRecursive() { return this.recursive; }

      /**

      • Access method to set the recursive setting when adding files for monitoring.
        */
        public void setRecursive(final boolean newRecursive) { this.recursive = newRecursive; }

      /**

      • Access method to get the current FileListener object notified when there
      • are changes with the files added.
        */
        FileListener getFileListener() { return this.listener; }

      /**

      • Adds a file to be monitored.
        */
        public void addFile(final FileObject file)
        {
        synchronized (this.monitorMap)
        {
        if (this.monitorMap.get(file.getName()) == null)
        {
        this.monitorMap.put(file.getName(), new FileMonitorAgent(this,
        file));

      try
      {
      if (this.listener != null)

      { file.getFileSystem().addListener(file, this.listener); }

      if (file.getType().hasChildren() && this.recursive)
      {
      // Traverse the children
      final FileObject[] children = file.getChildren();
      for (int i = 0; i < children.length; i++)

      { this.addFile(children[i]); // Add depth first }

      }

      }
      catch (FileSystemException fse)

      { log.error(fse.getLocalizedMessage(), fse); }

      }
      }
      }

      /**

      • Removes a file from being monitored.
        */
        public void removeFile(final FileObject file)
        {
        synchronized (this.monitorMap)
        {
        FileName fn = file.getName();
        if (this.monitorMap.get(fn) != null)
        {
        FileObject parent;
        try { parent = file.getParent(); }

        catch (FileSystemException fse)

        { parent = null; }

      this.monitorMap.remove(fn);

      if (parent != null)
      { // Not the root
      FileMonitorAgent parentAgent =
      (FileMonitorAgent) this.monitorMap.get(parent.getName());
      if (parentAgent != null)

      { parentAgent.resetChildrenList(); }

      }
      }
      }
      }

      /**

      • Queues a file for removal from being monitored.
        */
        protected void queueRemoveFile(final FileObject file) { this.deleteStack.push(file); }

      /**

      • Get the delay between runs
        */
        public long getDelay() { return delay; }

      /**

      • Set the delay between runs
        */
        public void setDelay(long delay)
        Unknown macro: { if (delay > 0) { this.delay = delay; } else { this.delay = 1000; } }

      /**

      • Queues a file for addition to be monitored.
        */
        protected void queueAddFile(final FileObject file) { this.addStack.push(file); }

      /**

      • Starts monitoring the files that have been added.
        */
        public void start()
        Unknown macro: { if (this.monitorThread == null) { this.monitorThread = new Thread(this); this.monitorThread.setDaemon(true); this.monitorThread.setPriority(Thread.MIN_PRIORITY); } this.monitorThread.start(); }

      /**

      • Stops monitoring the files that have been added.
        */
        public void stop() { this.shouldRun = false; }

      /**

      • Asks the agent for each file being monitored to check its file for changes.
        */
        public void run()
        {
        mainloop: while (!Thread.currentThread().isInterrupted() && this.shouldRun)
        {
        while (!this.deleteStack.empty()) { this.removeFile((FileObject) this.deleteStack.pop()); }

      // For each entry in the map
      Object fileNames[];
      synchronized (this.monitorMap)

      { fileNames = this.monitorMap.keySet().toArray(); }

      for (int iterFileNames = 0; iterFileNames < fileNames.length;
      iterFileNames++)
      {
      FileName fileName = (FileName) fileNames[iterFileNames];
      FileMonitorAgent agent;
      synchronized (this.monitorMap)

      { agent = (FileMonitorAgent) this.monitorMap.get(fileName); }

      if (agent != null)

      { agent.check(); }

      if ((iterFileNames % getDelay()) == 0) {
      try

      { Thread.sleep(getDelay()); }

      catch (InterruptedException e)
      {

      }
      }

      if (Thread.currentThread().isInterrupted() || !this.shouldRun)

      { continue mainloop; }

      }

      while (!this.addStack.empty())

      { this.addFile((FileObject) this.addStack.pop()); }

      try

      { Thread.sleep(getDelay()); }

      catch (InterruptedException e)

      { continue; }

      }

      this.shouldRun = true;
      }

      /**

      • File monitor agent.
        */
        private static class FileMonitorAgent
        {
        private final FileObject file;
        private final DefaultFileMonitor fm;

      private boolean exists;
      private long timestamp;
      private Map children = null;

      private FileMonitorAgent(DefaultFileMonitor fm, FileObject file)
      {
      this.fm = fm;
      this.file = file;

      this.refresh();
      this.resetChildrenList();

      try

      { this.exists = this.file.exists(); }

      catch (FileSystemException fse)

      { this.exists = false; }

      try

      { this.timestamp = this.file.getContent().getLastModifiedTime(); }

      catch (FileSystemException fse)

      { this.timestamp = -1; }

      }

      private void resetChildrenList()
      {
      try
      {
      if (this.file.getType() == FileType.FOLDER)
      {
      this.children = new HashMap();
      FileObject[] childrenList = this.file.getChildren();
      for (int i = 0; i < childrenList.length; i++)

      { this.children.put(childrenList[i].getName(), new Object()); // null? }

      }
      }
      catch (FileSystemException fse)

      { this.children = null; }

      }

      /**

      • Clear the cache and re-request the file object
        */
        private void refresh()
        Unknown macro: { try { // this.file = ((AbstractFileSystem) this.file.getFileSystem()).resolveFile(this.file.getName(), false); // close the file - this will detach and reattach its resources (for this thread) on the // next access this.file.close(); } catch (FileSystemException fse) { log.error(fse.getLocalizedMessage(), fse); }
        }


        /**
        * Recursively fires create events for all children if recursive descent is
        * enabled. Otherwise the create event is only fired for the initial
        * FileObject.
        */
        private void fireAllCreate(FileObject child)
        {
        // Add listener so that it can be triggered
        if (this.fm.getFileListener() != null)
        { child.getFileSystem().addListener(child, this.fm.getFileListener()); }

        ((AbstractFileSystem) child.getFileSystem()).fireFileCreated(child);

        // Remove it because a listener is added in the queueAddFile
        if (this.fm.getFileListener() != null)
        { child.getFileSystem().removeListener(child, this.fm.getFileListener()); }

        this.fm.queueAddFile(child); // Add

        try
        {

        if (this.fm.isRecursive())
        {
        if (child.getType() == FileType.FOLDER)
        {
        FileObject[] newChildren = child.getChildren();
        for (int i = 0; i < newChildren.length; i++)
        { fireAllCreate(newChildren[i]); }
        }
        }

        }
        catch (FileSystemException fse)
        { log.error(fse.getLocalizedMessage(), fse); } }

      /**

      • Only checks for new children. If children are removed, they'll
      • eventually be checked.
        */
        private void checkForNewChildren()
        {
        try
        {
        if (this.file.getType() == FileType.FOLDER)
        {
        FileObject[] newChildren = this.file.getChildren();
        if (this.children != null)
        {
        // See which new children are not listed in the current
        children map.
        Map newChildrenMap = new HashMap();
        Stack missingChildren = new Stack();

      for (int i = 0; i < newChildren.length; i++)
      {
      newChildrenMap.put(newChildren[i].getName(), new
      Object()); // null ?
      // If the child's not there
      if
      (!this.children.containsKey(newChildren[i].getName()))

      { missingChildren.push(newChildren[i]); }

      }

      this.children = newChildrenMap;

      // If there were missing children
      if (!missingChildren.empty())
      {

      while (!missingChildren.empty())

      { FileObject child = (FileObject) missingChildren.pop(); this.fireAllCreate(child); }

      }

      }
      else
      {
      // First set of children - Break out the cigars
      if (newChildren.length > 0)

      { this.children = new HashMap(); }

      for (int i = 0; i < newChildren.length; i++)

      { this.children.put(newChildren[i].getName(), new Object()); // null? this.fireAllCreate(newChildren[i]); }

      }
      }
      }
      catch (FileSystemException fse)

      { log.error(fse.getLocalizedMessage(), fse); }
      }

      private void check()
      {
      this.refresh();

      try
      {
      // If the file existed and now doesn't
      if (this.exists && !this.file.exists())
      {
      this.exists = this.file.exists();
      this.timestamp = -1;

      // Fire delete event

      ((AbstractFileSystem)
      this.file.getFileSystem()).fireFileDeleted(this.file);

      // Remove listener in case file is re-created. Don't want to
      fire twice.
      if (this.fm.getFileListener() != null)
      { this.file.getFileSystem().removeListener(this.file, this.fm.getFileListener()); }

      // Remove from map
      this.fm.queueRemoveFile(this.file);
      }
      else if (this.exists && this.file.exists())
      {

      // Check the timestamp to see if it has been modified
      if (this.timestamp !=
      this.file.getContent().getLastModifiedTime())
      {
      this.timestamp =
      this.file.getContent().getLastModifiedTime();
      // Fire change event

      // Don't fire if it's a folder because new file children
      // and deleted files in a folder have their own event
      triggered.
      if (this.file.getType() != FileType.FOLDER)
      { ((AbstractFileSystem) this.file.getFileSystem()).fireFileChanged(this.file); }
      }

      }

      this.checkForNewChildren();

      }
      catch (FileSystemException fse)
      { log.error(fse.getLocalizedMessage(), fse); }

      }

      }

      }

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                Unassigned
                Reporter:
                xknight@users.sourceforge.net Christopher Ottley
              • Votes:
                0 Vote for this issue
                Watchers:
                0 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: