[VFS-4] [vfs][PATCH] External File Monitor

• Type: Bug
• Status: Closed
• Priority: Major
• Resolution: Fixed

Environment

Operating System: other
Platform: Other

• Bugzilla Id: 35117
• Assignee: Unassigned
• Reporter: Christopher Ottley
• Created: 30/May/05 13:48
• Updated: 25/Mar/08 07:52
• Resolved: 25/Mar/08 07:52

Description

The VFS default external file monitor uses a fixed thread sleep time. This isn't
a problem if there are a small number of files monitored. If the number grows
however, the CPU usage grows rapidly as the processing required to check the
file states increases between calls to Thread.sleep. Monitoring 10 files per
second is fine, 1000 or 10000 files per second however starts getting too
intensive. This modification of the default monitor sleeps a modulus of the time
to sleep in milliseconds. This means that for 2000 files being monitored, a
worst case scenario would take 2 seconds before a file modification event was
broadcast, for 3000 files, 3 seconds and so on. This reduces the CPU load
considerably while still giving adequate performance for the number of files
monitored. Basically I added a sleep call if (iterFileNames % getDelay()) == 0).

/*

• Copyright 2002, 2003, 2004, 2005 The Apache Software Foundation.
  *
• Licensed under the Apache License, Version 2.0 (the "License");
• you may not use this file except in compliance with the License.
• You may obtain a copy of the License at
  *
• http://www.apache.org/licenses/LICENSE-2.0
  *
• Unless required by applicable law or agreed to in writing, software
• distributed under the License is distributed on an "AS IS" BASIS,
• WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
• See the License for the specific language governing permissions and
• limitations under the License.
  */
  package org.apache.commons.vfs.impl;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.vfs.FileListener;
import org.apache.commons.vfs.FileMonitor;
import org.apache.commons.vfs.FileName;
import org.apache.commons.vfs.FileObject;
import org.apache.commons.vfs.FileSystemException;
import org.apache.commons.vfs.FileType;
import org.apache.commons.vfs.provider.AbstractFileSystem;

import java.util.HashMap;
import java.util.Map;
import java.util.Stack;

/**

• A polling {@link FileMonitor}

  implementation.<br />

• <br />
• The DefaultFileMonitor is a Thread based polling file system monitor with a 1
  second delay.<br />
• <br />
• <b>Design:</b>
• <p/>
• There is a Map of monitors known as FileMonitorAgents. With the thread running,
• each FileMonitorAgent object is asked to "check" on the file it is
  responsible for.
• To do this check, the cache is cleared.
• </p>
• <ul>
• <li>If the file existed before the refresh and it no longer exists, a delete
  event is fired.</li>
• <li>If the file existed before the refresh and it still exists, check the
  last modified timestamp to see if that has changed.</li>
• <li>If it has, fire a change event.</li>
• </ul>
• <p/>
• With each file delete, the FileMonitorAgent of the parent is asked to
  re-build its
• list of children, so that they can be accurately checked when there are new
  children.<br/>
• New files are detected during each "check" as each file does a check for new
  children.
• If new children are found, create events are fired recursively if recursive
  descent is
• enabled.
• </p>
• <p>
• For performance reasons, added a delay that increases as the number of files
  monitored
• increases. The default is a delay of 1 second for every 1000 files processed.
• </p>
• <p/>
• <br /><b>Example usage:</b><pre>
• FileSystemManager fsManager = VFS.getManager();
• FileObject listendir = fsManager.resolveFile("/home/username/monitored/");
• <p/>
• DefaultFileMonitor fm = new DefaultFileMonitor(new CustomFileListener());
• fm.setRecursive(true);
• fm.addFile(listendir);
• fm.start();
• </pre>
• <i>(where CustomFileListener is a class that implements the FileListener
  interface.)</i>
  *
• @author <a href="mailto:[e-mail]">Christopher Ottley</a>
• @version $Revision: 1.4 $ $Date: 2004/12/29 19:47:34 $
  */
  public class DefaultFileMonitor implements Runnable, FileMonitor
  {
  private final static Log log = LogFactory.getLog(DefaultFileMonitor.class);

/**

• Map from FileName to FileObject being monitored.
  */
  private final Map monitorMap = new HashMap();

/**

• The low priority thread used for checking the files being monitored.
  */
  private Thread monitorThread;

/**

• File objects to be removed from the monitor map.
  */
  private Stack deleteStack = new Stack();

/**

• File objects to be added to the monitor map.
  */
  private Stack addStack = new Stack();

/**

• A flag used to determine if the monitor thread should be running.
  */
  private boolean shouldRun = true;

/**

• A flag used to determine if adding files to be monitored should be recursive.
  */
  private boolean recursive = false;

/**

• Set the delay between checks
  */
  private long delay = 1000;

/**

• A listener object that if set, is notified on file creation and deletion.
  */
  private final FileListener listener;

public DefaultFileMonitor(final FileListener listener)

{ this.listener = listener; }

/**

• Access method to get the recursive setting when adding files for monitoring.
  */
  public boolean isRecursive() { return this.recursive; }

/**

• Access method to set the recursive setting when adding files for monitoring.
  */
  public void setRecursive(final boolean newRecursive) { this.recursive = newRecursive; }

/**

• Access method to get the current FileListener object notified when there
• are changes with the files added.
  */
  FileListener getFileListener() { return this.listener; }

/**

• Adds a file to be monitored.
  */
  public void addFile(final FileObject file)
  {
  synchronized (this.monitorMap)
  {
  if (this.monitorMap.get(file.getName()) == null)
  {
  this.monitorMap.put(file.getName(), new FileMonitorAgent(this,
  file));

try
{
if (this.listener != null)

{ file.getFileSystem().addListener(file, this.listener); }

if (file.getType().hasChildren() && this.recursive)
{
// Traverse the children
final FileObject[] children = file.getChildren();
for (int i = 0; i < children.length; i++)

{ this.addFile(children[i]); // Add depth first }

}

}
catch (FileSystemException fse)

{ log.error(fse.getLocalizedMessage(), fse); }

}
}
}

/**

• Removes a file from being monitored.
  */
  public void removeFile(final FileObject file)
  {
  synchronized (this.monitorMap)
  {
  FileName fn = file.getName();
  if (this.monitorMap.get(fn) != null)
  {
  FileObject parent;
  try { parent = file.getParent(); }

  catch (FileSystemException fse)

  { parent = null; }

this.monitorMap.remove(fn);

if (parent != null)
{ // Not the root
FileMonitorAgent parentAgent =
(FileMonitorAgent) this.monitorMap.get(parent.getName());
if (parentAgent != null)

{ parentAgent.resetChildrenList(); }

}
}
}
}

/**

• Queues a file for removal from being monitored.
  */
  protected void queueRemoveFile(final FileObject file) { this.deleteStack.push(file); }

/**

• Get the delay between runs
  */
  public long getDelay() { return delay; }

/**

• Set the delay between runs
  */
  public void setDelay(long delay)
  Unknown macro: { if (delay > 0) { this.delay = delay; } else { this.delay = 1000; } }

/**

• Queues a file for addition to be monitored.
  */
  protected void queueAddFile(final FileObject file) { this.addStack.push(file); }

/**

• Starts monitoring the files that have been added.
  */
  public void start()
  Unknown macro: { if (this.monitorThread == null) { this.monitorThread = new Thread(this); this.monitorThread.setDaemon(true); this.monitorThread.setPriority(Thread.MIN_PRIORITY); } this.monitorThread.start(); }

/**

• Stops monitoring the files that have been added.
  */
  public void stop() { this.shouldRun = false; }

/**

• Asks the agent for each file being monitored to check its file for changes.
  */
  public void run()
  {
  mainloop: while (!Thread.currentThread().isInterrupted() && this.shouldRun)
  {
  while (!this.deleteStack.empty()) { this.removeFile((FileObject) this.deleteStack.pop()); }

// For each entry in the map
Object fileNames[];
synchronized (this.monitorMap)

{ fileNames = this.monitorMap.keySet().toArray(); }

for (int iterFileNames = 0; iterFileNames < fileNames.length;
iterFileNames++)
{
FileName fileName = (FileName) fileNames[iterFileNames];
FileMonitorAgent agent;
synchronized (this.monitorMap)

{ agent = (FileMonitorAgent) this.monitorMap.get(fileName); }

if (agent != null)

{ agent.check(); }

if ((iterFileNames % getDelay()) == 0) {
try

{ Thread.sleep(getDelay()); }

catch (InterruptedException e)
{

}
}

if (Thread.currentThread().isInterrupted() || !this.shouldRun)

{ continue mainloop; }

}

while (!this.addStack.empty())

{ this.addFile((FileObject) this.addStack.pop()); }

try

{ Thread.sleep(getDelay()); }

catch (InterruptedException e)

{ continue; }

}

this.shouldRun = true;
}

/**

• File monitor agent.
  */
  private static class FileMonitorAgent
  {
  private final FileObject file;
  private final DefaultFileMonitor fm;

private boolean exists;
private long timestamp;
private Map children = null;

private FileMonitorAgent(DefaultFileMonitor fm, FileObject file)
{
this.fm = fm;
this.file = file;

this.refresh();
this.resetChildrenList();

try

{ this.exists = this.file.exists(); }

catch (FileSystemException fse)

{ this.exists = false; }

try

{ this.timestamp = this.file.getContent().getLastModifiedTime(); }

catch (FileSystemException fse)

{ this.timestamp = -1; }

}

private void resetChildrenList()
{
try
{
if (this.file.getType() == FileType.FOLDER)
{
this.children = new HashMap();
FileObject[] childrenList = this.file.getChildren();
for (int i = 0; i < childrenList.length; i++)

{ this.children.put(childrenList[i].getName(), new Object()); // null? }

}
}
catch (FileSystemException fse)

{ this.children = null; }

}

/**

• Clear the cache and re-request the file object
  */
  private void refresh()
  Unknown macro: { try { // this.file = ((AbstractFileSystem) this.file.getFileSystem()).resolveFile(this.file.getName(), false); // close the file - this will detach and reattach its resources (for this thread) on the // next access this.file.close(); } catch (FileSystemException fse) { log.error(fse.getLocalizedMessage(), fse); }
  }
  
  
  /**
  * Recursively fires create events for all children if recursive descent is
  * enabled. Otherwise the create event is only fired for the initial
  * FileObject.
  */
  private void fireAllCreate(FileObject child)
  {
  // Add listener so that it can be triggered
  if (this.fm.getFileListener() != null)
  { child.getFileSystem().addListener(child, this.fm.getFileListener()); }
  
  ((AbstractFileSystem) child.getFileSystem()).fireFileCreated(child);
  
  // Remove it because a listener is added in the queueAddFile
  if (this.fm.getFileListener() != null)
  { child.getFileSystem().removeListener(child, this.fm.getFileListener()); }
  
  this.fm.queueAddFile(child); // Add
  
  try
  {
  
  if (this.fm.isRecursive())
  {
  if (child.getType() == FileType.FOLDER)
  {
  FileObject[] newChildren = child.getChildren();
  for (int i = 0; i < newChildren.length; i++)
  { fireAllCreate(newChildren[i]); }
  }
  }
  
  }
  catch (FileSystemException fse)
  { log.error(fse.getLocalizedMessage(), fse); } }

/**

• Only checks for new children. If children are removed, they'll
• eventually be checked.
  */
  private void checkForNewChildren()
  {
  try
  {
  if (this.file.getType() == FileType.FOLDER)
  {
  FileObject[] newChildren = this.file.getChildren();
  if (this.children != null)
  {
  // See which new children are not listed in the current
  children map.
  Map newChildrenMap = new HashMap();
  Stack missingChildren = new Stack();

for (int i = 0; i < newChildren.length; i++)
{
newChildrenMap.put(newChildren[i].getName(), new
Object()); // null ?
// If the child's not there
if
(!this.children.containsKey(newChildren[i].getName()))

{ missingChildren.push(newChildren[i]); }

}

this.children = newChildrenMap;

// If there were missing children
if (!missingChildren.empty())
{

while (!missingChildren.empty())

{ FileObject child = (FileObject) missingChildren.pop(); this.fireAllCreate(child); }

}

}
else
{
// First set of children - Break out the cigars
if (newChildren.length > 0)

{ this.children = new HashMap(); }

for (int i = 0; i < newChildren.length; i++)

{ this.children.put(newChildren[i].getName(), new Object()); // null? this.fireAllCreate(newChildren[i]); }

}
}
}
catch (FileSystemException fse)

{ log.error(fse.getLocalizedMessage(), fse); }
}

private void check()
{
this.refresh();

try
{
// If the file existed and now doesn't
if (this.exists && !this.file.exists())
{
this.exists = this.file.exists();
this.timestamp = -1;

// Fire delete event

((AbstractFileSystem)
this.file.getFileSystem()).fireFileDeleted(this.file);

// Remove listener in case file is re-created. Don't want to
fire twice.
if (this.fm.getFileListener() != null)
{ this.file.getFileSystem().removeListener(this.file, this.fm.getFileListener()); }

// Remove from map
this.fm.queueRemoveFile(this.file);
}
else if (this.exists && this.file.exists())
{

// Check the timestamp to see if it has been modified
if (this.timestamp !=
this.file.getContent().getLastModifiedTime())
{
this.timestamp =
this.file.getContent().getLastModifiedTime();
// Fire change event

// Don't fire if it's a folder because new file children
// and deleted files in a folder have their own event
triggered.
if (this.file.getType() != FileType.FOLDER)
{ ((AbstractFileSystem) this.file.getFileSystem()).fireFileChanged(this.file); }
}

}

this.checkForNewChildren();

}
catch (FileSystemException fse)
{ log.error(fse.getLocalizedMessage(), fse); }

}

}

}