From 87abe599995d5646f5d83cf2e3a225bd73148b3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Fri, 22 Nov 2024 00:52:34 -0300 Subject: [PATCH] Added Option::MacSanitizeEvents to avoid repeated Add events on macOS (issue #190). --- .ecode/project_build.json | 17 +++- include/efsw/efsw.h | 137 +++++++++++++++---------------- include/efsw/efsw.hpp | 24 +++--- src/efsw/FileWatcherFSEvents.cpp | 1 + src/efsw/WatcherFSEvents.cpp | 41 +++++---- src/efsw/WatcherFSEvents.hpp | 15 ++-- 6 files changed, 122 insertions(+), 113 deletions(-) diff --git a/.ecode/project_build.json b/.ecode/project_build.json index 62ba57a..b07e874 100644 --- a/.ecode/project_build.json +++ b/.ecode/project_build.json @@ -104,7 +104,7 @@ "macos-debug": { "build": [ { - "args": "--thread-sanitizer --verbose gmake2", + "args": "--verbose gmake2", "command": "premake5", "working_dir": "" }, @@ -117,8 +117,8 @@ "build_types": [], "clean": [ { - "args": "", - "command": "", + "args": "-C make/macosx clean", + "command": "make", "working_dir": "" } ], @@ -133,7 +133,16 @@ "preset": "generic", "relative_file_paths": true } - } + }, + "run": [ + { + "args": "", + "command": "efsw-test-debug", + "name": "efsw-test", + "run_in_terminal": true, + "working_dir": "${project_root}/bin" + } + ] }, "macos-release": { "build": [ diff --git a/include/efsw/efsw.h b/include/efsw/efsw.h index 8a682c4..fc9a3a3 100644 --- a/include/efsw/efsw.h +++ b/include/efsw/efsw.h @@ -1,7 +1,7 @@ /** @author Sepul Sepehr Taghdisian - Copyright (c) 2013 Martin Lucas Golini + Copyright (c) 2024 Martín Lucas Golini Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -32,31 +32,31 @@ extern "C" { #endif -#if defined(_WIN32) - #ifdef EFSW_DYNAMIC - // Windows platforms - #ifdef EFSW_EXPORTS - // From DLL side, we must export - #define EFSW_API __declspec(dllexport) - #else - // From client application side, we must import - #define EFSW_API __declspec(dllimport) - #endif - #else - // No specific directive needed for static build - #ifndef EFSW_API - #define EFSW_API - #endif - #endif +#if defined( _WIN32 ) +#ifdef EFSW_DYNAMIC +// Windows platforms +#ifdef EFSW_EXPORTS +// From DLL side, we must export +#define EFSW_API __declspec( dllexport ) #else - #if ( __GNUC__ >= 4 ) && defined( EFSW_EXPORTS ) - #define EFSW_API __attribute__ ((visibility("default"))) - #endif - - // Other platforms don't need to define anything - #ifndef EFSW_API - #define EFSW_API - #endif +// From client application side, we must import +#define EFSW_API __declspec( dllimport ) +#endif +#else +// No specific directive needed for static build +#ifndef EFSW_API +#define EFSW_API +#endif +#endif +#else +#if ( __GNUC__ >= 4 ) && defined( EFSW_EXPORTS ) +#define EFSW_API __attribute__( ( visibility( "default" ) ) ) +#endif + +// Other platforms don't need to define anything +#ifndef EFSW_API +#define EFSW_API +#endif #endif /// Type for a watch id @@ -65,27 +65,24 @@ typedef long efsw_watchid; /// Type for watcher typedef void* efsw_watcher; -enum efsw_action -{ - EFSW_ADD = 1, /// Sent when a file is created or renamed - EFSW_DELETE = 2, /// Sent when a file is deleted or renamed - EFSW_MODIFIED = 3, /// Sent when a file is modified - EFSW_MOVED = 4 /// Sent when a file is moved +enum efsw_action { + EFSW_ADD = 1, /// Sent when a file is created or renamed + EFSW_DELETE = 2, /// Sent when a file is deleted or renamed + EFSW_MODIFIED = 3, /// Sent when a file is modified + EFSW_MOVED = 4 /// Sent when a file is moved }; -enum efsw_error -{ - EFSW_NOTFOUND = -1, - EFSW_REPEATED = -2, - EFSW_OUTOFSCOPE = -3, - EFSW_NOTREADABLE = -4, - EFSW_REMOTE = -5, - EFSW_WATCHER_FAILED = -6, - EFSW_UNSPECIFIED = -7 +enum efsw_error { + EFSW_NOTFOUND = -1, + EFSW_REPEATED = -2, + EFSW_OUTOFSCOPE = -3, + EFSW_NOTREADABLE = -4, + EFSW_REMOTE = -5, + EFSW_WATCHER_FAILED = -6, + EFSW_UNSPECIFIED = -7 }; -enum efsw_option -{ +enum efsw_option { /// For Windows, the default buffer size of 63*1024 bytes sometimes is not enough and /// file system events may be dropped. For that, using a different (bigger) buffer size /// can be defined here, but note that this does not work for network drives, @@ -104,18 +101,18 @@ enum efsw_option // kFSEventStreamEventFlagItemInodeMetaMod // Default configuration will set the 3 flags EFSW_OPT_MAC_MODIFIED_FILTER = 3, + /// macOS sometimes informs incorrect or old file states that may confuse the consumer + /// The events sanitizer will try to sanitize incorrectly reported events in favor of reducing + /// the number of events reported. This will have an small performance and memory impact as a + /// consequence. + EFSW_OPT_MAC_SANITIZE_EVENTS = 4, }; /// Basic interface for listening for file events. -typedef void (*efsw_pfn_fileaction_callback) ( - efsw_watcher watcher, - efsw_watchid watchid, - const char* dir, - const char* filename, - enum efsw_action action, - const char* old_filename, - void* param -); +typedef void ( *efsw_pfn_fileaction_callback )( efsw_watcher watcher, efsw_watchid watchid, + const char* dir, const char* filename, + enum efsw_action action, const char* old_filename, + void* param ); typedef struct { enum efsw_option option; @@ -126,10 +123,10 @@ typedef struct { * Creates a new file-watcher * @param generic_mode Force the use of the Generic file watcher */ -efsw_watcher EFSW_API efsw_create(int generic_mode); +efsw_watcher EFSW_API efsw_create( int generic_mode ); /// Release the file-watcher and unwatch any directories -void EFSW_API efsw_release(efsw_watcher watcher); +void EFSW_API efsw_release( efsw_watcher watcher ); /// Retrieve last error occured by file-watcher EFSW_API const char* efsw_getlasterror(); @@ -139,47 +136,49 @@ EFSW_API void efsw_clearlasterror(); /// Add a directory watch /// On error returns WatchID with Error type. -efsw_watchid EFSW_API efsw_addwatch(efsw_watcher watcher, const char* directory, - efsw_pfn_fileaction_callback callback_fn, int recursive, void* param); +efsw_watchid EFSW_API efsw_addwatch( efsw_watcher watcher, const char* directory, + efsw_pfn_fileaction_callback callback_fn, int recursive, + void* param ); /// Add a directory watch, specifying options /// @param options Pointer to an array of watcher options /// @param nr_options Number of options referenced by \p options -efsw_watchid EFSW_API efsw_addwatch_withoptions(efsw_watcher watcher, const char* directory, - efsw_pfn_fileaction_callback callback_fn, int recursive, efsw_watcher_option *options, - int options_number, void* param); +efsw_watchid EFSW_API efsw_addwatch_withoptions( efsw_watcher watcher, const char* directory, + efsw_pfn_fileaction_callback callback_fn, + int recursive, efsw_watcher_option* options, + int options_number, void* param ); /// Remove a directory watch. This is a brute force search O(nlogn). -void EFSW_API efsw_removewatch(efsw_watcher watcher, const char* directory); +void EFSW_API efsw_removewatch( efsw_watcher watcher, const char* directory ); /// Remove a directory watch. This is a map lookup O(logn). -void EFSW_API efsw_removewatch_byid(efsw_watcher watcher, efsw_watchid watchid); +void EFSW_API efsw_removewatch_byid( efsw_watcher watcher, efsw_watchid watchid ); /// Starts watching ( in other thread ) -void EFSW_API efsw_watch(efsw_watcher watcher); +void EFSW_API efsw_watch( efsw_watcher watcher ); /** * Allow recursive watchers to follow symbolic links to other directories * followSymlinks is disabled by default */ -void EFSW_API efsw_follow_symlinks(efsw_watcher watcher, int enable); +void EFSW_API efsw_follow_symlinks( efsw_watcher watcher, int enable ); /** @return If can follow symbolic links to directorioes */ -int EFSW_API efsw_follow_symlinks_isenabled(efsw_watcher watcher); +int EFSW_API efsw_follow_symlinks_isenabled( efsw_watcher watcher ); /** * When enable this it will allow symlinks to watch recursively out of the pointed directory. * follorSymlinks must be enabled to this work. - * For example, added symlink to /home/folder, and the symlink points to /, this by default is not allowed, - * it's only allowed to symlink anything from /home/ and deeper. This is to avoid great levels of recursion. - * Enabling this could lead in infinite recursion, and crash the watcher ( it will try not to avoid this ). - * Buy enabling out of scope links, it will allow this behavior. + * For example, added symlink to /home/folder, and the symlink points to /, this by default is not + * allowed, it's only allowed to symlink anything from /home/ and deeper. This is to avoid great + * levels of recursion. Enabling this could lead in infinite recursion, and crash the watcher ( it + * will try not to avoid this ). Buy enabling out of scope links, it will allow this behavior. * allowOutOfScopeLinks are disabled by default. */ -void EFSW_API efsw_allow_outofscopelinks(efsw_watcher watcher, int allow); +void EFSW_API efsw_allow_outofscopelinks( efsw_watcher watcher, int allow ); /// @return Returns if out of scope links are allowed -int EFSW_API efsw_outofscopelinks_isallowed(efsw_watcher watcher); +int EFSW_API efsw_outofscopelinks_isallowed( efsw_watcher watcher ); #ifdef __cplusplus } diff --git a/include/efsw/efsw.hpp b/include/efsw/efsw.hpp index 25547db..3c0bb2b 100644 --- a/include/efsw/efsw.hpp +++ b/include/efsw/efsw.hpp @@ -1,7 +1,7 @@ /** @author Martín Lucas Golini - Copyright (c) 2013 Martín Lucas Golini + Copyright (c) 2024 Martín Lucas Golini Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -28,7 +28,6 @@ #ifndef ESFW_HPP #define ESFW_HPP -#include #include #include @@ -137,13 +136,18 @@ enum Option { /// FILE_NOTIFY_CHANGE_* flags. WinNotifyFilter = 2, /// For macOS (FSEvents backend), per default all modified event types are capture but we might - // only be interested in a subset; the value of the option should be set to a set of bitwise - // from: - // kFSEventStreamEventFlagItemFinderInfoMod - // kFSEventStreamEventFlagItemModified - // kFSEventStreamEventFlagItemInodeMetaMod - // Default configuration will set the 3 flags + /// only be interested in a subset; the value of the option should be set to a set of bitwise + /// from: + /// kFSEventStreamEventFlagItemFinderInfoMod + /// kFSEventStreamEventFlagItemModified + /// kFSEventStreamEventFlagItemInodeMetaMod + /// Default configuration will set the 3 flags MacModifiedFilter = 3, + /// macOS sometimes informs incorrect or old file states that may confuse the consumer + /// The events sanitizer will try to sanitize incorrectly reported events in favor of reducing + /// the number of events reported. This will have an small performance and memory impact as a + /// consequence. + MacSanitizeEvents = 4, }; } typedef Options::Option Option; @@ -177,7 +181,7 @@ class EFSW_API FileWatcher { /// @param options Allows customization of a watcher /// @return Returns the watch id for the directory or, on error, a WatchID with Error type. WatchID addWatch( const std::string& directory, FileWatchListener* watcher, bool recursive, - const std::vector &options ); + const std::vector& options ); /// Remove a directory watch. This is a brute force search O(nlogn). void removeWatch( const std::string& directory ); @@ -240,7 +244,7 @@ class FileWatchListener { /// @class WatcherOption class WatcherOption { public: - WatcherOption(Option option, int value) : mOption(option), mValue(value) {}; + WatcherOption( Option option, int value ) : mOption( option ), mValue( value ){}; Option mOption; int mValue; }; diff --git a/src/efsw/FileWatcherFSEvents.cpp b/src/efsw/FileWatcherFSEvents.cpp index dfb10a1..70ec2b1 100644 --- a/src/efsw/FileWatcherFSEvents.cpp +++ b/src/efsw/FileWatcherFSEvents.cpp @@ -169,6 +169,7 @@ WatchID FileWatcherFSEvents::addWatch( const std::string& directory, FileWatchLi pWatch->FWatcher = this; pWatch->ModifiedFlags = getOptionValue( options, Option::MacModifiedFilter, efswFSEventsModified ); + pWatch->SanitizeEvents = getOptionValue( options, Option::MacSanitizeEvents, 0 ) != 0; pWatch->init(); diff --git a/src/efsw/WatcherFSEvents.cpp b/src/efsw/WatcherFSEvents.cpp index 11b7e81..f963374 100644 --- a/src/efsw/WatcherFSEvents.cpp +++ b/src/efsw/WatcherFSEvents.cpp @@ -10,13 +10,6 @@ namespace efsw { WatcherFSEvents::WatcherFSEvents() : Watcher(), FWatcher( NULL ), FSStream( NULL ), WatcherGen( NULL ) {} -WatcherFSEvents::WatcherFSEvents( WatchID id, std::string directory, FileWatchListener* listener, - bool recursive, WatcherFSEvents* parent ) : - Watcher( id, directory, listener, recursive ), - FWatcher( NULL ), - FSStream( NULL ), - WatcherGen( NULL ) {} - WatcherFSEvents::~WatcherFSEvents() { if ( NULL != FSStream ) { FSEventStreamStop( FSStream ); @@ -50,13 +43,13 @@ void WatcherFSEvents::init() { ctx.release = NULL; ctx.copyDescription = NULL; - dispatch_queue_t queue = dispatch_queue_create(NULL, NULL); + dispatch_queue_t queue = dispatch_queue_create( NULL, NULL ); FSStream = FSEventStreamCreate( kCFAllocatorDefault, &FileWatcherFSEvents::FSEventCallback, &ctx, CFDirectoryArray, kFSEventStreamEventIdSinceNow, 0., streamFlags ); - FSEventStreamSetDispatchQueue(FSStream, queue); + FSEventStreamSetDispatchQueue( FSStream, queue ); FSEventStreamStart( FSStream ); @@ -68,27 +61,31 @@ void WatcherFSEvents::sendFileAction( WatchID watchid, const std::string& dir, const std::string& filename, Action action, std::string oldFilename ) { Listener->handleFileAction( watchid, FileSystem::precomposeFileName( dir ), - FileSystem::precomposeFileName( filename ), action, FileSystem::precomposeFileName( oldFilename ) ); + FileSystem::precomposeFileName( filename ), action, + FileSystem::precomposeFileName( oldFilename ) ); } void WatcherFSEvents::handleAddModDel( const Uint32& flags, const std::string& path, - std::string& dirPath, std::string& filePath ) { - if ( flags & efswFSEventStreamEventFlagItemCreated ) { - if ( FileInfo::exists( path ) ) { - sendFileAction( ID, dirPath, filePath, Actions::Add ); - } + std::string& dirPath, std::string& filePath, Uint64 inode ) { + if ( ( flags & efswFSEventStreamEventFlagItemCreated ) && FileInfo::exists( path ) && + ( !SanitizeEvents || FilesAdded.find( inode ) != FilesAdded.end() ) ) { + sendFileAction( ID, dirPath, filePath, Actions::Add ); + + if ( SanitizeEvents ) + FilesAdded.insert( inode ); } if ( flags & ModifiedFlags ) { sendFileAction( ID, dirPath, filePath, Actions::Modified ); } - if ( flags & efswFSEventStreamEventFlagItemRemoved ) { + if ( ( flags & efswFSEventStreamEventFlagItemRemoved ) && !FileInfo::exists( path ) ) { // Since i don't know the order, at least i try to keep the data consistent with the real // state - if ( !FileInfo::exists( path ) ) { - sendFileAction( ID, dirPath, filePath, Actions::Delete ); - } + sendFileAction( ID, dirPath, filePath, Actions::Delete ); + + if ( SanitizeEvents ) + FilesAdded.erase( inode ); } } @@ -159,7 +156,7 @@ void WatcherFSEvents::handleActions( std::vector& events ) { } } } else { - handleAddModDel( nEvent.Flags, nEvent.Path, dirPath, filePath ); + handleAddModDel( nEvent.Flags, nEvent.Path, dirPath, filePath, event.inode ); } if ( nEvent.Flags & ( efswFSEventStreamEventFlagItemCreated | @@ -182,7 +179,7 @@ void WatcherFSEvents::handleActions( std::vector& events ) { sendFileAction( ID, dirPath, filePath, Actions::Delete ); } } else { - handleAddModDel( event.Flags, event.Path, dirPath, filePath ); + handleAddModDel( event.Flags, event.Path, dirPath, filePath, event.inode ); } } else { efDEBUG( "Directory: %s changed\n", event.Path.c_str() ); @@ -192,7 +189,7 @@ void WatcherFSEvents::handleActions( std::vector& events ) { } void WatcherFSEvents::process() { - std::set::iterator it = DirsChanged.begin(); + std::unordered_set::iterator it = DirsChanged.begin(); for ( ; it != DirsChanged.end(); it++ ) { if ( !FileWatcherFSEvents::isGranular() ) { diff --git a/src/efsw/WatcherFSEvents.hpp b/src/efsw/WatcherFSEvents.hpp index 125c137..f05b094 100644 --- a/src/efsw/WatcherFSEvents.hpp +++ b/src/efsw/WatcherFSEvents.hpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include namespace efsw { @@ -45,8 +45,8 @@ class FSEvent { Path( path ), Flags( flags ), Id( id ), inode( inode ) {} std::string Path; - long Flags; - Uint64 Id; + long Flags{ 0 }; + Uint64 Id{ 0 }; Uint64 inode{ 0 }; }; @@ -54,9 +54,6 @@ class WatcherFSEvents : public Watcher { public: WatcherFSEvents(); - WatcherFSEvents( WatchID id, std::string directory, FileWatchListener* listener, bool recursive, - WatcherFSEvents* parent = NULL ); - ~WatcherFSEvents(); void init(); @@ -68,14 +65,16 @@ class WatcherFSEvents : public Watcher { Atomic FWatcher; FSEventStreamRef FSStream; Uint64 ModifiedFlags{ efswFSEventsModified }; + bool SanitizeEvents{ false }; protected: void handleAddModDel( const Uint32& flags, const std::string& path, std::string& dirPath, - std::string& filePath ); + std::string& filePath, Uint64 inode ); WatcherGeneric* WatcherGen; - std::set DirsChanged; + std::unordered_set DirsChanged; + std::unordered_set FilesAdded; void sendFileAction( WatchID watchid, const std::string& dir, const std::string& filename, Action action, std::string oldFilename = "" );