diff options
author | Daniel Wilhelm <daniel@wili.li> | 2014-04-18 17:13:13 +0200 |
---|---|---|
committer | Daniel Wilhelm <daniel@wili.li> | 2014-04-18 17:13:13 +0200 |
commit | 7f23ee90fd545995a29e2175f15e8b97e59ca67a (patch) | |
tree | f8d0afac51995032e58b9a475ccbbc73ba207baf /library | |
parent | 3.19 (diff) | |
download | FreeFileSync-7f23ee90fd545995a29e2175f15e8b97e59ca67a.tar.gz FreeFileSync-7f23ee90fd545995a29e2175f15e8b97e59ca67a.tar.bz2 FreeFileSync-7f23ee90fd545995a29e2175f15e8b97e59ca67a.zip |
3.20
Diffstat (limited to 'library')
-rw-r--r-- | library/binary.cpp | 13 | ||||
-rw-r--r-- | library/binary.h | 2 | ||||
-rw-r--r-- | library/custom_grid.cpp | 2 | ||||
-rw-r--r-- | library/custom_grid.h | 2 | ||||
-rw-r--r-- | library/db_file.cpp | 91 | ||||
-rw-r--r-- | library/db_file.h | 2 | ||||
-rw-r--r-- | library/detect_renaming.cpp | 2 | ||||
-rw-r--r-- | library/detect_renaming.h | 2 | ||||
-rw-r--r-- | library/dir_exist_async.h | 35 | ||||
-rw-r--r-- | library/dir_lock.cpp | 9 | ||||
-rw-r--r-- | library/error_log.cpp | 2 | ||||
-rw-r--r-- | library/error_log.h | 2 | ||||
-rw-r--r-- | library/hard_filter.cpp | 2 | ||||
-rw-r--r-- | library/hard_filter.h | 2 | ||||
-rw-r--r-- | library/icon_buffer.cpp | 332 | ||||
-rw-r--r-- | library/icon_buffer.h | 2 | ||||
-rw-r--r-- | library/lock_holder.h | 8 | ||||
-rw-r--r-- | library/norm_filter.h | 2 | ||||
-rw-r--r-- | library/parallel_scan.cpp | 617 | ||||
-rw-r--r-- | library/parallel_scan.h | 74 | ||||
-rw-r--r-- | library/process_xml.cpp | 10 | ||||
-rw-r--r-- | library/process_xml.h | 2 | ||||
-rw-r--r-- | library/resources.cpp | 2 | ||||
-rw-r--r-- | library/resources.h | 2 | ||||
-rw-r--r-- | library/soft_filter.h | 2 | ||||
-rw-r--r-- | library/statistics.cpp | 2 | ||||
-rw-r--r-- | library/statistics.h | 2 | ||||
-rw-r--r-- | library/status_handler.cpp | 2 | ||||
-rw-r--r-- | library/status_handler.h | 2 |
29 files changed, 975 insertions, 254 deletions
diff --git a/library/binary.cpp b/library/binary.cpp index 3a202711..1fd8c55f 100644 --- a/library/binary.cpp +++ b/library/binary.cpp @@ -3,12 +3,13 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #include "binary.h" #include "../shared/file_io.h" #include <vector> #include <wx/stopwatch.h> #include "../shared/int64.h" +#include <boost/thread/tss.hpp> inline void setMinSize(std::vector<char>& buffer, size_t minSize) @@ -71,11 +72,15 @@ bool zen::filesHaveSameContent(const Zstring& filename1, const Zstring& filename FileInput file1(filename1); //throw (FileError) FileInput file2(filename2); //throw (FileError) - BufferSize bufferSize; + static boost::thread_specific_ptr<std::vector<char>> cpyBuf1; + static boost::thread_specific_ptr<std::vector<char>> cpyBuf2; + if (!cpyBuf1.get()) cpyBuf1.reset(new std::vector<char>()); + if (!cpyBuf2.get()) cpyBuf2.reset(new std::vector<char>()); - static std::vector<char> memory1; - static std::vector<char> memory2; + std::vector<char>& memory1 = *cpyBuf1; + std::vector<char>& memory2 = *cpyBuf2; + BufferSize bufferSize; zen::UInt64 bytesCompared; wxLongLong lastDelayViolation = wxGetLocalTimeMillis(); diff --git a/library/binary.h b/library/binary.h index b796ddbb..4dbbcd45 100644 --- a/library/binary.h +++ b/library/binary.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef BINARY_H_INCLUDED #define BINARY_H_INCLUDED diff --git a/library/custom_grid.cpp b/library/custom_grid.cpp index 97b608fb..ccabbea9 100644 --- a/library/custom_grid.cpp +++ b/library/custom_grid.cpp @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #include "custom_grid.h" #include "resources.h" #include <wx/dc.h> diff --git a/library/custom_grid.h b/library/custom_grid.h index 8b70c417..6b577011 100644 --- a/library/custom_grid.h +++ b/library/custom_grid.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef CUSTOMGRID_H_INCLUDED #define CUSTOMGRID_H_INCLUDED diff --git a/library/db_file.cpp b/library/db_file.cpp index ec1c4464..9429dd1e 100644 --- a/library/db_file.cpp +++ b/library/db_file.cpp @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #include "db_file.h" #include <wx/wfstream.h> #include <wx/zstream.h> @@ -305,7 +305,7 @@ std::pair<DirInfoPtr, DirInfoPtr> zen::loadFromDisk(const BaseDirMapping& baseMa !streamLeft ->second.get() || !streamRight->second.get()) throw FileErrorDatabaseNotExisting(_("Initial synchronization:") + " \n\n" + - _("No matching synchronization session found in database files:") + " \n" + + _("Database files do not share a common synchronization session:") + " \n" + "\"" + fileNameLeft + "\"\n" + "\"" + fileNameRight + "\""); //read streams into DirInfo @@ -342,29 +342,27 @@ private: writeNumberC<bool>(false); //mark last entry } - void processFile(const FileMapping& fileMap, const DirContainer* oldDirInfo) + void processFile(const FileMapping& fileMap, const DirContainer* oldParentDir) { - const Zstring shortName = fileMap.getObjShortName(); - if (fileMap.getCategory() == FILE_EQUAL) //data in sync: write current state { if (!fileMap.isEmpty<side>()) { writeNumberC<bool>(true); //mark beginning of entry - writeStringC(shortName); + writeStringC(fileMap.getShortName<side>()); //save respecting case! (Windows) writeNumberC<boost::int64_t >(to<boost::int64_t>(fileMap.getLastWriteTime<side>())); //last modification time writeNumberC<boost::uint64_t>(to<boost::uint64_t>(fileMap.getFileSize<side>())); //filesize } } else //not in sync: reuse last synchronous state { - if (oldDirInfo) //no data is also a "synchronous state"! + if (oldParentDir) //no data is also a "synchronous state"! { - DirContainer::FileList::const_iterator iter = oldDirInfo->files.find(shortName); - if (iter != oldDirInfo->files.end()) + auto iter = oldParentDir->files.find(fileMap.getObjShortName()); + if (iter != oldParentDir->files.end()) { writeNumberC<bool>(true); //mark beginning of entry - writeStringC(shortName); + writeStringC(iter->first); //save respecting case! (Windows) writeNumberC<boost::int64_t >(to<boost::int64_t>(iter->second.lastWriteTimeRaw)); //last modification time writeNumberC<boost::uint64_t>(to<boost::uint64_t>(iter->second.fileSize)); //filesize } @@ -372,16 +370,14 @@ private: } } - void processLink(const SymLinkMapping& linkObj, const DirContainer* oldDirInfo) + void processLink(const SymLinkMapping& linkObj, const DirContainer* oldParentDir) { - const Zstring shortName = linkObj.getObjShortName(); - - if (linkObj.getCategory() == FILE_EQUAL) //data in sync: write current state + if (linkObj.getLinkCategory() == SYMLINK_EQUAL) //data in sync: write current state { if (!linkObj.isEmpty<side>()) { writeNumberC<bool>(true); //mark beginning of entry - writeStringC(shortName); + writeStringC(linkObj.getShortName<side>()); //save respecting case! (Windows) writeNumberC<boost::int64_t>(to<boost::int64_t>(linkObj.getLastWriteTime<side>())); //last modification time writeStringC(linkObj.getTargetPath<side>()); writeNumberC<boost::int32_t>(linkObj.getLinkType<side>()); @@ -389,13 +385,13 @@ private: } else //not in sync: reuse last synchronous state { - if (oldDirInfo) //no data is also a "synchronous state"! + if (oldParentDir) //no data is also a "synchronous state"! { - DirContainer::LinkList::const_iterator iter = oldDirInfo->links.find(shortName); - if (iter != oldDirInfo->links.end()) + auto iter = oldParentDir->links.find(linkObj.getObjShortName()); + if (iter != oldParentDir->links.end()) { writeNumberC<bool>(true); //mark beginning of entry - writeStringC(shortName); + writeStringC(iter->first); //save respecting case! (Windows) writeNumberC<boost::int64_t>(to<boost::int64_t>(iter->second.lastWriteTimeRaw)); //last modification time writeStringC(iter->second.targetPath); writeNumberC<boost::int32_t>(iter->second.type); @@ -404,35 +400,62 @@ private: } } - void processDir(const DirMapping& dirMap, const DirContainer* oldDirInfo) + void processDir(const DirMapping& dirMap, const DirContainer* oldParentDir) { - const Zstring shortName = dirMap.getObjShortName(); - - const DirContainer* subDirInfo = NULL; - if (oldDirInfo) //no data is also a "synchronous state"! + const DirContainer* oldDir = NULL; + const Zstring* oldDirName = NULL; + if (oldParentDir) //no data is also a "synchronous state"! { - DirContainer::DirList::const_iterator iter = oldDirInfo->dirs.find(shortName); - if (iter != oldDirInfo->dirs.end()) - subDirInfo = &iter->second; + auto iter = oldParentDir->dirs.find(dirMap.getObjShortName()); + if (iter != oldParentDir->dirs.end()) + { + oldDirName = &iter->first; + oldDir = &iter->second; + } } - if (dirMap.getCategory() == FILE_EQUAL) //data in sync: write current state + CompareDirResult cat = dirMap.getDirCategory(); + + if (cat == DIR_EQUAL) //data in sync: write current state { if (!dirMap.isEmpty<side>()) { writeNumberC<bool>(true); //mark beginning of entry - writeStringC(shortName); - execute(dirMap, subDirInfo); //recurse + writeStringC(dirMap.getShortName<side>()); //save respecting case! (Windows) + execute(dirMap, oldDir); //recurse } } else //not in sync: reuse last synchronous state { - if (subDirInfo) //no data is also a "synchronous state"! + if (oldDir) { - writeNumberC<bool>(true); //mark beginning of entry - writeStringC(shortName); + writeNumberC<bool>(true); //mark beginning of entry + writeStringC(*oldDirName); //save respecting case! (Windows) + execute(dirMap, oldDir); //recurse + return; + } + //no data is also a "synchronous state"! + + //else: not in sync AND no "last synchronous state" + //we cannot simply skip the whole directory, since sub-items might be in sync + //Example: directories on left and right differ in case while sub-files are equal + switch (cat) + { + case DIR_LEFT_SIDE_ONLY: //sub-items cannot be in sync + break; + case DIR_RIGHT_SIDE_ONLY: //sub-items cannot be in sync + break; + case DIR_EQUAL: + assert(false); + break; + case DIR_DIFFERENT_METADATA: + writeNumberC<bool>(true); + writeStringC(dirMap.getShortName<side>()); + //ATTENTION: strictly this is a violation of the principle of reporting last synchronous state! + //however in this case this will result in "last sync unsuccessful" for this directory within <automatic> algorithm, which is fine + execute(dirMap, oldDir); //recurse and save sub-items which are in sync + break; } - execute(dirMap, subDirInfo); //recurse } } }; diff --git a/library/db_file.h b/library/db_file.h index a68e6bcf..dc9eb141 100644 --- a/library/db_file.h +++ b/library/db_file.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef DBFILE_H_INCLUDED #define DBFILE_H_INCLUDED diff --git a/library/detect_renaming.cpp b/library/detect_renaming.cpp index 0e19451c..39e7eb4b 100644 --- a/library/detect_renaming.cpp +++ b/library/detect_renaming.cpp @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #include "detect_renaming.h" #include <map> #include <vector> diff --git a/library/detect_renaming.h b/library/detect_renaming.h index 9f360f8e..e94927c0 100644 --- a/library/detect_renaming.h +++ b/library/detect_renaming.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef DETECTRENAMING_H_INCLUDED #define DETECTRENAMING_H_INCLUDED diff --git a/library/dir_exist_async.h b/library/dir_exist_async.h new file mode 100644 index 00000000..661c70d6 --- /dev/null +++ b/library/dir_exist_async.h @@ -0,0 +1,35 @@ +// ************************************************************************** +// * This file is part of the FreeFileSync project. It is distributed under * +// * GNU General Public License: http://www.gnu.org/licenses/gpl.html * +// * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * +// ************************************************************************** + +#ifndef DIR_EXIST_HEADER_08173281673432158067342132467183267 +#define DIR_EXIST_HEADER_08173281673432158067342132467183267 + +#include "../shared/check_exist.h" +#include "status_handler.h" +#include "../shared/file_error.h" +#include "../shared/i18n.h" + +//dir existence checking may hang for non-existent network drives => run asynchronously and update UI! +namespace +{ +using namespace zen; //restricted to unnamed namespace! + +bool dirExistsUpdating(const Zstring& dirname, ProcessCallback& procCallback) +{ + using namespace util; + + std::wstring statusText = _("Searching for directory %x..."); + replace(statusText, L"%x", std::wstring(L"\"") + dirname + L"\"", false); + procCallback.reportInfo(statusText); + + auto ft = dirExistsAsync(dirname); + while (!ft.timed_wait(boost::posix_time::milliseconds(UI_UPDATE_INTERVAL))) + procCallback.requestUiRefresh(); //may throw! + return ft.get(); +} +} + +#endif //DIR_EXIST_HEADER_08173281673432158067342132467183267 diff --git a/library/dir_lock.cpp b/library/dir_lock.cpp index 1775026b..6123449b 100644 --- a/library/dir_lock.cpp +++ b/library/dir_lock.cpp @@ -42,15 +42,13 @@ const size_t DETECT_EXITUS_INTERVAL = 30000; //assume abandoned lock; unit [ms] const char LOCK_FORMAT_DESCR[] = "FreeFileSync"; const int LOCK_FORMAT_VER = 1; //lock file format version - -typedef Zbase<Zchar, StorageDeepCopy> BasicString; //thread safe string class } //worker thread class LifeSigns { public: - LifeSigns(const BasicString& lockfilename) : //throw()!!! siehe SharedDirLock() + LifeSigns(const Zstring& lockfilename) : //throw()!!! siehe SharedDirLock() lockfilename_(lockfilename) {} //thread safety: make deep copy! void operator()() const //thread entry @@ -120,8 +118,7 @@ public: } private: - //make sure this instance is safely copyable! - const BasicString lockfilename_; //thread local! Not ref-counted! + const Zstring lockfilename_; //thread local! atomic ref-count => binary value-type semantics! }; @@ -494,7 +491,7 @@ public: while (!::tryLock(lockfilename)) //throw (FileError) ::waitOnDirLock(lockfilename, callback); // - threadObj = boost::thread(LifeSigns(lockfilename.c_str())); + threadObj = std::move(boost::thread(LifeSigns(lockfilename))); } ~SharedDirLock() diff --git a/library/error_log.cpp b/library/error_log.cpp index eef8572a..28819f40 100644 --- a/library/error_log.cpp +++ b/library/error_log.cpp @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #include "error_log.h" #include <wx/datetime.h> #include "../shared/i18n.h" diff --git a/library/error_log.h b/library/error_log.h index 323a4297..f8a0c909 100644 --- a/library/error_log.h +++ b/library/error_log.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef ERRORLOGGING_H_INCLUDED #define ERRORLOGGING_H_INCLUDED diff --git a/library/hard_filter.cpp b/library/hard_filter.cpp index e96d32fa..c6d18f47 100644 --- a/library/hard_filter.cpp +++ b/library/hard_filter.cpp @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #include "hard_filter.h" #include "../shared/zstring.h" #include <wx/string.h> diff --git a/library/hard_filter.h b/library/hard_filter.h index 0711175a..bb0b6d54 100644 --- a/library/hard_filter.h +++ b/library/hard_filter.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef FFS_FILTER_H_INCLUDED #define FFS_FILTER_H_INCLUDED diff --git a/library/icon_buffer.cpp b/library/icon_buffer.cpp index 91487498..5d4dd19b 100644 --- a/library/icon_buffer.cpp +++ b/library/icon_buffer.cpp @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #include "icon_buffer.h" #include <wx/msgdlg.h> #include <map> @@ -13,6 +13,8 @@ #include <wx/log.h> #include "../shared/i18n.h" #include "../shared/boost_thread_wrap.h" //include <boost/thread.hpp> +#include "../shared/loki/ScopeGuard.h" +#include <boost/thread/once.hpp> #ifdef FFS_WIN #include <wx/msw/wrapwin.h> //includes "windows.h" @@ -28,38 +30,39 @@ using namespace zen; const size_t BUFFER_SIZE_MAX = 800; //maximum number of icons to buffer -//--------------------------------------------------------------------------------------------------- -typedef Zbase<Zchar, StorageDeepCopy> BasicString; //thread safe string class -//avoid reference-counted objects for shared data: NOT THREADSAFE!!! (implicitly shared variable: ref-count) -//--------------------------------------------------------------------------------------------------- - #ifdef FFS_WIN -BasicString getFileExtension(const BasicString& filename) +Zstring getFileExtension(const Zstring& filename) { - const BasicString shortName = filename.AfterLast(Zchar('\\')); //warning: using windows file name separator! + const Zstring shortName = afterLast(filename, Zchar('\\')); //warning: using windows file name separator! - return shortName.find(Zchar('.')) != BasicString::npos ? + return shortName.find(Zchar('.')) != Zstring::npos ? filename.AfterLast(Zchar('.')) : - BasicString(); + Zstring(); } +namespace +{ +std::set<Zstring, LessFilename> exceptions; //thread-safe! +boost::once_flag once = BOOST_ONCE_INIT; // +} + //test for extension for icons that physically have to be retrieved from disc -bool isPriceyExtension(const BasicString& extension) +bool isPriceyExtension(const Zstring& extension) { - static std::set<BasicString, LessFilename> exceptions; //not thread-safe, but called from worker thread only! - if (exceptions.empty()) + boost::call_once(once, []() { - exceptions.insert(Zstr("exe")); - exceptions.insert(Zstr("lnk")); - exceptions.insert(Zstr("ico")); - exceptions.insert(Zstr("ani")); - exceptions.insert(Zstr("cur")); - exceptions.insert(Zstr("url")); - exceptions.insert(Zstr("msc")); - exceptions.insert(Zstr("scr")); - } + exceptions.insert(L"exe"); + exceptions.insert(L"lnk"); + exceptions.insert(L"ico"); + exceptions.insert(L"ani"); + exceptions.insert(L"cur"); + exceptions.insert(L"url"); + exceptions.insert(L"msc"); + exceptions.insert(L"scr"); + }); + return exceptions.find(extension) != exceptions.end(); } #endif @@ -82,7 +85,7 @@ public: #ifdef FFS_WIN ::CopyIcon(other.handle_) #elif defined FFS_LINUX - gdk_pixbuf_copy(other.handle_) //create new Pix buf with reference count 1 or return 0 on error + ::gdk_pixbuf_copy(other.handle_) //create new Pix buf with reference count 1 or return 0 on error #endif ) {} @@ -98,14 +101,11 @@ public: #ifdef FFS_WIN ::DestroyIcon(handle_); #elif defined FFS_LINUX - g_object_unref(handle_); + ::g_object_unref(handle_); #endif } - void swap(IconHolder& other) //throw() - { - std::swap(handle_, other.handle_); - } + void swap(IconHolder& other) { std::swap(handle_, other.handle_); } //throw() wxIcon toWxIcon() const //copy HandleType, caller needs to take ownership! { @@ -130,7 +130,7 @@ private: }; -IconHolder getAssociatedIcon(const BasicString& filename) +IconHolder getAssociatedIcon(const Zstring& filename) { #ifdef FFS_WIN //despite what docu says about SHGetFileInfo() it can't handle all relative filenames, e.g. "\DirName" @@ -195,7 +195,7 @@ IconHolder getAssociatedIcon(const BasicString& filename) } #ifdef FFS_WIN -IconHolder getAssociatedIconByExt(const BasicString& extension) +IconHolder getAssociatedIconByExt(const Zstring& extension) { SHFILEINFO fileInfo = {}; //initialize hIcon -> fix for weird error: SHGetFileInfo() might return successfully WITHOUT filling fileInfo.hIcon!! @@ -211,80 +211,69 @@ IconHolder getAssociatedIconByExt(const BasicString& extension) #endif -const wxIcon& getDirectoryIcon() +wxIcon getDirectoryIcon() { - static wxIcon folderIcon; +#ifdef FFS_WIN + wxIcon folderIcon; - static bool isInitalized = false; //not thread-safe, but called from GUI thread only! - if (!isInitalized) - { - isInitalized = true; + SHFILEINFO fileInfo = {}; //initialize hIcon -#ifdef FFS_WIN - SHFILEINFO fileInfo = {}; //initialize hIcon + //NOTE: CoInitializeEx()/CoUninitialize() needs to be called for THIS thread! + if (::SHGetFileInfo(Zstr("dummy"), //Windows Seven doesn't like this parameter to be an empty string + FILE_ATTRIBUTE_DIRECTORY, + &fileInfo, + sizeof(fileInfo), + SHGFI_ICON | SHGFI_SMALLICON | SHGFI_USEFILEATTRIBUTES) && - //NOTE: CoInitializeEx()/CoUninitialize() needs to be called for THIS thread! - if (::SHGetFileInfo(Zstr("dummy"), //Windows Seven doesn't like this parameter to be an empty string - FILE_ATTRIBUTE_DIRECTORY, - &fileInfo, - sizeof(fileInfo), - SHGFI_ICON | SHGFI_SMALLICON | SHGFI_USEFILEATTRIBUTES) && + fileInfo.hIcon != 0) //fix for weird error: SHGetFileInfo() might return successfully WITHOUT filling fileInfo.hIcon!! + { + folderIcon.SetHICON(fileInfo.hIcon); //transfer ownership! + folderIcon.SetSize(IconBuffer::ICON_SIZE, IconBuffer::ICON_SIZE); + } - fileInfo.hIcon != 0) //fix for weird error: SHGetFileInfo() might return successfully WITHOUT filling fileInfo.hIcon!! - { - folderIcon.SetHICON(fileInfo.hIcon); //transfer ownership! - folderIcon.SetSize(IconBuffer::ICON_SIZE, IconBuffer::ICON_SIZE); - } + return folderIcon; #elif defined FFS_LINUX - folderIcon = ::getAssociatedIcon(Zstr("/usr/")).toWxIcon(); //all directories will look like "/usr/" + return ::getAssociatedIcon(Zstr("/usr/")).toWxIcon(); //all directories will look like "/usr/" #endif - } - return folderIcon; } -const wxIcon& getFileIcon() +wxIcon getFileIcon() { - static wxIcon fileIcon; - - static bool isInitalized = false; //not thread-safe, but called from GUI thread only! - if (!isInitalized) - { - isInitalized = true; + wxIcon fileIcon; #ifdef FFS_WIN - SHFILEINFO fileInfo = {}; //initialize hIcon + SHFILEINFO fileInfo = {}; //initialize hIcon - //NOTE: CoInitializeEx()/CoUninitialize() needs to be called for THIS thread! - if (::SHGetFileInfo(Zstr("dummy"), //Windows Seven doesn't like this parameter to be an empty string - FILE_ATTRIBUTE_NORMAL, - &fileInfo, - sizeof(fileInfo), - SHGFI_ICON | SHGFI_SMALLICON | SHGFI_USEFILEATTRIBUTES) && + //NOTE: CoInitializeEx()/CoUninitialize() needs to be called for THIS thread! + if (::SHGetFileInfo(Zstr("dummy"), //Windows Seven doesn't like this parameter to be an empty string + FILE_ATTRIBUTE_NORMAL, + &fileInfo, + sizeof(fileInfo), + SHGFI_ICON | SHGFI_SMALLICON | SHGFI_USEFILEATTRIBUTES) && - fileInfo.hIcon != 0) //fix for weird error: SHGetFileInfo() might return successfully WITHOUT filling fileInfo.hIcon!! - { - fileIcon.SetHICON(fileInfo.hIcon); //transfer ownership! - fileIcon.SetSize(IconBuffer::ICON_SIZE, IconBuffer::ICON_SIZE); - } + fileInfo.hIcon != 0) //fix for weird error: SHGetFileInfo() might return successfully WITHOUT filling fileInfo.hIcon!! + { + fileIcon.SetHICON(fileInfo.hIcon); //transfer ownership! + fileIcon.SetSize(IconBuffer::ICON_SIZE, IconBuffer::ICON_SIZE); + } #elif defined FFS_LINUX - try + try + { + Glib::RefPtr<Gtk::IconTheme> iconTheme = Gtk::IconTheme::get_default(); + if (iconTheme) { - Glib::RefPtr<Gtk::IconTheme> iconTheme = Gtk::IconTheme::get_default(); - if (iconTheme) - { - Glib::RefPtr<Gdk::Pixbuf> iconPixbuf = iconTheme->load_icon("misc", IconBuffer::ICON_SIZE, Gtk::ICON_LOOKUP_USE_BUILTIN); - if (!iconPixbuf) - iconPixbuf = iconTheme->load_icon("text-x-generic", IconBuffer::ICON_SIZE, Gtk::ICON_LOOKUP_USE_BUILTIN); - if (iconPixbuf) - fileIcon.SetPixbuf(iconPixbuf->gobj_copy()); // transfer ownership!! - } + Glib::RefPtr<Gdk::Pixbuf> iconPixbuf = iconTheme->load_icon("misc", IconBuffer::ICON_SIZE, Gtk::ICON_LOOKUP_USE_BUILTIN); + if (!iconPixbuf) + iconPixbuf = iconTheme->load_icon("text-x-generic", IconBuffer::ICON_SIZE, Gtk::ICON_LOOKUP_USE_BUILTIN); + if (iconPixbuf) + fileIcon.SetPixbuf(iconPixbuf->gobj_copy()); // transfer ownership!! } - catch (const Glib::Error&) {} -#endif } + catch (const Glib::Error&) {} +#endif return fileIcon; } @@ -293,37 +282,66 @@ const wxIcon& getFileIcon() //---------------------- Shared Data ------------------------- struct WorkLoad { - std::vector<BasicString> filesToLoad; //processes last elements of vector first! - boost::mutex mutex; - boost::condition_variable condition; //signal event: data for processing available +public: + Zstring extractNextFile() //context of worker thread, blocking + { + boost::unique_lock<boost::mutex> dummy(lockFiles); + + while (filesToLoad.empty()) + conditionNewFiles.timed_wait(dummy, boost::get_system_time() + boost::posix_time::milliseconds(50)); //interruption point! + + Zstring fileName = filesToLoad.back(); + filesToLoad.pop_back(); + return fileName; + } + + void setWorkload(const std::vector<Zstring>& newLoad) //context of main thread + { + boost::unique_lock<boost::mutex> dummy(lockFiles); + filesToLoad = newLoad; + + conditionNewFiles.notify_one(); + //condition handling, see: http://www.boost.org/doc/libs/1_43_0/doc/html/thread/synchronization.html#thread.synchronization.condvar_ref + } + +private: + std::vector<Zstring> filesToLoad; //processes last elements of vector first! + boost::mutex lockFiles; + boost::condition_variable conditionNewFiles; //signal event: data for processing available }; -typedef std::map<BasicString, IconHolder, LessFilename> NameIconMap; //entryName/icon -> ATTENTION: avoid ref-counting for this shared data structure! -typedef std::queue<BasicString> IconDbSequence; //entryName -struct Buffer +typedef std::map<Zstring, IconHolder, LessFilename> NameIconMap; //entryName/icon -> note: Zstring is thread-safe +typedef std::queue<Zstring> IconDbSequence; //entryName + +class Buffer { - boost::mutex lockAccess; - NameIconMap iconMappping; //use synchronisation when accessing this! +public: + bool requestFileIcon(const Zstring& fileName, wxIcon* icon = NULL); + void insertIntoBuffer(const Zstring& entryName, const IconHolder& icon); //called by worker thread + +private: + boost::mutex lockBuffer; + NameIconMap iconMappping; //use synchronisation when accessing this! IconDbSequence iconSequence; //save sequence of buffer entry to delete oldest elements }; //------------------------------------------------------------ -bool requestFileIcon(Buffer& buf, const Zstring& fileName, wxIcon* icon = NULL) +bool Buffer::requestFileIcon(const Zstring& fileName, wxIcon* icon) //context of main AND worker thread { - boost::lock_guard<boost::mutex> dummy(buf.lockAccess); + boost::lock_guard<boost::mutex> dummy(lockBuffer); #ifdef FFS_WIN //"pricey" extensions are stored with fullnames and are read from disk, while cheap ones require just the extension - const BasicString extension = getFileExtension(BasicString(fileName)); - const BasicString searchString = isPriceyExtension(extension) ? BasicString(fileName) : extension; - auto iter = buf.iconMappping.find(searchString); + const Zstring extension = getFileExtension(fileName); + const Zstring searchString = isPriceyExtension(extension) ? fileName : extension; + auto iter = iconMappping.find(searchString); #elif defined FFS_LINUX - auto iter = buf.iconMappping.find(BasicString(fileName)); + auto iter = iconMappping.find(fileName); #endif - if (iter == buf.iconMappping.end()) + if (iter == iconMappping.end()) return false; if (icon != NULL) @@ -332,23 +350,23 @@ bool requestFileIcon(Buffer& buf, const Zstring& fileName, wxIcon* icon = NULL) } -void insertIntoBuffer(Buffer& buf, const BasicString& entryName, const IconHolder& icon) //called by worker thread +void Buffer::insertIntoBuffer(const Zstring& entryName, const IconHolder& icon) //called by worker thread { - boost::lock_guard<boost::mutex> dummy(buf.lockAccess); + boost::lock_guard<boost::mutex> dummy(lockBuffer); //thread saftey: icon uses ref-counting! But is NOT shared with main thread! - auto rc = buf.iconMappping.insert(std::make_pair(entryName, icon)); + auto rc = iconMappping.insert(std::make_pair(entryName, icon)); if (rc.second) //if insertion took place - buf.iconSequence.push(entryName); //note: sharing Zstring with IconDB!!! + iconSequence.push(entryName); //note: sharing Zstring with IconDB!!! - assert(buf.iconMappping.size() == buf.iconSequence.size()); + assert(iconMappping.size() == iconSequence.size()); //remove elements if buffer becomes too big: - if (buf.iconMappping.size() > BUFFER_SIZE_MAX) //limit buffer size: critical because GDI resources are limited (e.g. 10000 on XP per process) + if (iconMappping.size() > BUFFER_SIZE_MAX) //limit buffer size: critical because GDI resources are limited (e.g. 10000 on XP per process) { //remove oldest element - buf.iconMappping.erase(buf.iconSequence.front()); - buf.iconSequence.pop(); + iconMappping.erase(iconSequence.front()); + iconSequence.pop(); } } //################################################################################################################################################ @@ -364,10 +382,8 @@ public: void operator()(); //thread entry private: - void doWork(); - - std::shared_ptr<WorkLoad> workload_; - std::shared_ptr<Buffer> buffer_; + std::shared_ptr<WorkLoad> workload_; //main/worker thread may access different shared_ptr instances safely (even though they have the same target!) + std::shared_ptr<Buffer> buffer_; //http://www.boost.org/doc/libs/1_43_0/libs/smart_ptr/shared_ptr.htm?sess=8153b05b34d890e02d48730db1ff7ddc#ThreadSafety }; @@ -375,79 +391,34 @@ void WorkerThread::operator()() //thread entry { //failure to initialize COM for each thread is a source of hard to reproduce bugs: https://sourceforge.net/tracker/?func=detail&aid=3160472&group_id=234430&atid=1093080 #ifdef FFS_WIN - struct ThreadInitializer - { - ThreadInitializer () { ::CoInitializeEx(NULL, COINIT_MULTITHREADED); } - ~ThreadInitializer() { ::CoUninitialize(); } - } dummy1; + ::CoInitializeEx(NULL, COINIT_MULTITHREADED | COINIT_DISABLE_OLE1DDE); + Loki::ScopeGuard dummy = Loki::MakeGuard([]() { ::CoUninitialize(); }); + (void)dummy; #endif - try - { - while (true) - { - { - boost::unique_lock<boost::mutex> dummy(workload_->mutex); - while(workload_->filesToLoad.empty()) - workload_->condition.wait(dummy); //interruption point! - //shared.condition.timed_wait(dummy, boost::get_system_time() + boost::posix_time::milliseconds(100)); - } - - doWork(); //no need to lock the complete method! - } - } - catch (boost::thread_interrupted&) - { - throw; //this is the only reasonable exception! - } - catch (const std::exception& e) //exceptions must be catched per thread - { - wxSafeShowMessage(wxString(_("An exception occurred!")) + wxT("(Icon buffer)"), wxString::FromAscii(e.what())); //simple wxMessageBox won't do for threads - } - catch (...) //exceptions must be catched per thread - { - wxSafeShowMessage(wxString(_("An exception occurred!")) + wxT("(Icon buffer2)"), wxT("Unknown exception in icon thread!")); //simple wxMessageBox won't do for threads - } -} - - -void WorkerThread::doWork() -{ - //do work: get the file icon. while (true) { - BasicString fileName; - { - boost::lock_guard<boost::mutex> dummy(workload_->mutex); - if (workload_->filesToLoad.empty()) - break; //enter waiting state - fileName = workload_->filesToLoad.back(); //deep copy - workload_->filesToLoad.pop_back(); - } + boost::this_thread::interruption_point(); - if (requestFileIcon(*buffer_, Zstring(fileName))) //thread safety: Zstring okay, won't be reference-counted in requestIcon() + const Zstring fileName = workload_->extractNextFile(); //start work: get next icon to load + + if (buffer_->requestFileIcon(fileName)) continue; //icon already in buffer: skip #ifdef FFS_WIN - const BasicString extension = getFileExtension(fileName); //thread-safe: no sharing! + const Zstring extension = getFileExtension(fileName); if (isPriceyExtension(extension)) //"pricey" extensions are stored with fullnames and are read from disk, while cheap ones require just the extension - { - const IconHolder newIcon = getAssociatedIcon(fileName); - insertIntoBuffer(*buffer_, fileName, newIcon); - } + buffer_->insertIntoBuffer(fileName, getAssociatedIcon(fileName)); else //no read-access to disk! determine icon by extension - { - const IconHolder newIcon = getAssociatedIconByExt(extension); - insertIntoBuffer(*buffer_, extension, newIcon); - } + buffer_->insertIntoBuffer(extension, getAssociatedIconByExt(extension)); + #elif defined FFS_LINUX const IconHolder newIcon = getAssociatedIcon(fileName); - insertIntoBuffer(*buffer_, fileName, newIcon); + buffer_->insertIntoBuffer(fileName, newIcon); #endif } } - //######################### redirect to impl ##################################################### struct IconBuffer::Pimpl @@ -456,7 +427,6 @@ struct IconBuffer::Pimpl workload(std::make_shared<WorkLoad>()), buffer(std::make_shared<Buffer>()) {} - std::shared_ptr<WorkLoad> workload; std::shared_ptr<Buffer> buffer; @@ -477,9 +447,6 @@ IconBuffer::~IconBuffer() pimpl->worker.join(); } -const wxIcon& IconBuffer::getDirectoryIcon() { return ::getDirectoryIcon(); } - -const wxIcon& IconBuffer::getFileIcon() { return ::getFileIcon(); } IconBuffer& IconBuffer::getInstance() { @@ -487,19 +454,18 @@ IconBuffer& IconBuffer::getInstance() return instance; } -bool IconBuffer::requestFileIcon(const Zstring& fileName, wxIcon* icon) { return ::requestFileIcon(*pimpl->buffer, fileName, icon); } - -void IconBuffer::setWorkload(const std::vector<Zstring>& load) -{ - { - boost::lock_guard<boost::mutex> dummy(pimpl->workload->mutex); +bool IconBuffer::requestFileIcon(const Zstring& fileName, wxIcon* icon) { return pimpl->buffer->requestFileIcon(fileName, icon); } - pimpl->workload->filesToLoad.clear(); +void IconBuffer::setWorkload(const std::vector<Zstring>& load) { pimpl->workload->setWorkload(load); } - std::transform(load.begin(), load.end(), std::back_inserter(pimpl->workload->filesToLoad), - [](const Zstring& file) { return BasicString(file); }); //make DEEP COPY from Zstring - } +const wxIcon& IconBuffer::getDirectoryIcon() +{ + static wxIcon dirIcon = ::getDirectoryIcon(); + return dirIcon; +} - pimpl->workload->condition.notify_one(); - //condition handling, see: http://www.boost.org/doc/libs/1_43_0/doc/html/thread/synchronization.html#thread.synchronization.condvar_ref +const wxIcon& IconBuffer::getFileIcon() +{ + static wxIcon fileIcon = ::getFileIcon(); + return fileIcon; } diff --git a/library/icon_buffer.h b/library/icon_buffer.h index 87f0cd4a..00370d1b 100644 --- a/library/icon_buffer.h +++ b/library/icon_buffer.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef ICONBUFFER_H_INCLUDED #define ICONBUFFER_H_INCLUDED diff --git a/library/lock_holder.h b/library/lock_holder.h index dc88ce71..172209dc 100644 --- a/library/lock_holder.h +++ b/library/lock_holder.h @@ -5,6 +5,7 @@ #include "../shared/zstring.h" #include "dir_lock.h" #include "status_handler.h" +#include "dir_exist_async.h" namespace zen { @@ -16,7 +17,10 @@ class LockHolder public: void addDir(const Zstring& dirnameFmt, ProcessCallback& procCallback) //resolved dirname ending with path separator { - if (dirnameFmt.empty()) return; + if (dirnameFmt.empty() || + !dirExistsUpdating(dirnameFmt, procCallback)) + return; + if (lockHolder.find(dirnameFmt) != lockHolder.end()) return; assert(dirnameFmt.EndsWith(FILE_NAME_SEPARATOR)); //this is really the contract, formatting does other things as well, e.g. macro substitution @@ -37,7 +41,7 @@ public: catch (const FileError& e) { bool dummy = false; //this warning shall not be shown but logged only - procCallback.reportWarning(e.msg(), dummy); + procCallback.reportWarning(e.msg(), dummy); //may throw! } } diff --git a/library/norm_filter.h b/library/norm_filter.h index 2e55b43f..609c81db 100644 --- a/library/norm_filter.h +++ b/library/norm_filter.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef NORM_FILTER_H_INCLUDED #define NORM_FILTER_H_INCLUDED diff --git a/library/parallel_scan.cpp b/library/parallel_scan.cpp new file mode 100644 index 00000000..2f91763a --- /dev/null +++ b/library/parallel_scan.cpp @@ -0,0 +1,617 @@ +// ************************************************************************** +// * This file is part of the FreeFileSync project. It is distributed under * +// * GNU General Public License: http://www.gnu.org/licenses/gpl.html * +// * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * +// ************************************************************************** + +#include "parallel_scan.h" +#include <boost/detail/atomic_count.hpp> +#include "db_file.h" +#include "lock_holder.h" +#include "../shared/i18n.h" +#include "../shared/file_traverser.h" +#include "../shared/file_error.h" +#include "../shared/string_conv.h" +#include "../shared/check_exist.h" +#include "../shared/boost_thread_wrap.h" //include <boost/thread.hpp> +#include "loki/ScopeGuard.h" +//#include "../shared/file_id.h" + +/* +#ifdef FFS_WIN +#include <wx/msw/wrapwin.h> //includes "windows.h" +#include "WinIoCtl.h" + +#elif defined FFS_LINUX +#endif +*/ +using namespace zen; + + +#ifndef BOOST_HAS_THREADS +#error just some paranoia check... +#endif + + +namespace +{ +/* +#ifdef FFS_WIN + +struct DiskInfo +{ + DiskInfo() : + driveType(DRIVE_UNKNOWN), + diskID(-1) {} + + UINT driveType; + int diskID; // -1 if id could not be determined, this one is filled if driveType == DRIVE_FIXED or DRIVE_REMOVABLE; +}; + +inline +bool operator<(const DiskInfo& lhs, const DiskInfo& rhs) +{ + if (lhs.driveType != rhs.driveType) + return lhs.driveType < rhs.driveType; + + if (lhs.diskID < 0 || rhs.diskID < 0) + return false; + //consider "same", reason: one volume may be uniquely associated with one disk, while the other volume is associated to the same disk AND another one! + //volume <-> disk is 0..N:1..N + + return lhs.diskID < rhs.diskID ; +} + + +DiskInfo retrieveDiskInfo(const Zstring& pathName) +{ + std::vector<wchar_t> volName(std::max(pathName.size(), static_cast<size_t>(10000))); + + DiskInfo output; + + //full pathName need not yet exist! + if (!::GetVolumePathName(pathName.c_str(), //__in LPCTSTR lpszFileName, + &volName[0], //__out LPTSTR lpszVolumePathName, + static_cast<DWORD>(volName.size()))) //__in DWORD cchBufferLength + return output; + + const Zstring rootPathName = &volName[0]; + + output.driveType = ::GetDriveType(rootPathName.c_str()); + + if (output.driveType == DRIVE_NO_ROOT_DIR) //these two should be the same error category + output.driveType = DRIVE_UNKNOWN; + + if (output.driveType != DRIVE_FIXED && output.driveType != DRIVE_REMOVABLE) + return output; //no reason to get disk ID + + //go and find disk id: + + //format into form: "\\.\C:" + Zstring volnameFmt = rootPathName; + if (endsWith(volnameFmt, FILE_NAME_SEPARATOR)) + volnameFmt.resize(volnameFmt.size() - 1); + volnameFmt = L"\\\\.\\" + volnameFmt; + + HANDLE hVolume = ::CreateFile(volnameFmt.c_str(), + 0, + FILE_SHARE_READ | FILE_SHARE_WRITE, + NULL, + OPEN_EXISTING, + 0, + NULL); + if (hVolume == INVALID_HANDLE_VALUE) + return output; + + Loki::ScopeGuard dummy = Loki::MakeGuard(::CloseHandle, hVolume); + (void)dummy; //silence warning "unused variable" + + std::vector<char> buffer(sizeof(VOLUME_DISK_EXTENTS) + sizeof(DISK_EXTENT)); //reserve buffer for at most one disk! call below will then fail if volume spans multiple disks! + + DWORD bytesReturned = 0; + if (!::DeviceIoControl(hVolume, // handle to device + IOCTL_VOLUME_GET_VOLUME_DISK_EXTENTS, // dwIoControlCode + NULL, // lpInBuffer + 0, // nInBufferSize + &buffer[0], // output buffer + static_cast<DWORD>(buffer.size()), // size of output buffer + &bytesReturned, // number of bytes returned + NULL)) // OVERLAPPED structure + return output; + + const VOLUME_DISK_EXTENTS& volDisks = *reinterpret_cast<VOLUME_DISK_EXTENTS*>(&buffer[0]); + + if (volDisks.NumberOfDiskExtents != 1) + return output; + + output.diskID = volDisks.Extents[0].DiskNumber; + + return output; +} + +#elif defined FFS_LINUX +#endif +*/ + +/* +PERF NOTE + +-------------------------------------------- +|Testcase: Reading from two different disks| +-------------------------------------------- +Windows 7: + 1st(unbuffered) |2nd (OS buffered) + ---------------------------------- +1 Thread: 57s | 8s +2 Threads: 39s | 7s + +-------------------------------------------------- +|Testcase: Reading two directories from same disk| +-------------------------------------------------- +Windows 7: Windows XP: + 1st(unbuffered) |2nd (OS buffered) 1st(unbuffered) |2nd (OS buffered) + ---------------------------------- ---------------------------------- +1 Thread: 41s | 13s 1 Thread: 45s | 13s +2 Threads: 42s | 11s 2 Threads: 38s | 8s + +=> Traversing does not take any advantage of file locality so that even multiple threads operating on the same disk impose no performance overhead. +*/ + + +std::vector<std::set<DirectoryKey>> separateByDistinctDisk(const std::set<DirectoryKey>& dirkeys) +{ + //see perf note: use one thread per dirkey: + typedef std::map<int, std::set<DirectoryKey>> DiskKeyMapping; + DiskKeyMapping tmp; + int index = 0; + std::for_each(dirkeys.begin(), dirkeys.end(), + [&](const DirectoryKey& key) { tmp[++index].insert(key); }); + + /* + //use one thread per physical disk: + typedef std::map<DiskInfo, std::set<DirectoryKey>> DiskKeyMapping; + DiskKeyMapping tmp; + std::for_each(dirkeys.begin(), dirkeys.end(), + [&](const DirectoryKey& key) { tmp[retrieveDiskInfo(key.dirnameFull_)].insert(key); }); + */ + std::vector<std::set<DirectoryKey>> buckets; + std::transform(tmp.begin(), tmp.end(), std::back_inserter(buckets), + [&](const DiskKeyMapping::value_type& diskToKey) { return diskToKey.second; }); + return buckets; +} + +//------------------------------------------------------------------------------------------ +typedef Zbase<wchar_t, StorageRefCountThreadSafe> BasicWString; //thread safe string class for UI texts + + +class AsyncCallback +{ +public: + AsyncCallback() : + notifyingThreadID(-1), + textScanning(_("Scanning:")), + itemsScanned(0), + activeWorker(0) {} + + FillBufferCallback::HandleError reportError(const std::wstring& msg) //blocking call: context of worker thread + { + boost::unique_lock<boost::mutex> dummy(lockErrorMsg); + while(!errorMsg.empty() || errorResponse.get()) + conditionCanReportError.timed_wait(dummy, boost::posix_time::milliseconds(50)); //interruption point! + + errorMsg = BasicWString(msg); + + while(!errorResponse.get()) + conditionGotResponse.timed_wait(dummy, boost::posix_time::milliseconds(50)); //interruption point! + + FillBufferCallback::HandleError rv = *errorResponse; + + errorMsg.clear(); + errorResponse.reset(); + conditionCanReportError.notify_one(); + return rv; + } + + void processErrors(FillBufferCallback& callback) //context of main thread, call repreatedly + { + boost::lock_guard<boost::mutex> dummy(lockErrorMsg); + if (!errorMsg.empty() && !errorResponse.get()) + { + FillBufferCallback::HandleError rv = callback.reportError(cvrtString<std::wstring>(errorMsg)); //throw! + errorResponse.reset(new FillBufferCallback::HandleError(rv)); + conditionGotResponse.notify_one(); + } + } + + void setNotifyingThread(int threadID) { notifyingThreadID = threadID; } //context of main thread + + void reportCurrentFile(const Zstring& filename, int threadID) //context of worker thread + { + if (threadID != notifyingThreadID) return; //only one thread may report status + + boost::lock_guard<boost::mutex> dummy(lockCurrentStatus); + currentFile = filename; + currentStatus.clear(); + } + + void reportCurrentStatus(const std::wstring& status, int threadID) //context of worker thread + { + if (threadID != notifyingThreadID) return; //only one thread may report status + + boost::lock_guard<boost::mutex> dummy(lockCurrentStatus); + currentFile.clear(); + currentStatus = BasicWString(status); //we cannot assume std::wstring to be thread safe (yet)! + } + + std::wstring getCurrentStatus() //context of main thread, call repreatedly + { + std::wstring filename; + std::wstring statusMsg; + { + boost::lock_guard<boost::mutex> dummy(lockCurrentStatus); + if (!currentFile.empty()) + filename = utf8CvrtTo<std::wstring>(currentFile); + else if (!currentStatus.empty()) + statusMsg = cvrtString<std::wstring>(currentStatus); + } + + if (!filename.empty()) + { + std::wstring statusText = cvrtString<std::wstring>(textScanning); + const long activeCount = activeWorker; + if (activeCount >= 2) + { + statusText += L" " + _P("[1 Thread]", "[%x Threads]", activeCount); + replace(statusText, L"%x", toString<std::wstring>(activeCount)); + } + statusText += std::wstring(L" \n") + L'\"' + filename + L'\"'; + return statusText; + } + else + return statusMsg; + } + + void incItemsScanned() { ++itemsScanned; } + long getItemsScanned() const { return itemsScanned; } + + void incActiveWorker() { ++activeWorker; } + void decActiveWorker() { --activeWorker; } + long getActiveWorker() const { return activeWorker; } + +private: + //---- error handling ---- + boost::mutex lockErrorMsg; + boost::condition_variable conditionCanReportError; + boost::condition_variable conditionGotResponse; + BasicWString errorMsg; + std::unique_ptr<FillBufferCallback::HandleError> errorResponse; + + //---- status updates ---- + volatile int notifyingThreadID; //theoretically racy, but there is nothing that could go wrong... + //CAVEAT: do NOT use boost::thread::id as long as this showstopper exists: https://svn.boost.org/trac/boost/ticket/5754 + boost::mutex lockCurrentStatus; //use a different lock for current file: continue traversing while some thread may process an error + Zstring currentFile; //only one of these two is filled at a time! + BasicWString currentStatus; // + + const BasicWString textScanning; //this one is (currently) not shared and could be made a std::wstring, but we stay consistent and use thread-safe variables in this class only! + + //---- status updates II (lock free) ---- + boost::detail::atomic_count itemsScanned; + boost::detail::atomic_count activeWorker; +}; +//------------------------------------------------------------------------------------------------- + +class DirCallback; + +struct TraverserConfig +{ +public: + TraverserConfig(int threadID, + SymLinkHandling handleSymlinks, + const HardFilter::FilterRef& filter, + std::set<Zstring>& failedReads, + AsyncCallback& acb) : + handleSymlinks_(handleSymlinks), + filterInstance(filter), + failedReads_(failedReads), + acb_(acb), + threadID_(threadID) {} + + typedef std::shared_ptr<DirCallback> CallbackPointer; + std::vector<CallbackPointer> callBackBox; //collection of callback pointers to handle ownership + + const SymLinkHandling handleSymlinks_; + const HardFilter::FilterRef filterInstance; //always bound! + std::set<Zstring>& failedReads_; //relative postfixed names of directories that could not be read (empty for root) + + AsyncCallback& acb_; + int threadID_; +}; + + +class DirCallback : public zen::TraverseCallback +{ +public: + DirCallback(TraverserConfig& config, + const Zstring& relNameParentPf, //postfixed with FILE_NAME_SEPARATOR! + DirContainer& output) : + cfg(config), + relNameParentPf_(relNameParentPf), + output_(output) {} + + virtual void onFile (const Zchar* shortName, const Zstring& fullName, const FileInfo& details); + virtual void onSymlink(const Zchar* shortName, const Zstring& fullName, const SymlinkInfo& details); + virtual ReturnValDir onDir (const Zchar* shortName, const Zstring& fullName); + virtual HandleError onError (const std::wstring& errorText); + +private: + TraverserConfig& cfg; + const Zstring relNameParentPf_; + DirContainer& output_; +}; + + +void DirCallback::onFile(const Zchar* shortName, const Zstring& fullName, const FileInfo& details) +{ + boost::this_thread::interruption_point(); + + const Zstring fileNameShort = shortName; + + //do not list the database file(s) sync.ffs_db, sync.x64.ffs_db, etc. or lock files + if (endsWith(fileNameShort, SYNC_DB_FILE_ENDING) || + endsWith(fileNameShort, LOCK_FILE_ENDING)) + return; + + //update status information no matter whether object is excluded or not! + cfg.acb_.reportCurrentFile(fullName, cfg.threadID_); + + //------------------------------------------------------------------------------------ + //apply filter before processing (use relative name!) + if (!cfg.filterInstance->passFileFilter(relNameParentPf_ + fileNameShort)) + return; + + // std::string fileId = details.fileSize >= 1024 * 1024U ? + // util::retrieveFileID(fullName) : + // std::string(); + /* + Perf test Windows 7, SSD, 350k files, 50k dirs, files > 1MB: 7000 + regular: 6.9s + ID per file: 43.9s + ID per file > 1MB: 7.2s + ID per dir: 8.4s + + Linux: retrieveFileID takes about 50% longer in VM! (avoidable because of redundant stat() call!) + */ + + output_.addSubFile(fileNameShort, FileDescriptor(details.lastWriteTimeRaw, details.fileSize)); + + cfg.acb_.incItemsScanned(); //add 1 element to the progress indicator +} + + +void DirCallback::onSymlink(const Zchar* shortName, const Zstring& fullName, const SymlinkInfo& details) +{ + boost::this_thread::interruption_point(); + + if (cfg.handleSymlinks_ == SYMLINK_IGNORE) + return; + + //update status information no matter whether object is excluded or not! + cfg.acb_.reportCurrentFile(fullName, cfg.threadID_); + + //------------------------------------------------------------------------------------ + const Zstring& relName = relNameParentPf_ + shortName; + + //apply filter before processing (use relative name!) + if (!cfg.filterInstance->passFileFilter(relName)) //always use file filter: Link type may not be "stable" on Linux! + return; + + output_.addSubLink(shortName, LinkDescriptor(details.lastWriteTimeRaw, details.targetPath, details.dirLink ? LinkDescriptor::TYPE_DIR : LinkDescriptor::TYPE_FILE)); + + cfg.acb_.incItemsScanned(); //add 1 element to the progress indicator +} + + +TraverseCallback::ReturnValDir DirCallback::onDir(const Zchar* shortName, const Zstring& fullName) +{ + boost::this_thread::interruption_point(); + + //update status information no matter whether object is excluded or not! + cfg.acb_.reportCurrentFile(fullName, cfg.threadID_); + + //------------------------------------------------------------------------------------ + const Zstring& relName = relNameParentPf_ + shortName; + + //apply filter before processing (use relative name!) + bool subObjMightMatch = true; + if (!cfg.filterInstance->passDirFilter(relName, &subObjMightMatch)) + { + if (!subObjMightMatch) + return Loki::Int2Type<ReturnValDir::TRAVERSING_DIR_IGNORE>(); //do NOT traverse subdirs + } + else + cfg.acb_.incItemsScanned(); //add 1 element to the progress indicator + + DirContainer& subDir = output_.addSubDir(shortName); + + TraverserConfig::CallbackPointer subDirCallback = std::make_shared<DirCallback>(cfg, relName + FILE_NAME_SEPARATOR, subDir); + cfg.callBackBox.push_back(subDirCallback); //handle lifetime + //attention: ensure directory filtering is applied later to exclude actually filtered directories + return ReturnValDir(Loki::Int2Type<ReturnValDir::TRAVERSING_DIR_CONTINUE>(), *subDirCallback.get()); +} + + +DirCallback::HandleError DirCallback::onError(const std::wstring& errorText) +{ + switch (cfg.acb_.reportError(errorText)) + { + case FillBufferCallback::TRAV_ERROR_IGNORE: + cfg.failedReads_.insert(relNameParentPf_); + return TRAV_ERROR_IGNORE; + + case FillBufferCallback::TRAV_ERROR_RETRY: + return TRAV_ERROR_RETRY; + } + + assert(false); + return TRAV_ERROR_IGNORE; +} + + +#ifdef FFS_WIN +class DstHackCallbackImpl : public DstHackCallback +{ +public: + DstHackCallbackImpl(AsyncCallback& acb, int threadID) : + acb_(acb), + threadID_(threadID), + textApplyingDstHack(toZ(_("Encoding extended time information: %x")).Replace(Zstr("%x"), Zstr("\n\"%x\""))) {} + +private: + virtual void requestUiRefresh(const Zstring& filename) //applying DST hack imposes significant one-time performance drawback => callback to inform user + { + Zstring statusText = textApplyingDstHack; + replace(statusText, Zstr("%x"), filename); + acb_.reportCurrentStatus(utf8CvrtTo<std::wstring>(statusText), threadID_); + } + + AsyncCallback& acb_; + int threadID_; + const Zstring textApplyingDstHack; +}; +#endif +//------------------------------------------------------------------------------------------ + + +class WorkerThread +{ +public: + WorkerThread(int threadID, + const std::shared_ptr<AsyncCallback>& acb, + const std::vector<std::pair<DirectoryKey, DirectoryValue*>>& workload) : + threadID_(threadID), + acb_(acb), + workload_(workload) {} + + void operator()() //thread entry + { + acb_->incActiveWorker(); + Loki::ScopeGuard dummy = Loki::MakeGuard([&]() { acb_->decActiveWorker(); }); + (void)dummy; + + std::for_each(workload_.begin(), workload_.end(), + [&](std::pair<DirectoryKey, DirectoryValue*>& item) + { + const Zstring& directoryName = item.first.dirnameFull_; + DirectoryValue& dirVal = *item.second; + + acb_->reportCurrentFile(directoryName, threadID_); //just in case directory existence check is blocking! + + if (!directoryName.empty() && + util::dirExistsAsync(directoryName).get()) //blocking + interruption point! + //folder existence already checked in startCompareProcess(): do not treat as error when arriving here! + //perf note: missing network drives should not delay here, as Windows buffers result of last existence check for a short time + { + TraverserConfig travCfg(threadID_, + item.first.handleSymlinks_, //shared by all(!) instances of DirCallback while traversing a folder hierarchy + item.first.filter_, + dirVal.failedReads, + *acb_); + + DirCallback traverser(travCfg, + Zstring(), + dirVal.dirCont); + + bool followSymlinks = false; + switch (item.first.handleSymlinks_) + { + case SYMLINK_IGNORE: + followSymlinks = false; //=> symlinks will be reported via onSymlink() where they are excluded + break; + case SYMLINK_USE_DIRECTLY: + followSymlinks = false; + break; + case SYMLINK_FOLLOW_LINK: + followSymlinks = true; + break; + } + + DstHackCallback* dstCallbackPtr = NULL; +#ifdef FFS_WIN + DstHackCallbackImpl dstCallback(*acb_, threadID_); + dstCallbackPtr = &dstCallback; +#endif + + //get all files and folders from directoryPostfixed (and subdirectories) + traverseFolder(directoryName, followSymlinks, traverser, dstCallbackPtr); //exceptions may be thrown! + } + }); + } + +private: + int threadID_; + std::shared_ptr<AsyncCallback> acb_; + std::vector<std::pair<DirectoryKey, DirectoryValue*>> workload_; +}; +} + + +void zen::fillBuffer(const std::set<DirectoryKey>& keysToRead, //in + std::map<DirectoryKey, DirectoryValue>& buf, //out + FillBufferCallback& callback, + size_t statusInterval) +{ + buf.clear(); + + std::vector<std::set<DirectoryKey>> buckets = separateByDistinctDisk(keysToRead); //one bucket per physical device + + std::vector<boost::thread> worker; //note: GCC doesn't allow to construct an array of empty threads since they would be initialized by const boost::thread& + worker.reserve(buckets.size()); + + Loki::ScopeGuard guardWorker = Loki::MakeGuard([&]() + { + std::for_each(worker.begin(), worker.end(), std::mem_fun_ref(&boost::thread::interrupt)); //interrupt all at once, then join + std::for_each(worker.begin(), worker.end(), std::mem_fun_ref(&boost::thread::join)); + }); + + std::shared_ptr<AsyncCallback> acb = std::make_shared<AsyncCallback>(); + + //init worker threads + for (auto iter = buckets.begin(); iter != buckets.end(); ++iter) + { + int threadID = iter - buckets.begin(); + const std::set<DirectoryKey>& bucket = *iter; + + std::vector<std::pair<DirectoryKey, DirectoryValue*>> workload; + std::for_each(bucket.begin(), bucket.end(), + [&](const DirectoryKey& key) + { + auto rv = buf.insert(std::make_pair(key, DirectoryValue())); + assert(rv.second); + workload.push_back(std::make_pair(key, &rv.first->second)); + }); + + worker.push_back(boost::thread(WorkerThread(threadID, acb, workload))); + } + + //wait until done + for (auto iter = worker.begin(); iter != worker.end(); ++iter) + { + boost::thread& wt = *iter; + int threadID = iter - worker.begin(); + + acb->setNotifyingThread(threadID); //process info messages of first (active) thread only + + do + { + //update status + callback.reportStatus(acb->getCurrentStatus(), acb->getItemsScanned()); //throw! + + //process errors + acb->processErrors(callback); + } + while (!wt.timed_join(boost::posix_time::milliseconds(statusInterval))); + } + + guardWorker.Dismiss(); +} diff --git a/library/parallel_scan.h b/library/parallel_scan.h new file mode 100644 index 00000000..f36c5ec7 --- /dev/null +++ b/library/parallel_scan.h @@ -0,0 +1,74 @@ +// ************************************************************************** +// * This file is part of the FreeFileSync project. It is distributed under * +// * GNU General Public License: http://www.gnu.org/licenses/gpl.html * +// * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * +// ************************************************************************** + +#ifndef PARALLEL_SCAN_H_INCLUDED +#define PARALLEL_SCAN_H_INCLUDED + +#include <map> +#include <set> +#include "hard_filter.h" +#include "../structures.h" +#include "../file_hierarchy.h" + +namespace zen +{ +struct DirectoryKey +{ + DirectoryKey(const Zstring& dirnameFull, + const HardFilter::FilterRef& filter, + SymLinkHandling handleSymlinks) : + dirnameFull_(dirnameFull), + filter_(filter), + handleSymlinks_(handleSymlinks) {} + + Zstring dirnameFull_; + HardFilter::FilterRef filter_; //filter interface: always bound by design! + SymLinkHandling handleSymlinks_; +}; + +inline +bool operator<(const DirectoryKey& lhs, const DirectoryKey& rhs) +{ + if (lhs.handleSymlinks_ != rhs.handleSymlinks_) + return lhs.handleSymlinks_ < rhs.handleSymlinks_; + + if (!EqualFilename()(lhs.dirnameFull_, rhs.dirnameFull_)) + return LessFilename()(lhs.dirnameFull_, rhs.dirnameFull_); + + return *lhs.filter_ < *rhs.filter_; +} + + +struct DirectoryValue +{ + DirContainer dirCont; + std::set<Zstring> failedReads; //relative postfixed names of directories that could not be read (empty for root), e.g. access denied, or temporal network drop +}; + + +class FillBufferCallback +{ +public: + virtual ~FillBufferCallback() {} + + enum HandleError + { + TRAV_ERROR_RETRY, + TRAV_ERROR_IGNORE + }; + virtual HandleError reportError (const std::wstring& errorText) = 0; //may throw! + virtual void reportStatus(const std::wstring& statusMsg, int itemTotal) = 0; // +}; + +//attention: ensure directory filtering is applied later to exclude filtered directories which have been erroneously kept + +void fillBuffer(const std::set<DirectoryKey>& keysToRead, //in + std::map<DirectoryKey, DirectoryValue>& buf, //out + FillBufferCallback& callback, + size_t statusInterval); //unit: [ms] +} + +#endif // PARALLEL_SCAN_H_INCLUDED diff --git a/library/process_xml.cpp b/library/process_xml.cpp index 69dad7f3..276bc977 100644 --- a/library/process_xml.cpp +++ b/library/process_xml.cpp @@ -3,9 +3,9 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #include "process_xml.h" -#include <zenxml/zenxml.h> +#include <zenXml/zenxml.h> #include "../shared/i18n.h" #include "../shared/global_func.h" #include "../shared/standard_paths.h" @@ -42,10 +42,10 @@ XmlType xmlAccess::getXmlType(const wxString& filename) //throw() XmlDoc doc; try { - std::string stream = loadStream(filename); //throw XmlFileError - parse(stream, doc); //throw XmlParsingError + //do NOT use zen::loadStream as it will superfluously load even huge files! + loadXmlDocument(filename, doc); //throw FfsXmlError, quick exit if file is not an FFS XML } - catch (const zen::XmlError&) //catch XmlFileError, XmlParsingError + catch (const FfsXmlError&) { return XML_TYPE_OTHER; } diff --git a/library/process_xml.h b/library/process_xml.h index d7437825..e8a11962 100644 --- a/library/process_xml.h +++ b/library/process_xml.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef PROCESSXML_H_INCLUDED #define PROCESSXML_H_INCLUDED diff --git a/library/resources.cpp b/library/resources.cpp index 179033a7..cbfd7c06 100644 --- a/library/resources.cpp +++ b/library/resources.cpp @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #include "resources.h" #include <wx/wfstream.h> #include <wx/zipstrm.h> diff --git a/library/resources.h b/library/resources.h index 691d5a6b..f812ac90 100644 --- a/library/resources.h +++ b/library/resources.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef RESOURCES_H_INCLUDED #define RESOURCES_H_INCLUDED diff --git a/library/soft_filter.h b/library/soft_filter.h index 0a406907..bde812b0 100644 --- a/library/soft_filter.h +++ b/library/soft_filter.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef SOFT_FILTER_H_INCLUDED #define SOFT_FILTER_H_INCLUDED diff --git a/library/statistics.cpp b/library/statistics.cpp index 812c869b..a29f60af 100644 --- a/library/statistics.cpp +++ b/library/statistics.cpp @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #include "statistics.h" #include <wx/ffile.h> diff --git a/library/statistics.h b/library/statistics.h index a9692d77..0ec82c14 100644 --- a/library/statistics.h +++ b/library/statistics.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef STATISTICS_H_INCLUDED #define STATISTICS_H_INCLUDED diff --git a/library/status_handler.cpp b/library/status_handler.cpp index 0a131899..9c2fdd67 100644 --- a/library/status_handler.cpp +++ b/library/status_handler.cpp @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #include "status_handler.h" #include <wx/app.h> #include <wx/timer.h> diff --git a/library/status_handler.h b/library/status_handler.h index a7984790..a295b6a0 100644 --- a/library/status_handler.h +++ b/library/status_handler.h @@ -3,7 +3,7 @@ // * GNU General Public License: http://www.gnu.org/licenses/gpl.html * // * Copyright (C) 2008-2011 ZenJu (zhnmju123 AT gmx.de) * // ************************************************************************** -// + #ifndef STATUSHANDLER_H_INCLUDED #define STATUSHANDLER_H_INCLUDED |