Enhance Folder as Workspace performance while adding/removing files in bulk

Added batch processing of added and removed files in Folder as Workspace.

Fix #9203,close #9651

PowerEditor/src/WinControls/FileBrowser/fileBrowser.cpp

@@ -244,27 +244,12 @@ INT_PTR CALLBACK FileBrowser::run_dlgProc(UINT message, WPARAM wParam, LPARAM lP
 
 		case FB_ADDFILE:
 		{
-			const std::vector<generic_string> file2Change = *(std::vector<generic_string> *)lParam;
-			generic_string separator = TEXT("\\\\");
-
-			size_t sepPos = file2Change[0].find(separator);
-			if (sepPos == generic_string::npos)
-				return false;
 
-			generic_string pathSuffix = file2Change[0].substr(sepPos + separator.length(), file2Change[0].length() - 1);
+			std::vector<FilesToChange> groupedFiles = getFilesFromParam(lParam);
-
-			// remove prefix of file/folder in changeInfo, splite the remained path
-			vector<generic_string> linarPathArray = split(pathSuffix, '\\');
-
-			generic_string rootPath = file2Change[0].substr(0, sepPos);
-			generic_string path = rootPath;
 
-			generic_string addedFilePath = file2Change[0].substr(0, sepPos + 1);
+			for (auto & group : groupedFiles) 
-			addedFilePath += pathSuffix;
-			bool isAdded = addInTree(rootPath, addedFilePath, nullptr, linarPathArray);
-			if (!isAdded)
 			{
-				//MessageBox(NULL, addedFilePath.c_str(), TEXT("file/folder is not added"), MB_OK);
+				addToTree(group, nullptr);
 			}
 
 			break;
@@ -272,26 +257,14 @@ INT_PTR CALLBACK FileBrowser::run_dlgProc(UINT message, WPARAM wParam, LPARAM lP
 
 		case FB_RMFILE:
 		{
-			const std::vector<generic_string> file2Change = *(std::vector<generic_string> *)lParam;
-			generic_string separator = TEXT("\\\\");
-
-			size_t sepPos = file2Change[0].find(separator);
-			if (sepPos == generic_string::npos)
-				return false;
 
-			generic_string pathSuffix = file2Change[0].substr(sepPos + separator.length(), file2Change[0].length() - 1);
+			std::vector<FilesToChange> groupedFiles = getFilesFromParam(lParam);
-
-			// remove prefix of file/folder in changeInfo, splite the remained path
-			vector<generic_string> linarPathArray = split(pathSuffix, '\\');
-
-			generic_string rootPath = file2Change[0].substr(0, sepPos);
-			// search recursively and modify the tree structure
 
-			bool isRemoved = deleteFromTree(rootPath, nullptr, linarPathArray);
+			for (auto & group : groupedFiles) 
-			if (!isRemoved)
 			{
-				//MessageBox(NULL, file2Change[0].c_str(), TEXT("file/folder is not removed"), MB_OK);
+				deleteFromTree(group);
 			}
+
 			break;
 		}
 
@@ -1138,10 +1111,8 @@ HTREEITEM FileBrowser::getRootFromFullPath(const generic_string & rootPath) cons
 
 HTREEITEM FileBrowser::findChildNodeFromName(HTREEITEM parent, const generic_string& label) const
 {
-	HTREEITEM childNodeFound = nullptr;
-
 	for (HTREEITEM hItemNode = _treeView.getChildFrom(parent);
-		hItemNode != NULL && childNodeFound == nullptr;
+		hItemNode != NULL;
 		hItemNode = _treeView.getNextSibling(hItemNode))
 	{
 		TCHAR textBuffer[MAX_PATH];
@@ -1154,10 +1125,10 @@ HTREEITEM FileBrowser::findChildNodeFromName(HTREEITEM parent, const generic_str
 
 		if (label == tvItem.pszText)
 		{
-			childNodeFound = hItemNode;
+			return hItemNode;
 		}
 	}
-	return childNodeFound;
+	return nullptr;
 }
 
 vector<generic_string> FileBrowser::getRoots() const
@@ -1190,49 +1161,118 @@ generic_string FileBrowser::getSelectedItemPath() const
 	return itemPath;
 }
 
-bool FileBrowser::addInTree(const generic_string& rootPath, const generic_string& addItemFullPath, HTREEITEM node, vector<generic_string> linarPathArray)
+std::vector<FileBrowser::FilesToChange> FileBrowser::getFilesFromParam(LPARAM lParam) const
+{
+	const std::vector<generic_string> filesToChange = *(std::vector<generic_string>*)lParam;
+	const generic_string separator = TEXT("\\\\");
+	const size_t separatorLength = separator.length();
+
+	std::vector<FilesToChange> groupedFiles;
+	for (size_t i = 0; i < filesToChange.size(); i++)
+	{
+		const size_t sepPos = filesToChange[i].find(separator);
+		if (sepPos == generic_string::npos)
+			continue;
+
+		const generic_string pathSuffix = filesToChange[i].substr(sepPos + separatorLength, filesToChange[i].length() - 1);
+
+		// remove prefix of file/folder in changeInfo, split the remained path
+		vector<generic_string> linarPathArray = split(pathSuffix, '\\');
+
+		const generic_string lastElement = linarPathArray.back();
+		linarPathArray.pop_back();
+
+		const generic_string rootPath = filesToChange[i].substr(0, sepPos);
+
+		const generic_string addedFilePath = filesToChange[i].substr(0, sepPos + 1) + pathSuffix;
+
+		generic_string commonPath = rootPath;
+
+		for (const auto & element : linarPathArray)
+		{
+			commonPath.append(TEXT("\\"));
+			commonPath.append(element);
+		}
+
+		commonPath.append(TEXT("\\"));
+
+		const auto it = std::find_if(groupedFiles.begin(), groupedFiles.end(), [&commonPath](const auto & group) { return group._commonPath == commonPath; });
+
+		if (it == groupedFiles.end())
+		{
+			// Add a new file group
+			FilesToChange group;
+			group._commonPath = commonPath;
+			group._rootPath = rootPath;
+			group._linarWithoutLastPathElement = linarPathArray;
+			group._files.push_back(lastElement);
+			groupedFiles.push_back(group);
+		}
+		else
+		{
+			// Add to an existing file group
+			it->_files.push_back(lastElement);
+		}
+	}
+
+	return groupedFiles;
+}
+
+bool FileBrowser::addToTree(FilesToChange & group, HTREEITEM node)
 {
 	if (node == nullptr) // it's a root. Search the right root with rootPath
 	{
 		// Search
-		if ((node = getRootFromFullPath(rootPath)) == nullptr)
+		if ((node = getRootFromFullPath(group._rootPath)) == nullptr)
 			return false;
 	}
 
-	if (linarPathArray.size() == 1)
+	if (group._linarWithoutLastPathElement.size() == 0)
 	{
-		// Of course item to add should be exist on the disk
+		// Items to add should exist on the disk
-		if (!::PathFileExists(addItemFullPath.c_str()))
+		group._files.erase(std::remove_if(group._files.begin(), group._files.end(), 
-			return false;
+			[&group](const auto & file)
+			{
+				return !::PathFileExists((group._commonPath + file).c_str());
+			}),
+			group._files.end());
 
-		// Search : if no found, add
+		if (group._files.empty()) 
-		HTREEITEM childNodeFound = findChildNodeFromName(node, linarPathArray[0]);
+		{
-		if (childNodeFound != nullptr)
 			return false;
+		}
 
-		// No found, good - Action
+		// Search: if not found, add
-		if (::PathIsDirectory(addItemFullPath.c_str()))
+		removeNamesAlreadyInNode(node, group._files);
+		if (group._files.empty()) 
 		{
-			SortingData4lParam* customData = new SortingData4lParam(TEXT(""), linarPathArray[0], true);
+			return false;
-			sortingDataArray.push_back(customData);
-
-			_treeView.addItem(linarPathArray[0].c_str(), node, INDEX_CLOSE_NODE, reinterpret_cast<LPARAM>(customData));
 		}
-		else
-		{
-			SortingData4lParam* customData = new SortingData4lParam(TEXT(""), linarPathArray[0], false);
-			sortingDataArray.push_back(customData);
 
-			_treeView.addItem(linarPathArray[0].c_str(), node, INDEX_LEAF, reinterpret_cast<LPARAM>(customData));
+		// Not found, good - Action
-		}
+		for (auto & file : group._files) {
+			if (::PathIsDirectory((group._commonPath + file).c_str()))
+			{
+				SortingData4lParam* customData = new SortingData4lParam(TEXT(""), file, true);
+				sortingDataArray.push_back(customData);
 
+				_treeView.addItem(file.c_str(), node, INDEX_CLOSE_NODE, reinterpret_cast<LPARAM>(customData));
+			}
+			else
+			{
+				SortingData4lParam* customData = new SortingData4lParam(TEXT(""), file, false);
+				sortingDataArray.push_back(customData);
+
+				_treeView.addItem(file.c_str(), node, INDEX_LEAF, reinterpret_cast<LPARAM>(customData));
+			}
+		}
 		_treeView.customSorting(node, categorySortFunc, 0);
 		return true;
 	}
 	else
 	{
 		for (HTREEITEM hItemNode = _treeView.getChildFrom(node);
-			hItemNode != NULL ;
+			hItemNode != NULL;
 			hItemNode = _treeView.getNextSibling(hItemNode))
 		{
 			TCHAR textBuffer[MAX_PATH];
@@ -1243,15 +1283,33 @@ bool FileBrowser::addInTree(const generic_string& rootPath, const generic_string
 			tvItem.hItem = hItemNode;
 			SendMessage(_treeView.getHSelf(), TVM_GETITEM, 0, reinterpret_cast<LPARAM>(&tvItem));
 
-			if (linarPathArray[0] == tvItem.pszText)
+			if (group._linarWithoutLastPathElement[0] == tvItem.pszText)
 			{
 				// search recursively the node for an action
-				linarPathArray.erase(linarPathArray.begin());
+				group._linarWithoutLastPathElement.erase(group._linarWithoutLastPathElement.begin());
-				return addInTree(rootPath, addItemFullPath, hItemNode, linarPathArray);
+				return addToTree(group, hItemNode);
 			}
 		}
 		return false;
 	}
+
+}
+
+bool FileBrowser::deleteFromTree(FilesToChange & group)
+{
+	std::vector<HTREEITEM> foundItems = findInTree(group, nullptr);
+
+	if (foundItems.empty() == true)
+	{
+		return false;
+	}
+
+	for (auto & item : foundItems)
+	{
+		_treeView.removeItem(item);
+	}
+
+	return true;
 }
 
 HTREEITEM FileBrowser::findInTree(const generic_string& rootPath, HTREEITEM node, std::vector<generic_string> linarPathArray) const
@@ -1297,15 +1355,96 @@ HTREEITEM FileBrowser::findInTree(const generic_string& rootPath, HTREEITEM node
 	}
 }
 
-bool FileBrowser::deleteFromTree(const generic_string& rootPath, HTREEITEM node, const std::vector<generic_string>& linarPathArray)
+std::vector<HTREEITEM> FileBrowser::findInTree(FilesToChange & group, HTREEITEM node) const
 {
-	HTREEITEM foundItem = findInTree(rootPath, node, linarPathArray);
+	if (node == nullptr) // it's a root. Search the right root with rootPath
-	if (foundItem == nullptr)
+	{
-			return false;
+		// Search
+		if ((node = getRootFromFullPath(group._rootPath)) == nullptr)
+		{
+			return {};
+		}
+	}
 
-	// found it, delete it
+	if (group._linarWithoutLastPathElement.empty())
-	_treeView.removeItem(foundItem);
+	{
-	return true;
+		// Search
+		return findChildNodesFromNames(node, group._files);
+	}
+	else
+	{
+		for (HTREEITEM hItemNode = _treeView.getChildFrom(node);
+			hItemNode != NULL;
+			hItemNode = _treeView.getNextSibling(hItemNode))
+		{
+			TCHAR textBuffer[MAX_PATH];
+			TVITEM tvItem;
+			tvItem.mask = TVIF_TEXT;
+			tvItem.pszText = textBuffer;
+			tvItem.cchTextMax = MAX_PATH;
+			tvItem.hItem = hItemNode;
+			SendMessage(_treeView.getHSelf(), TVM_GETITEM, 0, reinterpret_cast<LPARAM>(&tvItem));
+
+			if (group._linarWithoutLastPathElement[0] == tvItem.pszText)
+			{
+				// search recursively the node for an action
+				group._linarWithoutLastPathElement.erase(group._linarWithoutLastPathElement.begin());
+				return findInTree(group, hItemNode);
+			}
+		}
+		return {};
+	}
+}
+
+std::vector<HTREEITEM> FileBrowser::findChildNodesFromNames(HTREEITEM parent, std::vector<generic_string> & labels) const
+{
+	std::vector<HTREEITEM> itemNodes;
+
+	for (HTREEITEM hItemNode = _treeView.getChildFrom(parent);
+		hItemNode != NULL && !labels.empty();
+		hItemNode = _treeView.getNextSibling(hItemNode)
+		)
+	{
+		TCHAR textBuffer[MAX_PATH];
+		TVITEM tvItem;
+		tvItem.mask = TVIF_TEXT;
+		tvItem.pszText = textBuffer;
+		tvItem.cchTextMax = MAX_PATH;
+		tvItem.hItem = hItemNode;
+		SendMessage(_treeView.getHSelf(), TVM_GETITEM, 0, reinterpret_cast<LPARAM>(&tvItem));
+
+		auto it = std::find(labels.begin(), labels.end(), tvItem.pszText);
+		if (it != labels.end())
+		{
+			labels.erase(it); // remove, as it was already found
+			itemNodes.push_back(hItemNode);
+		}
+	}
+	return itemNodes;
+}
+
+void FileBrowser::removeNamesAlreadyInNode(HTREEITEM parent, std::vector<generic_string> & labels) const
+{
+	// We have to search for the labels in the child nodes of parent, and remove the ones that already exist
+	for (HTREEITEM hItemNode = _treeView.getChildFrom(parent);
+		hItemNode != NULL && !labels.empty();
+		hItemNode = _treeView.getNextSibling(hItemNode)
+		)
+	{
+		TCHAR textBuffer[MAX_PATH];
+		TVITEM tvItem;
+		tvItem.mask = TVIF_TEXT;
+		tvItem.pszText = textBuffer;
+		tvItem.cchTextMax = MAX_PATH;
+		tvItem.hItem = hItemNode;
+		SendMessage(_treeView.getHSelf(), TVM_GETITEM, 0, reinterpret_cast<LPARAM>(&tvItem));
+
+		auto it = std::find(labels.begin(), labels.end(), tvItem.pszText);
+		if (it != labels.end())
+		{
+			labels.erase(it);
+		}
+	}
 }
 
 bool FileBrowser::renameInTree(const generic_string& rootPath, HTREEITEM node, const std::vector<generic_string>& linarPathArrayFrom, const generic_string & renameTo)
@@ -1527,53 +1666,57 @@ DWORD WINAPI FolderUpdater::watching(void *params)
 			case WAIT_OBJECT_0 + 1:
 			// We've received a notification in the queue.
 			{
+				static const unsigned int MAX_BATCH_SIZE = 100;
+
+				DWORD dwPreviousAction = 0;
 				DWORD dwAction;
 				generic_string wstrFilename;
+
+				std::vector<generic_string> filesToChange;
 				// Process all available changes, ignore User actions
 				while (changes.Pop(dwAction, wstrFilename))
 				{
-					static generic_string oldName;
-					static std::vector<generic_string> file2Change;
-					file2Change.clear();
 
-					switch (dwAction)
+					// FILE_ACTION_ADDED and FILE_ACTION_REMOVED are done in batches
+					if (dwAction != FILE_ACTION_ADDED && dwAction != FILE_ACTION_REMOVED)
+					{
+						processChange(dwAction, { wstrFilename }, thisFolderUpdater);
+					}
+					else 
 					{
-						case FILE_ACTION_ADDED:
+						// first iteration
-							file2Change.push_back(wstrFilename);
+						if (dwPreviousAction == 0)
-							::SendMessage((thisFolderUpdater->_pFileBrowser)->getHSelf(), FB_ADDFILE, reinterpret_cast<WPARAM>(nullptr), reinterpret_cast<LPARAM>(&file2Change));
+						{
-							oldName = TEXT("");
+							dwPreviousAction = dwAction;
-							break;
+						}
-
+
-						case FILE_ACTION_REMOVED:
+						if (dwPreviousAction == dwAction)
-							file2Change.push_back(wstrFilename);
+						{
-							::SendMessage((thisFolderUpdater->_pFileBrowser)->getHSelf(), FB_RMFILE, reinterpret_cast<WPARAM>(nullptr), reinterpret_cast<LPARAM>(&file2Change));
+							filesToChange.push_back(wstrFilename);
-							oldName = TEXT("");
+
-							break;
+							if (filesToChange.size() > MAX_BATCH_SIZE) // Process some so the editor doesn't block for too long
-
-						case FILE_ACTION_MODIFIED:
-							oldName = TEXT("");
-							break;
-
-						case FILE_ACTION_RENAMED_OLD_NAME:
-							oldName = wstrFilename;
-							break;
-
-						case FILE_ACTION_RENAMED_NEW_NAME:
-							if (!oldName.empty())
 							{
-								file2Change.push_back(oldName);
+								processChange(dwAction, filesToChange, thisFolderUpdater);
-								file2Change.push_back(wstrFilename);
+								filesToChange.clear();
-								//thisFolderUpdater->updateTree(dwAction, file2Change);
-								::SendMessage((thisFolderUpdater->_pFileBrowser)->getHSelf(), FB_RNFILE, reinterpret_cast<WPARAM>(nullptr), reinterpret_cast<LPARAM>(&file2Change));
 							}
-							oldName = TEXT("");
+						}
-							break;
+						else
-
+						{
-						default:
+							// Different action. Process the previous batch and start saving a new one
-							oldName = TEXT("");
+							processChange(dwPreviousAction, filesToChange, thisFolderUpdater);
-							break;
+							filesToChange.clear();
+
+							dwPreviousAction = dwAction;
+							filesToChange.push_back(wstrFilename);
+						}
 					}
 				}
+
+				// process the last changes
+				if (dwAction == FILE_ACTION_ADDED || dwAction == FILE_ACTION_REMOVED)
+				{
+					processChange(dwAction, filesToChange, thisFolderUpdater);
+				}
 			}
 			break;
 
@@ -1589,3 +1732,47 @@ DWORD WINAPI FolderUpdater::watching(void *params)
 	//printStr(L"Quit watching thread");
 	return EXIT_SUCCESS;
 }
+
+void FolderUpdater::processChange(DWORD dwAction, std::vector<generic_string> filesToChange, FolderUpdater* thisFolderUpdater)
+{
+	static generic_string oldName;
+
+	switch (dwAction)
+	{
+	case FILE_ACTION_ADDED:
+
+		::SendMessage((thisFolderUpdater->_pFileBrowser)->getHSelf(), FB_ADDFILE, reinterpret_cast<WPARAM>(nullptr), reinterpret_cast<LPARAM>(&filesToChange));
+		oldName = TEXT("");
+		break;
+
+	case FILE_ACTION_REMOVED:
+		
+		::SendMessage((thisFolderUpdater->_pFileBrowser)->getHSelf(), FB_RMFILE, reinterpret_cast<WPARAM>(nullptr), reinterpret_cast<LPARAM>(&filesToChange));
+		oldName = TEXT("");
+		break;
+
+	case FILE_ACTION_MODIFIED:
+		oldName = TEXT("");
+		break;
+
+	case FILE_ACTION_RENAMED_OLD_NAME:
+		oldName = filesToChange.back();
+		break;
+
+	case FILE_ACTION_RENAMED_NEW_NAME:
+		if (!oldName.empty())
+		{
+			std::vector<generic_string> fileRename;
+			fileRename.push_back(oldName);
+			fileRename.push_back(filesToChange.back());
+			//thisFolderUpdater->updateTree(dwAction, fileRename);
+			::SendMessage((thisFolderUpdater->_pFileBrowser)->getHSelf(), FB_RNFILE, reinterpret_cast<WPARAM>(nullptr), reinterpret_cast<LPARAM>(&fileRename));
+		}
+		oldName = TEXT("");
+		break;
+
+	default:
+		oldName = TEXT("");
+		break;
+	}
+}

PowerEditor/src/WinControls/FileBrowser/fileBrowser.h

@@ -103,6 +103,8 @@ private:
 	HANDLE _watchThreadHandle = nullptr;
 	HANDLE _EventHandle = nullptr;
 	static DWORD WINAPI watching(void *param);
+
+	static void processChange(DWORD dwAction, std::vector<generic_string> filesToChange, FolderUpdater* thisFolderUpdater);
 };
 
 struct SortingData4lParam {
@@ -143,11 +145,10 @@ public:
 	void addRootFolder(generic_string rootFolderPath);
 
 	HTREEITEM getRootFromFullPath(const generic_string & rootPath) const;
-	HTREEITEM findChildNodeFromName(HTREEITEM parent, const generic_string&) const;
+	HTREEITEM findChildNodeFromName(HTREEITEM parent, const generic_string& label) const;
 
-	bool addInTree(const generic_string& rootPath, const generic_string& addItemFullPath, HTREEITEM node, std::vector<generic_string> linarPathArray);
 	HTREEITEM findInTree(const generic_string& rootPath, HTREEITEM node, std::vector<generic_string> linarPathArray) const;
-	bool deleteFromTree(const generic_string& rootPath, HTREEITEM node, const std::vector<generic_string>& linarPathArray);
+
 	void deleteAllFromTree() {
 		popupMenuCmd(IDM_FILEBROWSER_REMOVEALLROOTS);
 	};
@@ -188,6 +189,25 @@ protected:
 
 	bool selectCurrentEditingFile() const;
 
+	struct FilesToChange {
+		generic_string _commonPath; // Common path between all the files. _rootPath + _linarWithoutLastPathElement
+		generic_string _rootPath;
+		std::vector<generic_string> _linarWithoutLastPathElement;
+		std::vector<generic_string> _files; // file/folder names
+	};
+
+	std::vector<FilesToChange> getFilesFromParam(LPARAM lParam) const;
+
+	bool addToTree(FilesToChange & group, HTREEITEM node);
+
+	bool deleteFromTree(FilesToChange & group);
+
+	std::vector<HTREEITEM> findInTree(FilesToChange & group, HTREEITEM node) const;
+
+	std::vector<HTREEITEM> findChildNodesFromNames(HTREEITEM parent, std::vector<generic_string> & labels) const;
+
+	void removeNamesAlreadyInNode(HTREEITEM parent, std::vector<generic_string> & labels) const;
+
 	virtual INT_PTR CALLBACK run_dlgProc(UINT message, WPARAM wParam, LPARAM lParam);
 	void notified(LPNMHDR notification);
 	void showContextMenu(int x, int y);