Too many changes to list

plugins/RenameFile/README.md

@@ -48,8 +48,8 @@ RenameFile is a [Stash](https://github.com/stashapp/stash) plugin.
 - pip install -r requirements.txt
 - Or manually install each requirement:
   - `pip install stashapp-tools --upgrade`
-  - `pip install pyYAML`
   - `pip install requests`
+  - `pip install psutil`
 - For (Windows-Only) optional feature **handleExe**, download handle.exe:
   - https://learn.microsoft.com/en-us/sysinternals/downloads/handle
 

plugins/RenameFile/StashPluginHelper.py

@@ -1,6 +1,6 @@
 from stashapi.stashapp import StashInterface
 from logging.handlers import RotatingFileHandler
-import re, inspect, sys, os, pathlib, logging, json
+import re, inspect, sys, os, pathlib, logging, json, platform, subprocess, traceback, time
 import concurrent.futures
 from stashapi.stash_types import PhashDistance
 import __main__
@@ -61,6 +61,14 @@ class StashPluginHelper(StashInterface):
     LOG_FILE_DIR = None
     LOG_FILE_NAME = None
     STDIN_READ = None
+    stopProcessBarSpin = True
+    
+    IS_DOCKER = False
+    IS_WINDOWS = False
+    IS_LINUX = False
+    IS_FREEBSD = False
+    IS_MAC_OS = False
+    
     pluginLog = None
     logLinePreviousHits = []
     thredPool = None
@@ -107,6 +115,16 @@ class StashPluginHelper(StashInterface):
                     DryRunFieldName = "zzdryRun",
                     setStashLoggerAsPluginLogger = False):              
         self.thredPool = concurrent.futures.ThreadPoolExecutor(max_workers=2)
+        if any(platform.win32_ver()):
+            self.IS_WINDOWS = True
+        elif platform.system().lower().startswith("linux"):
+            self.IS_LINUX = True
+            if self.isDocker():
+                self.IS_DOCKER = True
+        elif platform.system().lower().startswith("freebsd"):
+            self.IS_FREEBSD = True
+        elif sys.platform == "darwin":
+            self.IS_MAC_OS = True
         if logToWrnSet: self.log_to_wrn_set = logToWrnSet
         if logToErrSet: self.log_to_err_set = logToErrSet
         if logToNormSet: self.log_to_norm = logToNormSet
@@ -300,37 +318,43 @@ class StashPluginHelper(StashInterface):
         lineNo = inspect.currentframe().f_back.f_lineno
         self.Log(logMsg, printTo, logging.ERROR, lineNo, toAscii=toAscii)
     
-    def Status(self, printTo = 0, logLevel = logging.INFO, lineNo = -1):
+    # Above logging functions all use UpperCamelCase naming convention to avoid conflict with parent class logging function names.
+    # The below non-loggging functions use (lower) camelCase naming convention.
+    def status(self, printTo = 0, logLevel = logging.INFO, lineNo = -1):
         if printTo == 0: printTo = self.log_to_norm
         if lineNo == -1:
             lineNo = inspect.currentframe().f_back.f_lineno
         self.Log(f"StashPluginHelper Status: (CALLED_AS_STASH_PLUGIN={self.CALLED_AS_STASH_PLUGIN}), (RUNNING_IN_COMMAND_LINE_MODE={self.RUNNING_IN_COMMAND_LINE_MODE}), (DEBUG_TRACING={self.DEBUG_TRACING}), (DRY_RUN={self.DRY_RUN}), (PLUGIN_ID={self.PLUGIN_ID}), (PLUGIN_TASK_NAME={self.PLUGIN_TASK_NAME}), (STASH_URL={self.STASH_URL}), (MAIN_SCRIPT_NAME={self.MAIN_SCRIPT_NAME})",
             printTo, logLevel, lineNo)
     
-    def ExecuteProcess(self, args, ExecDetach=False):
-        import platform, subprocess
-        is_windows = any(platform.win32_ver())
+    def executeProcess(self, args, ExecDetach=False):
         pid = None
-        self.Trace(f"is_windows={is_windows} args={args}")
-        if is_windows:
+        self.Trace(f"self.IS_WINDOWS={self.IS_WINDOWS} args={args}")
+        if self.IS_WINDOWS:
             if ExecDetach:
-                self.Trace("Executing process using Windows DETACHED_PROCESS")
+                self.Trace(f"Executing process using Windows DETACHED_PROCESS; args=({args})")
                 DETACHED_PROCESS = 0x00000008
                 pid = subprocess.Popen(args,creationflags=DETACHED_PROCESS, shell=True).pid
             else:
                 pid = subprocess.Popen(args, shell=True).pid
         else:
-            self.Trace("Executing process using normal Popen")
-            pid = subprocess.Popen(args).pid
+            if ExecDetach:
+                # For linux detached, use nohup. I.E. subprocess.Popen(["nohup", "python", "test.py"])
+                if self.IS_LINUX:
+                    args = ["nohup"] + args
+                self.Trace(f"Executing detached process using Popen({args})")
+            else:
+                self.Trace(f"Executing process using normal Popen({args})")
+            pid = subprocess.Popen(args).pid # On detach, may need the following for MAC OS subprocess.Popen(args, shell=True, start_new_session=True)
         self.Trace(f"pid={pid}")
         return pid
     
-    def ExecutePythonScript(self, args, ExecDetach=True):
+    def executePythonScript(self, args, ExecDetach=True):
         PythonExe = f"{sys.executable}"
         argsWithPython = [f"{PythonExe}"] + args
-        return self.ExecuteProcess(argsWithPython,ExecDetach=ExecDetach)
+        return self.executeProcess(argsWithPython,ExecDetach=ExecDetach)
     
-    def Submit(self, *args, **kwargs):
+    def submit(self, *args, **kwargs):
         return self.thredPool.submit(*args, **kwargs)
     
     def asc2(self, data, convertToAscii=None):
@@ -340,24 +364,214 @@ class StashPluginHelper(StashInterface):
         # data = str(data).encode('ascii','ignore') # This works better for logging than ascii function
         # return str(data)[2:-1] # strip out b'str'
     
-    def init_mergeMetadata(self, excludeMergeTags=None):
+    def initMergeMetadata(self, excludeMergeTags=None):
         self.excludeMergeTags = excludeMergeTags
         self._mergeMetadata = mergeMetadata(self, self.excludeMergeTags)
     
-    # Must call init_mergeMetadata, before calling merge_metadata
-    def merge_metadata(self, SrcData, DestData): # Input arguments can be scene ID or scene metadata
+    # Must call initMergeMetadata, before calling mergeMetadata
+    def mergeMetadata(self, SrcData, DestData): # Input arguments can be scene ID or scene metadata
         if type(SrcData) is int:
             SrcData = self.find_scene(SrcData)
             DestData = self.find_scene(DestData)
         return self._mergeMetadata.merge(SrcData, DestData)
     
-    def Progress(self, currentIndex, maxCount):
+    def progressBar(self, currentIndex, maxCount):
         progress = (currentIndex / maxCount) if currentIndex < maxCount else (maxCount / currentIndex)
         self.log.progress(progress)
     
-    def run_plugin(self, plugin_id, task_mode=None, args:dict={}, asyn=False):
+    # Test via command line: pip uninstall -y pyYAML watchdog schedule requests
+    def modulesInstalled(self, moduleNames, install=True, silent=False): # moduleNames=["stashapp-tools", "requests", "pyYAML"]
+        retrnValue = True
+        for moduleName in moduleNames:
+            try: # Try Python 3.3 > way
+                import importlib
+                import importlib.util
+                if moduleName in sys.modules:
+                    if not silent: self.Trace(f"{moduleName!r} already in sys.modules")
+                elif self.isModuleInstalled(moduleName):
+                    if not silent: self.Trace(f"Module {moduleName!r} is available.")
+                else:
+                    if install and (results:=self.installModule(moduleName)) > 0:
+                        if results == 1:
+                            self.Log(f"Module {moduleName!r} has been installed")
+                        else:
+                            if not silent: self.Trace(f"Module {moduleName!r} is already installed")
+                        continue
+                    else:
+                        if install:
+                            self.Error(f"Can't find the {moduleName!r} module") 
+                        retrnValue = False
+            except Exception as e:
+                try:
+                    i = importlib.import_module(moduleName)
+                except ImportError as e:
+                    if install and (results:=self.installModule(moduleName)) > 0:
+                        if results == 1:
+                            self.Log(f"Module {moduleName!r} has been installed")
+                        else:
+                            if not silent: self.Trace(f"Module {moduleName!r} is already installed")
+                        continue
+                    else:
+                        if install:
+                            tb = traceback.format_exc()
+                            self.Error(f"Can't find the {moduleName!r} module! Error: {e}\nTraceBack={tb}") 
+                        retrnValue = False
+        return retrnValue
+    
+    def isModuleInstalled(self, moduleName):
+        try:
+            __import__(moduleName)
+            # self.Trace(f"Module {moduleName!r} is installed")
+            return True
+        except Exception as e:
+            tb = traceback.format_exc()
+            self.Warn(f"Module {moduleName!r} is NOT installed!") 
+            self.Trace(f"Error: {e}\nTraceBack={tb}")
+            pass
+        return False
+    
+    def installModule(self,moduleName):
+        # if not self.IS_DOCKER:
+            # try:
+                # self.Log(f"Attempting to install package {moduleName!r} using pip import method.")
+                # First try pip import method. (This may fail in a future version of pip.)
+                # self.installPackage(moduleName)
+                # self.Trace(f"installPackage called for module {moduleName!r}")
+                # if self.modulesInstalled(moduleNames=[moduleName], install=False):
+                    # self.Trace(f"Module {moduleName!r} installed")
+                    # return 1
+                # self.Trace(f"Module {moduleName!r} still not installed.")
+            # except Exception as e:
+                # tb = traceback.format_exc()
+                # self.Warn(f"pip import method failed for module {moduleName!r}. Will try command line method; Error: {e}\nTraceBack={tb}") 
+                # pass
+        # else:
+            # self.Trace("Running in Docker, so skipping pip import method.")
+        try:
+            if self.IS_LINUX:
+                # Note: Linux may first need : sudo apt install python3-pip
+                #       if error starts with "Command 'pip' not found"
+                #       or includes "No module named pip"
+                self.Log("Checking if pip installed.")
+                results = os.popen(f"pip --version").read()
+                if results.find("Command 'pip' not found") != -1 or results.find("No module named pip") != -1:
+                    results = os.popen(f"sudo apt install python3-pip").read()
+                    results = os.popen(f"pip --version").read()
+                    if results.find("Command 'pip' not found") != -1 or results.find("No module named pip") != -1:
+                        self.Error(f"Error while calling 'pip'. Make sure pip is installed, and make sure module {moduleName!r} is installed. Results = '{results}'")
+                        return -1
+                self.Trace("pip good.")
+            if self.IS_FREEBSD:
+                self.Warn("installModule may NOT work on freebsd")
+            pipArg = ""
+            if self.IS_DOCKER:
+                pipArg = " --break-system-packages"
+            self.Log(f"Attempting to install package {moduleName!r} via popen.")
+            results = os.popen(f"{sys.executable} -m pip install {moduleName}{pipArg}").read() # May need to be f"{sys.executable} -m pip install {moduleName}"
+            results = results.strip("\n")
+            self.Trace(f"pip results = {results}")
+            if results.find("Requirement already satisfied:") > -1:
+                self.Trace(f"Requirement already satisfied for module {moduleName!r}")
+                return 2
+            elif results.find("Successfully installed") > -1:
+                self.Trace(f"Successfully installed module {moduleName!r}")
+                return 1
+            elif self.modulesInstalled(moduleNames=[moduleName], install=False):
+                self.Trace(f"modulesInstalled returned True for module {moduleName!r}")
+                return 1
+            self.Error(f"Failed to install module {moduleName!r}")
+        except Exception as e:
+            tb = traceback.format_exc()
+            self.Error(f"Failed to install module {moduleName!r}. Error: {e}\nTraceBack={tb}") 
+        return 0
+    
+    def installPackage(self,package): # Should delete this.  It doesn't work consistently
+        try:
+            import pip
+            if hasattr(pip, 'main'):
+                pip.main(['install', package])
+                self.Trace()
+            else:
+                pip._internal.main(['install', package])
+                self.Trace()
+        except Exception as e:
+            tb = traceback.format_exc()
+            self.Error(f"Failed to install module {moduleName!r}. Error: {e}\nTraceBack={tb}")
+            return False
+        return True
+    
+    def isDocker(self):
+        cgroup = pathlib.Path('/proc/self/cgroup')
+        return pathlib.Path('/.dockerenv').is_file() or cgroup.is_file() and 'docker' in cgroup.read_text()
+    
+    def spinProcessBar(self, sleepSeconds = 1, maxPos = 30, trace = False):
+        if trace:
+            self.Trace(f"Starting spinProcessBar loop; sleepSeconds={sleepSeconds}, maxPos={maxPos}")
+        pos = 1
+        while self.stopProcessBarSpin == False:
+            if trace:
+                self.Trace(f"progressBar({pos}, {maxPos})")
+            self.progressBar(pos, maxPos)
+            pos +=1
+            if pos > maxPos:
+                pos = 1
+            time.sleep(sleepSeconds)
+    
+    def startSpinningProcessBar(self, sleepSeconds = 1, maxPos = 30, trace = False):
+        self.stopProcessBarSpin = False
+        if trace:
+            self.Trace(f"submitting spinProcessBar; sleepSeconds={sleepSeconds}, maxPos={maxPos}, trace={trace}")
+        self.submit(self.spinProcessBar, sleepSeconds, maxPos, trace)
+    
+    def stopSpinningProcessBar(self, sleepSeconds = 1):
+        self.stopProcessBarSpin = True
+        time.sleep(sleepSeconds)
+    
+    def createTagId(self, tagName, tagName_descp = "", deleteIfExist = False, ignoreAutoTag = False):
+        tagId = self.find_tags(q=tagName)
+        if len(tagId):
+            tagId = tagId[0]
+            if deleteIfExist:
+                self.destroy_tag(int(tagId['id']))
+            else:
+                return tagId['id']
+        tagId = self.create_tag({"name":tagName, "description":tagName_descp, "ignore_auto_tag": ignoreAutoTag})
+        self.Log(f"Dup-tagId={tagId['id']}")
+        return tagId['id']
+    
+    def removeTag(self, scene, tagName): # scene can be scene ID or scene metadata
+        scene_details = scene
+        if 'id' not in scene:
+            scene_details = self.find_scene(scene)
+        tagIds = []
+        doesHaveTagName = False
+        for tag in scene_details['tags']:
+            if tag['name'] != tagName:
+                tagIds += [tag['id']]
+            else:
+                doesHaveTagName = True
+        if doesHaveTagName:
+            dataDict = {'id' : scene_details['id']}
+            dataDict.update({'tag_ids' : tagIds})
+            self.update_scene(dataDict)
+        return doesHaveTagName
+    
+    def addTag(self, scene, tagName): # scene can be scene ID or scene metadata
+        scene_details = scene
+        if 'id' not in scene:
+            scene_details = self.find_scene(scene)
+        tagIds = [self.createTagId(tagName)]
+        for tag in scene_details['tags']:
+            if tag['name'] != tagName:
+                tagIds += [tag['id']]
+        dataDict = {'id' : scene_details['id']}
+        dataDict.update({'tag_ids' : tagIds})
+        self.update_scene(dataDict)
+    
+    def runPlugin(self, plugin_id, task_mode=None, args:dict={}, asyn=False):
         """Runs a plugin operation.
            The operation is run immediately and does not use the job queue.
+           This is a blocking call, and does not return until plugin completes.
         Args:
             plugin_id (ID):             plugin_id
             task_name (str, optional):  Plugin task to perform
@@ -375,43 +589,26 @@ class StashPluginHelper(StashInterface):
             "args": args,
         }
         if asyn:
-            self.Submit(self.call_GQL, query, variables)
+            self.submit(self.call_GQL, query, variables)
             return f"Made asynchronous call for plugin {plugin_id}"
         else:
             return self.call_GQL(query, variables)
-       
-    def find_duplicate_scenes_diff(self, distance: PhashDistance=PhashDistance.EXACT, fragment='id', duration_diff: float=10.00 ):
-        query = """
-        	query FindDuplicateScenes($distance: Int, $duration_diff: Float) {
-        		findDuplicateScenes(distance: $distance, duration_diff: $duration_diff) {
-        			...SceneSlim
-        		}
-        	}
-        """
-        if fragment:
-        	query = re.sub(r'\.\.\.SceneSlim', fragment, query)
-        else:
-        	query += "fragment SceneSlim on Scene { id  }"
-        
-        variables = { "distance": distance, "duration_diff": duration_diff }
-        result = self.call_GQL(query, variables)
-        return result['findDuplicateScenes'] 
     
-    # #################################################################################################
+    # ############################################################################################################
+    # Functions which are candidates to be added to parent class use snake_case naming convention.
+    # ############################################################################################################
     # The below functions extends class StashInterface with functions which are not yet in the class or
     # fixes for functions which have not yet made it into official class.
-    def metadata_scan(self, paths:list=[], flags={}):
+    def metadata_scan(self, paths:list=[], flags={}): # ToDo: Add option to add path to library if path not included when calling metadata_scan
         query = "mutation MetadataScan($input:ScanMetadataInput!) { metadataScan(input: $input) }"
         scan_metadata_input = {"paths": paths}
         if flags:
             scan_metadata_input.update(flags)
-        else:
-            scanData = self.get_configuration_defaults("scan { ...ScanMetadataOptions }")
-            if scanData['scan'] != None:
-                scan_metadata_input.update(scanData.get("scan",{}))
+        elif scan_config := self.get_configuration_defaults("scan { ...ScanMetadataOptions }").get("scan"):
+            scan_metadata_input.update(scan_config)
         result = self.call_GQL(query, {"input": scan_metadata_input})
         return result["metadataScan"]
-    
+        
     def get_all_scenes(self):
         query_all_scenes = """
             query AllScenes {
@@ -464,6 +661,43 @@ class StashPluginHelper(StashInterface):
     
     def rename_generated_files(self):
         return self.call_GQL("mutation MigrateHashNaming {migrateHashNaming}")
+       
+    def find_duplicate_scenes_diff(self, distance: PhashDistance=PhashDistance.EXACT, fragment='id', duration_diff: float=10.00 ):
+        query = """
+        	query FindDuplicateScenes($distance: Int, $duration_diff: Float) {
+        		findDuplicateScenes(distance: $distance, duration_diff: $duration_diff) {
+        			...SceneSlim
+        		}
+        	}
+        """
+        if fragment:
+        	query = re.sub(r'\.\.\.SceneSlim', fragment, query)
+        else:
+        	query += "fragment SceneSlim on Scene { id  }"
+        
+        variables = { "distance": distance, "duration_diff": duration_diff }
+        result = self.call_GQL(query, variables)
+        return result['findDuplicateScenes'] 
+    
+    # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+    # Direct SQL associated functions
+    def get_file_metadata(self, data, raw_data = False): # data is either file ID or scene metadata
+        results = None
+        if data == None:
+            return results
+        if 'files' in data and len(data['files']) > 0 and 'id' in data['files'][0]:
+            results = self.sql_query(f"select * from files where id =  {data['files'][0]['id']}")
+        else:
+            results = self.sql_query(f"select * from files where id =  {data}")
+        if raw_data:
+            return results
+        if 'rows' in results:
+            return results['rows'][0]
+        self.Error(f"Unknown error while SQL query with data='{data}'; Results='{results}'.")
+        return None
+    
+    def set_file_basename(self, id, basename):
+        return self.sql_commit(f"update files set basename = '{basename}' where id = {id}")
 
 class mergeMetadata: # A class to merge scene metadata from source scene to destination scene
     srcData = None
@@ -537,3 +771,54 @@ class mergeMetadata: # A class to merge scene metadata from source scene to dest
                         listToAdd += [item['id']]
             self.dataDict.update({ updateFieldName : listToAdd})
             # self.stash.Trace(f"Added {fieldName} ({dataAdded}) to scene ID({self.destData['id']})", toAscii=True)
+
+class taskQueue:
+    taskqueue = None
+    def __init__(self, taskqueue):
+        self.taskqueue = taskqueue
+    
+    def tooManyScanOnTaskQueue(self, tooManyQty = 5):
+        count = 0
+        if self.taskqueue == None:
+            return False
+        for jobDetails in self.taskqueue:
+            if jobDetails['description'] == "Scanning...":
+                count += 1
+        if count < tooManyQty:
+            return False
+        return True
+    
+    def cleanJobOnTaskQueue(self):
+        for jobDetails in self.taskqueue:
+            if jobDetails['description'] == "Cleaning...":
+                return True
+        return False
+    
+    def cleanGeneratedJobOnTaskQueue(self):
+        for jobDetails in self.taskqueue:
+            if jobDetails['description'] == "Cleaning generated files...":
+                return True
+        return False
+    
+    def isRunningPluginTaskJobOnTaskQueue(self, taskName):
+        for jobDetails in self.taskqueue:
+            if jobDetails['description'] == "Running plugin task: {taskName}":
+                return True
+        return False
+    
+    def tagDuplicatesJobOnTaskQueue(self):
+        return self.isRunningPluginTaskJobOnTaskQueue("Tag Duplicates")
+    
+    def clearDupTagsJobOnTaskQueue(self):
+        return self.isRunningPluginTaskJobOnTaskQueue("Clear Tags")
+    
+    def generatePhashMatchingJobOnTaskQueue(self):
+        return self.isRunningPluginTaskJobOnTaskQueue("Generate PHASH Matching")
+    
+    def deleteDuplicatesJobOnTaskQueue(self):
+        return self.isRunningPluginTaskJobOnTaskQueue("Delete Duplicates")
+    
+    def deleteTaggedScenesJobOnTaskQueue(self):
+        return self.isRunningPluginTaskJobOnTaskQueue("Delete Tagged Scenes")
+
+

plugins/RenameFile/openedFile.py

@@ -17,7 +17,7 @@ Example Usage:
     of.closeFile(r"B:\V\V\testdup\deleme2.mp4")
     
 """
-import ctypes, os, sys, psutil, argparse, traceback, logging, numbers, string
+import ctypes, os, sys, argparse, traceback, logging, numbers, string
 from ctypes import wintypes
 # from StashPluginHelper import StashPluginHelper
 # Look at the following links to enhance this code:
@@ -30,8 +30,8 @@ from ctypes import wintypes
 #            getPid is the only function which does NOT require elevated admin privileges.
 class openedFile():
     # generic strings and constants
-    ntdll = ctypes.WinDLL('ntdll')
-    kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
+    ntdll = None
+    kernel32 = None
     NTSTATUS = wintypes.LONG
     INVALID_HANDLE_VALUE = wintypes.HANDLE(-1).value
     FILE_READ_ATTRIBUTES = 0x80
@@ -51,57 +51,62 @@ class openedFile():
         self.stash = stash
         if handleExe == None or handleExe == "" or not os.path.isfile(handleExe):
             raise Exception(f"handleExe requires a valid path to Sysinternals 'handle.exe' or 'handle64.exe' executable. Can be downloaded from following link:\nhttps://learn.microsoft.com/en-us/sysinternals/downloads/handle") 
-        # create handle on concerned file with dwDesiredAccess == self.FILE_READ_ATTRIBUTES
-        self.kernel32.CreateFileW.restype = wintypes.HANDLE
-        self.kernel32.CreateFileW.argtypes = (
-            wintypes.LPCWSTR,      # In     lpFileName
-            wintypes.DWORD,        # In     dwDesiredAccess
-            wintypes.DWORD,        # In     dwShareMode
-            self.LPSECURITY_ATTRIBUTES,  # In_opt lpSecurityAttributes
-            wintypes.DWORD,        # In     dwCreationDisposition
-            wintypes.DWORD,        # In     dwFlagsAndAttributes
-            wintypes.HANDLE)       # In_opt hTemplateFile
+        if self.stash != None and self.stash.IS_WINDOWS:
+            self.ntdll = ctypes.WinDLL('ntdll')
+            self.kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
+            # create handle on concerned file with dwDesiredAccess == self.FILE_READ_ATTRIBUTES
+            self.kernel32.CreateFileW.restype = wintypes.HANDLE
+            self.kernel32.CreateFileW.argtypes = (
+                wintypes.LPCWSTR,      # In     lpFileName
+                wintypes.DWORD,        # In     dwDesiredAccess
+                wintypes.DWORD,        # In     dwShareMode
+                self.LPSECURITY_ATTRIBUTES,  # In_opt lpSecurityAttributes
+                wintypes.DWORD,        # In     dwCreationDisposition
+                wintypes.DWORD,        # In     dwFlagsAndAttributes
+                wintypes.HANDLE)       # In_opt hTemplateFile
     
     def getPid(self, path):
         self.lastPath = path
-        hFile = self.kernel32.CreateFileW(
-            path, self.FILE_READ_ATTRIBUTES, self.FILE_SHARE_READ, None, self.OPEN_EXISTING,
-            self.FILE_FLAG_BACKUP_SEMANTICS, None)
-        if hFile == self.INVALID_HANDLE_VALUE:
-            raise ctypes.WinError(ctypes.get_last_error())
-        # prepare data types for system call
-        class IO_STATUS_BLOCK(ctypes.Structure):
-            class _STATUS(ctypes.Union):
-                _fields_ = (('Status', self.NTSTATUS),
-                            ('Pointer', wintypes.LPVOID))
-            _anonymous_ = '_Status',
-            _fields_ = (('_Status', _STATUS),
-                        ('Information', self.ULONG_PTR))
-        iosb = IO_STATUS_BLOCK()
-        class FILE_PROCESS_IDS_USING_FILE_INFORMATION(ctypes.Structure):
-            _fields_ = (('NumberOfProcessIdsInList', wintypes.LARGE_INTEGER),
-                        ('ProcessIdList', wintypes.LARGE_INTEGER * 64))
-        info = FILE_PROCESS_IDS_USING_FILE_INFORMATION()
-        PIO_STATUS_BLOCK = ctypes.POINTER(IO_STATUS_BLOCK)
-        self.ntdll.NtQueryInformationFile.restype = self.NTSTATUS
-        self.ntdll.NtQueryInformationFile.argtypes = (
-            wintypes.HANDLE,        # In  FileHandle
-            PIO_STATUS_BLOCK,       # Out IoStatusBlock
-            wintypes.LPVOID,        # Out FileInformation
-            wintypes.ULONG,         # In  Length
-            self.FILE_INFORMATION_CLASS)  # In  FileInformationClass
-        # system call to retrieve list of PIDs currently using the file
-        status = self.ntdll.NtQueryInformationFile(hFile, ctypes.byref(iosb),
-                                              ctypes.byref(info),
-                                              ctypes.sizeof(info),
-                                              self.FileProcessIdsUsingFileInformation)
-        pidList = info.ProcessIdList[0:info.NumberOfProcessIdsInList]
-        if len(pidList) > 0:
-            return pidList
+        # ToDo: Add Linux implementation
+        if self.stash != None and self.stash.IS_WINDOWS:
+            hFile = self.kernel32.CreateFileW(
+                path, self.FILE_READ_ATTRIBUTES, self.FILE_SHARE_READ, None, self.OPEN_EXISTING,
+                self.FILE_FLAG_BACKUP_SEMANTICS, None)
+            if hFile == self.INVALID_HANDLE_VALUE:
+                raise ctypes.WinError(ctypes.get_last_error())
+            # prepare data types for system call
+            class IO_STATUS_BLOCK(ctypes.Structure):
+                class _STATUS(ctypes.Union):
+                    _fields_ = (('Status', self.NTSTATUS),
+                                ('Pointer', wintypes.LPVOID))
+                _anonymous_ = '_Status',
+                _fields_ = (('_Status', _STATUS),
+                            ('Information', self.ULONG_PTR))
+            iosb = IO_STATUS_BLOCK()
+            class FILE_PROCESS_IDS_USING_FILE_INFORMATION(ctypes.Structure):
+                _fields_ = (('NumberOfProcessIdsInList', wintypes.LARGE_INTEGER),
+                            ('ProcessIdList', wintypes.LARGE_INTEGER * 64))
+            info = FILE_PROCESS_IDS_USING_FILE_INFORMATION()
+            PIO_STATUS_BLOCK = ctypes.POINTER(IO_STATUS_BLOCK)
+            self.ntdll.NtQueryInformationFile.restype = self.NTSTATUS
+            self.ntdll.NtQueryInformationFile.argtypes = (
+                wintypes.HANDLE,        # In  FileHandle
+                PIO_STATUS_BLOCK,       # Out IoStatusBlock
+                wintypes.LPVOID,        # Out FileInformation
+                wintypes.ULONG,         # In  Length
+                self.FILE_INFORMATION_CLASS)  # In  FileInformationClass
+            # system call to retrieve list of PIDs currently using the file
+            status = self.ntdll.NtQueryInformationFile(hFile, ctypes.byref(iosb),
+                                                  ctypes.byref(info),
+                                                  ctypes.sizeof(info),
+                                                  self.FileProcessIdsUsingFileInformation)
+            pidList = info.ProcessIdList[0:info.NumberOfProcessIdsInList]
+            if len(pidList) > 0:
+                return pidList
         return None
     
     def isAdmin(self):
-        if os.name=='nt':
+        if self.stash != None and self.stash.IS_WINDOWS:
             try:
                 return ctypes.windll.shell32.IsUserAnAdmin()
             except:
@@ -112,7 +117,7 @@ class openedFile():
     def runMeAsAdmin(self):
         if self.isAdmin() == True:
             return
-        if os.name=='nt':
+        if self.stash != None and self.stash.IS_WINDOWS:
             # Below is a Windows only method which does NOT popup a console.
             import win32com.shell.shell as shell # Requires: pip install pywin32
             script = os.path.abspath(sys.argv[0])
@@ -130,6 +135,7 @@ class openedFile():
         return filename
     
     def getFilesOpen(self, pid:int): # Requires running with admin privileges.
+        import psutil # Requires: pip install psutil
         p = psutil.Process(pid1)
         return p.open_files()
     

plugins/RenameFile/renamefile.py

@@ -2,13 +2,13 @@
 # By David Maisonave (aka Axter) Jul-2024 (https://www.axter.com/)
 # Get the latest developers version from following link: https://github.com/David-Maisonave/Axter-Stash/tree/main/plugins/RenameFile
 # Based on source code from  https://github.com/Serechops/Serechops-Stash/tree/main/plugins/Renamer
-import os, sys, shutil, json, requests, hashlib, pathlib, logging
+import os, sys, shutil, json, hashlib, pathlib, logging, time, traceback
 from pathlib import Path
 import stashapi.log as log # Importing stashapi.log as log for critical events ONLY
 from stashapi.stashapp import StashInterface
 from StashPluginHelper import StashPluginHelper
+from StashPluginHelper import taskQueue
 from renamefile_settings import config # Import settings from renamefile_settings.py
-from openedFile import openedFile
 
 # **********************************************************************
 # Constant global variables --------------------------------------------
@@ -26,6 +26,7 @@ QUERY_ALL_SCENES = """
 # **********************************************************************
 # Global variables          --------------------------------------------
 inputToUpdateScenePost = False
+doNothing = False
 exitMsg = "Change success!!"
 
 # **********************************************************************
@@ -47,9 +48,11 @@ stash = StashPluginHelper(
         config=config,
         maxbytes=10*1024*1024,
         )
-stash.Status(logLevel=logging.DEBUG)
+# stash.status(logLevel=logging.DEBUG)
 if stash.PLUGIN_ID in stash.PLUGIN_CONFIGURATION:
     stash.pluginSettings.update(stash.PLUGIN_CONFIGURATION[stash.PLUGIN_ID])
+if stash.IS_DOCKER:
+    stash.log_to_wrn_set = stash.LOG_TO_STASH + stash.LOG_TO_FILE
 # ----------------------------------------------------------------------
 WRAPPER_STYLES = config["wrapper_styles"]
 POSTFIX_STYLES = config["postfix_styles"]
@@ -58,10 +61,14 @@ POSTFIX_STYLES = config["postfix_styles"]
 dry_run = stash.pluginSettings["zzdryRun"]
 dry_run_prefix = ''
 try:
-    if stash.JSON_INPUT['args']['hookContext']['input']: inputToUpdateScenePost = True # This avoids calling rename logic twice
+    stash.Trace(f"hookContext={stash.JSON_INPUT['args']['hookContext']}")
+    if stash.JSON_INPUT['args']['hookContext']['input']:
+        if stash.JSON_INPUT['args']['hookContext']['input'] == None:
+            doNothing = True
+        else:
+            inputToUpdateScenePost = True # This avoids calling rename logic twice
 except:
     pass
-stash.Trace("settings: %s " % (stash.pluginSettings,))
 
 if dry_run:
     stash.Log("Dry run mode is enabled.")
@@ -70,16 +77,20 @@ max_tag_keys = stash.pluginSettings["zmaximumTagKeys"] if stash.pluginSettings["
 # ToDo: Add split logic here to slpit possible string array into an array
 exclude_paths = config["pathToExclude"]
 exclude_paths = exclude_paths.split()
-stash.Trace(f"(exclude_paths={exclude_paths})")
+if len(exclude_paths) > 0:
+    stash.Trace(f"(exclude_paths={exclude_paths})")
 excluded_tags = config["excludeTags"]
 # Extract tag whitelist from settings
 tag_whitelist = config["tagWhitelist"]
 if not tag_whitelist:
     tag_whitelist = ""
-stash.Trace(f"(tag_whitelist={tag_whitelist})")
+if len(tag_whitelist) > 0:
+    stash.Trace(f"(tag_whitelist={tag_whitelist})")
 handleExe = stash.pluginConfig['handleExe']
 openedfile = None
 if handleExe != None and handleExe != "" and os.path.isfile(handleExe):
+    stash.modulesInstalled(["psutil"], silent=True)
+    from openedFile import openedFile
     openedfile = openedFile(handleExe, stash)
 
 endpointHost = stash.JSON_INPUT['server_connection']['Host']
@@ -87,7 +98,7 @@ if endpointHost == "0.0.0.0":
     endpointHost = "localhost"
 endpoint = f"{stash.JSON_INPUT['server_connection']['Scheme']}://{endpointHost}:{stash.JSON_INPUT['server_connection']['Port']}/graphql"
 
-stash.Trace(f"(endpoint={endpoint})")
+# stash.Trace(f"(endpoint={endpoint})")
 move_files = stash.pluginSettings["zafileRenameViaMove"]
 fieldKeyList = stash.pluginSettings["zfieldKeyList"] # Default Field Key List with the desired order
 if not fieldKeyList or fieldKeyList == "":
@@ -95,13 +106,13 @@ if not fieldKeyList or fieldKeyList == "":
 fieldKeyList = fieldKeyList.replace(" ", "")
 fieldKeyList = fieldKeyList.replace(";", ",")
 fieldKeyList = fieldKeyList.split(",")
-stash.Trace(f"(fieldKeyList={fieldKeyList})")
+# stash.Trace(f"(fieldKeyList={fieldKeyList})")
 separator = stash.pluginSettings["zseparators"]
 # ----------------------------------------------------------------------
 # **********************************************************************
 
 double_separator = separator + separator
-stash.Trace(f"(WRAPPER_STYLES={WRAPPER_STYLES}) (POSTFIX_STYLES={POSTFIX_STYLES})")
+# stash.Trace(f"(WRAPPER_STYLES={WRAPPER_STYLES}) (POSTFIX_STYLES={POSTFIX_STYLES})")
 
 # Function to replace illegal characters in filenames
 def replace_illegal_characters(filename):
@@ -123,6 +134,7 @@ def form_filename(original_file_stem, scene_details):
     tag_keys_added = 0
     default_title = ''
     if_notitle_use_org_filename = config["if_notitle_use_org_filename"]
+    excludeIgnoreAutoTags = config["excludeIgnoreAutoTags"]
     include_keyField_if_in_name = stash.pluginSettings["z_keyFIeldsIncludeInFileName"]
     if if_notitle_use_org_filename:
         default_title = original_file_stem
@@ -253,12 +265,11 @@ def form_filename(original_file_stem, scene_details):
                     stash.Trace(f"(gallery_name={gallery_name})")
         elif key == 'tags':
             if stash.pluginSettings["tagAppend"]:
-                tags = [tag.get('name', '') for tag in scene_details.get('tags', [])]
-                for tag_name in tags:
-                    stash.Trace(f"(include_keyField_if_in_name={include_keyField_if_in_name}) (tag_name={tag_name})")
-                    if include_keyField_if_in_name or tag_name.lower() not in title.lower():
-                        add_tag(tag_name + POSTFIX_STYLES.get('tag'))
-                        stash.Trace(f"(tag_name={tag_name})")
+                for tag in scene_details['tags']:
+                    stash.Trace(f"(include_keyField_if_in_name={include_keyField_if_in_name}) (tag_name={tag['name']}; ignore_auto_tag={tag['ignore_auto_tag']})")
+                    if (excludeIgnoreAutoTags == False or tag['ignore_auto_tag'] == False) and (include_keyField_if_in_name or tag['name'].lower() not in title.lower()):
+                        add_tag(tag['name'] + POSTFIX_STYLES.get('tag'))
+                        stash.Trace(f"(tag_name={tag['name']})")
     
     stash.Trace(f"(filename_parts={filename_parts})")
     new_filename = separator.join(filename_parts).replace(double_separator, separator)
@@ -273,13 +284,26 @@ def form_filename(original_file_stem, scene_details):
 
 def rename_scene(scene_id):  
     global exitMsg
+    POST_SCAN_DELAY = 3
     scene_details = stash.find_scene(scene_id)
-    stash.Trace(f"(scene_details1={scene_details})")
+    stash.Trace(f"(scene_details={scene_details})")
     if not scene_details:
         stash.Error(f"Scene with ID {scene_id} not found.")
         return None
+    taskqueue = taskQueue(stash.job_queue())
     original_file_path = scene_details['files'][0]['path']
     original_parent_directory = Path(original_file_path).parent
+    maxScanCountDefault = 5
+    maxScanCountForUpdate = 10
+    if scene_details['title'] == None or scene_details['title'] == "":
+        maxScanCountDefault = 1
+        maxScanCountForUpdate = 1
+    if not os.path.isfile(original_file_path) and not taskqueue.clearDupTagsJobOnTaskQueue() and not taskqueue.deleteTaggedScenesJobOnTaskQueue() and not taskqueue.tooManyScanOnTaskQueue(maxScanCountDefault):
+        stash.Warn(f"[metadata_scan] Have to rescan scene ID {scene_id}, because Stash library path '{original_file_path}' does not exist. Scanning path: {original_parent_directory.resolve().as_posix()}")
+        stash.metadata_scan(paths=[original_parent_directory.resolve().as_posix()])
+        time.sleep(POST_SCAN_DELAY) # After a scan, need a few seconds delay before fetching data.
+        scene_details = stash.find_scene(scene_id)
+        original_file_path = scene_details['files'][0]['path']
     stash.Trace(f"(original_file_path={original_file_path})")
     # Check if the scene's path matches any of the excluded paths
     if exclude_paths and any(Path(original_file_path).match(exclude_path) for exclude_path in exclude_paths):
@@ -318,17 +342,60 @@ def rename_scene(scene_id):
                 os.rename(original_file_path, new_file_path)
             exitMsg = f"{dry_run_prefix}Renamed file to '{new_file_path}' from '{original_file_path}'"
     except OSError as e:
-        exitMsg = f"Failed to move/rename file: From {original_file_path} to {new_file_path}. Error: {e}"
+        exitMsg = f"Failed to move/rename file: From {original_file_path} to {new_file_path}; targetDidExist={targetDidExist}. Error: {e}"
         stash.Error(exitMsg)
-        if not targetDidExist and os.path.isfile(new_file_path):
+        if not taskqueue.tooManyScanOnTaskQueue(maxScanCountDefault):
+            stash.Trace(f"Calling [metadata_scan] for path {original_parent_directory.resolve().as_posix()}")
+            stash.metadata_scan(paths=[original_parent_directory.resolve().as_posix()])
+        if targetDidExist:
+            raise
+        if os.path.isfile(new_file_path):
             if os.path.isfile(original_file_path):
                 os.remove(original_file_path)
             pass
         else:
+            # ToDo: Add delay rename here
             raise
     
-    stash.Trace(f"scan path={original_parent_directory.resolve().as_posix()}")
-    stash.metadata_scan(paths=[original_parent_directory.resolve().as_posix()])
+    if not taskqueue.tooManyScanOnTaskQueue(maxScanCountForUpdate):
+        stash.Trace(f"Calling [metadata_scan] for path {original_parent_directory.resolve().as_posix()}")
+        stash.metadata_scan(paths=[original_parent_directory.resolve().as_posix()])
+        time.sleep(POST_SCAN_DELAY) # After a scan, need a few seconds delay before fetching data.
+        scene_details = stash.find_scene(scene_id)
+        if new_file_path != scene_details['files'][0]['path'] and not targetDidExist and not taskqueue.tooManyScanOnTaskQueue(maxScanCountDefault):
+            stash.Trace(f"Calling [metadata_scan] for path {original_parent_directory.resolve().as_posix()}")
+            stash.metadata_scan(paths=[original_parent_directory.resolve().as_posix()])
+            time.sleep(POST_SCAN_DELAY) # After a scan, need a few seconds delay before fetching data.
+            scene_details = stash.find_scene(scene_id)
+            if new_file_path != scene_details['files'][0]['path']:
+                if not os.path.isfile(new_file_path):
+                    stash.Error(f"Failed to rename file from {scene_details['files'][0]['path']} to {new_file_path}.")
+                elif os.path.isfile(scene_details['files'][0]['path']):
+                    stash.Warn(f"Failed to rename file from {scene_details['files'][0]['path']} to {new_file_path}. Old file still exist. Will attempt delay deletion.")
+                    for i in range(1, 5*60):
+                        time.sleep(60)
+                        if not os.path.isfile(new_file_path):
+                            stash.Error(f"Not deleting old file name {original_file_path} because new file name (new_file_path) does NOT exist.")
+                            break
+                        os.remove(original_file_path)
+                        if not os.path.isfile(original_file_path):
+                            stash.Log(f"Deleted {original_file_path} in delay deletion after {i} minutes.")
+                            stash.Trace(f"Calling [metadata_scan] for path {original_parent_directory.resolve().as_posix()}")
+                            stash.metadata_scan(paths=[original_parent_directory.resolve().as_posix()])
+                            break
+                else:
+                    org_stem = Path(scene_details['files'][0]['path']).stem
+                    new_stem = Path(new_file_path).stem
+                    file_id = scene_details['files'][0]['id']
+                    stash.Warn(f"Failed to update Stash library with new name. Will try direct SQL update. org_name={org_stem}; new_name={new_stem}; file_id={file_id}")
+                    # stash.set_file_basename(file_id, new_stem)
+    else:
+        stash.Warn(f"Not performming [metadata_scan] because too many scan jobs are already on the Task Queue. Recommend running a full scan, and a clean job to make sure Stash DB is up to date.")
+        if not taskqueue.cleanJobOnTaskQueue():
+            stash.metadata_scan()
+            stash.metadata_clean()
+        if not taskqueue.cleanGeneratedJobOnTaskQueue():
+            stash.metadata_clean_generated()
     stash.Log(exitMsg)
     return new_filename 
     
@@ -353,13 +420,16 @@ def rename_files_task():
 
 try:
     if stash.PLUGIN_TASK_NAME == "rename_files_task":
+        stash.Trace(f"PLUGIN_TASK_NAME={stash.PLUGIN_TASK_NAME}")
         rename_files_task()
     elif inputToUpdateScenePost:
         rename_files_task()
+    else:
+        stash.Trace(f"Nothing to do. doNothing={doNothing}")
 except Exception as e:
     tb = traceback.format_exc()
     stash.Error(f"Exception while running Plugin. Error: {e}\nTraceBack={tb}")
-    stash.log.exception('Got exception on main handler')
+    # stash.log.exception('Got exception on main handler')
 
 stash.Trace("\n*********************************\nEXITING   ***********************\n*********************************")
 

plugins/RenameFile/renamefile_settings.py

@@ -38,7 +38,7 @@ config = {
         "date": '',      
     },
     # Add tags to exclude from RenameFile.
-    "excludeTags": ["DuplicateMarkForDeletion", "DuplicateMarkForSwap", "DuplicateWhitelistFile","_DuplicateMarkForDeletion","_DuplicateMarkForSwap", "_DuplicateWhitelistFile"],
+    "excludeTags": ["DuplicateMarkForDeletion", "DuplicateMarkForSwap", "DuplicateWhitelistFile","_DuplicateMarkForDeletion","_DuplicateMarkForSwap", "_DuplicateWhitelistFile","ExcludeDuplicateMarkForDeletion", "_ExcludeDuplicateMarkForDeletion"],
     # Add path(s) to exclude from RenameFile. Example Usage: r"/path/to/exclude1"  When entering multiple paths, use space. Example: r"/path_1_to/exclude" r"/someOtherPath2Exclude" r"/yetAnotherPath"
     "pathToExclude": "",
     # Define a whitelist of allowed tags or EMPTY to allow all tags. Example Usage: "tag1", "tag2", "tag3"
@@ -47,6 +47,8 @@ config = {
     "if_notitle_use_org_filename": True, # Warning: Do not recommend setting this to False.
     # Current Stash DB schema only allows maximum base file name length to be 255
     "max_filename_length": 255,
+    # Exclude tags with ignore_auto_tag set to True
+    "excludeIgnoreAutoTags": True,
     
     # handleExe is for Windows only.
     # In Windows, a file can't be renamed if the file is opened by another process.

plugins/RenameFile/requirements.txt

@@ -1,3 +1,3 @@
 stashapp-tools >= 0.2.50
-pyYAML
-requests
+requests
+psutil