Adding development plugin DupFileManager

plugins/DupFileManager/DupFileManager.py

@@ -0,0 +1,156 @@
+# Description: This is a Stash plugin which manages duplicate files.
+# By David Maisonave (aka Axter) Jul-2024 (https://www.axter.com/)
+# Get the latest developers version from following link: https://github.com/David-Maisonave/Axter-Stash/tree/main/plugins/DupFileManager
+# Note: To call this script outside of Stash, pass any argument. 
+#       Example:    python DupFileManager.py start
+import os
+import sys
+import time
+import shutil
+import fileinput
+import hashlib
+import json
+from pathlib import Path
+import requests
+import logging
+from logging.handlers import RotatingFileHandler
+import stashapi.log as log # Importing stashapi.log as log for critical events ONLY
+from stashapi.stashapp import StashInterface
+from DupFileManager_config import config # Import config from DupFileManager_config.py
+
+# **********************************************************************
+# Constant global variables --------------------------------------------
+LOG_FILE_PATH = log_file_path = f"{Path(__file__).resolve().parent}\\{Path(__file__).stem}.log" 
+FORMAT = "[%(asctime)s - LN:%(lineno)s] %(message)s"
+PLUGIN_ARGS_MODE = False
+PLUGIN_ID = Path(__file__).stem
+
+RFH = RotatingFileHandler(
+    filename=LOG_FILE_PATH, 
+    mode='a',
+    maxBytes=2*1024*1024, # Configure logging for this script with max log file size of 2000K
+    backupCount=2,
+    encoding=None,
+    delay=0
+)
+TIMEOUT = 5
+CONTINUE_RUNNING_SIG = 99
+
+# **********************************************************************
+# Global variables          --------------------------------------------
+exitMsg = "Change success!!"
+runningInPluginMode = False
+
+# Configure local log file for plugin within plugin folder having a limited max log file size 
+logging.basicConfig(level=logging.INFO, format=FORMAT, datefmt="%y%m%d %H:%M:%S", handlers=[RFH])
+logger = logging.getLogger(Path(__file__).stem)
+    
+# **********************************************************************
+# ----------------------------------------------------------------------
+# Code section to fetch variables from Plugin UI and from DupFileManager_settings.py
+# Check if being called as Stash plugin
+gettingCalledAsStashPlugin = True
+mangeDupFilesTask = True
+StdInRead = None
+try:
+    if len(sys.argv) == 1:
+        print(f"Attempting to read stdin. (len(sys.argv)={len(sys.argv)})", file=sys.stderr)
+        StdInRead = sys.stdin.read()
+        # for line in fileinput.input():
+            # StdInRead = line
+            # break
+    else:
+        raise Exception("Not called in plugin mode.") 
+except:
+    gettingCalledAsStashPlugin = False
+    print(f"Either len(sys.argv) not expected value OR sys.stdin.read() failed! (StdInRead={StdInRead}) (len(sys.argv)={len(sys.argv)})", file=sys.stderr)
+    pass
+    
+if gettingCalledAsStashPlugin and StdInRead:
+    print(f"StdInRead={StdInRead} (len(sys.argv)={len(sys.argv)})", file=sys.stderr)
+    runningInPluginMode = True
+    json_input = json.loads(StdInRead)
+    FRAGMENT_SERVER = json_input["server_connection"]
+else:
+    runningInPluginMode = False
+    FRAGMENT_SERVER = {'Scheme': config['endpoint_Scheme'], 'Host': config['endpoint_Host'], 'Port': config['endpoint_Port'], 'SessionCookie': {'Name': 'session', 'Value': '', 'Path': '', 'Domain': '', 'Expires': '0001-01-01T00:00:00Z', 'RawExpires': '', 'MaxAge': 0, 'Secure': False, 'HttpOnly': False, 'SameSite': 0, 'Raw': '', 'Unparsed': None}, 'Dir': os.path.dirname(Path(__file__).resolve().parent), 'PluginDir': Path(__file__).resolve().parent}
+    print("Running in non-plugin mode!", file=sys.stderr)
+
+stash = StashInterface(FRAGMENT_SERVER)
+PLUGINCONFIGURATION = stash.get_configuration()["plugins"]
+STASHCONFIGURATION = stash.get_configuration()["general"]
+STASHPATHSCONFIG = STASHCONFIGURATION['stashes']
+stashPaths = []
+settings = {
+    "ignoreReparsepoints": True,
+    "ignoreSymbolicLinks": True,
+    "mergeDupFilename": True,
+    "moveToTrashCan": False,
+    "zzdebugTracing": False,
+    "zzdryRun": False,
+}
+
+if PLUGIN_ID in PLUGINCONFIGURATION:
+    settings.update(PLUGINCONFIGURATION[PLUGIN_ID])
+# ----------------------------------------------------------------------
+debugTracing = settings["zzdebugTracing"]
+debugTracing = True
+
+
+if PLUGIN_ID in PLUGINCONFIGURATION:
+    if 'ignoreSymbolicLinks' not in PLUGINCONFIGURATION[PLUGIN_ID]:
+        logger.info(f"Debug Tracing (PLUGIN_ID={PLUGIN_ID})................")
+        logger.info(f"Debug Tracing (PLUGINCONFIGURATION={PLUGINCONFIGURATION})................")
+        try:
+            plugin_configuration = stash.find_plugins_config()
+            logger.info(f"Debug Tracing (plugin_configuration={plugin_configuration})................")
+            stash.configure_plugin(PLUGIN_ID, settings)
+            stash.configure_plugin(PLUGIN_ID, {"zmaximumTagKeys": 12})
+        except Exception as e:
+            logger.exception('Got exception on main handler')
+            try:
+                if debugTracing: logger.info("Debug Tracing................") 
+                stash.configure_plugin(plugin_id=PLUGIN_ID, values=[{"zzdebugTracing": False}], init_defaults=True)
+                if debugTracing: logger.info("Debug Tracing................") 
+            except Exception as e:
+                logger.exception('Got exception on main handler')
+                pass
+            pass
+        # stash.configure_plugin(PLUGIN_ID, settings) # , init_defaults=True
+    if debugTracing: logger.info("Debug Tracing................")    
+
+for item in STASHPATHSCONFIG: 
+    stashPaths.append(item["path"])
+
+# Extract dry_run setting from settings
+DRY_RUN = settings["zzdryRun"]
+dry_run_prefix = ''
+try:
+    PLUGIN_ARGS_MODE    = json_input['args']["mode"]
+except:
+    pass
+logger.info(f"\nStarting (runningInPluginMode={runningInPluginMode}) (debugTracing={debugTracing}) (DRY_RUN={DRY_RUN}) (PLUGIN_ARGS_MODE={PLUGIN_ARGS_MODE})************************************************")
+if debugTracing: logger.info(f"Debug Tracing (stash.get_configuration()={stash.get_configuration()})................")
+if debugTracing: logger.info("settings: %s " % (settings,))
+if debugTracing: logger.info(f"Debug Tracing (STASHCONFIGURATION={STASHCONFIGURATION})................")
+if debugTracing: logger.info(f"Debug Tracing (stashPaths={stashPaths})................")
+if debugTracing: logger.info(f"Debug Tracing (PLUGIN_ID={PLUGIN_ID})................")
+if debugTracing: logger.info(f"Debug Tracing (PLUGINCONFIGURATION={PLUGINCONFIGURATION})................")
+
+if DRY_RUN:
+    logger.info("Dry run mode is enabled.")
+    dry_run_prefix = "Would've "
+if debugTracing: logger.info("Debug Tracing................")
+# ----------------------------------------------------------------------
+# **********************************************************************
+
+def mangeDupFiles():
+    return
+
+if mangeDupFilesTask:
+    mangeDupFiles()
+    if debugTracing: logger.info(f"stop_library_monitor EXIT................")
+else:
+    logger.info(f"Nothing to do!!! (PLUGIN_ARGS_MODE={PLUGIN_ARGS_MODE})")
+
+if debugTracing: logger.info("\n*********************************\nEXITING   ***********************\n*********************************")

plugins/DupFileManager/DupFileManager.yml

@@ -0,0 +1,46 @@
+name: DupFileManager
+description: Manages duplicate files.
+version: 0.1.0
+url: https://github.com/David-Maisonave/Axter-Stash/tree/main/plugins/DupFileManager
+settings:
+  ignoreReparsepoints:
+    displayName: Ignore Reparse Points
+    description: Enable to ignore reparse-points when deleting duplicates.
+    type: BOOLEAN
+  ignoreSymbolicLinks:
+    displayName: Ignore Symbolic Links
+    description: Enable to ignore symbolic links when deleting duplicates.
+    type: BOOLEAN
+  mergeDupFilename:
+    displayName: Before deletion, merge potential source in the duplicate file names for tag names, performers, and studios.
+    description: Enable to 
+    type: BOOLEAN
+  moveToTrashCan:
+    displayName: Trash Can
+    description: Enable to move files to trash can instead of permanently delete file.
+    type: BOOLEAN
+  zzdebugTracing:
+    displayName: Debug Tracing
+    description: (Default=false) [***For Advanced Users***] Enable debug tracing. When enabled, additional tracing logging is added to Stash\plugins\DupFileManager\DupFileManager.log
+    type: BOOLEAN
+  zzdryRun:
+    displayName: Dry Run
+    description: Enable to run script in [Dry Run] mode. In this mode, Stash does NOT call meta_scan, and only logs the action it would have taken.
+    type: BOOLEAN
+exec:
+  - python
+  - "{pluginDir}/DupFileManager.py"
+interface: raw
+tasks:
+  - name: Merge Duplicate Filename
+    description: Merge duplicate filename sourcetag names, performers, and studios.
+    defaultArgs:
+      mode: merge_dup_filename_task
+  - name: Delete Duplicates
+    description: Delete duplicate files
+    defaultArgs:
+      mode: delete_duplicates
+  - name: Dry Run Delete Duplicates
+    description: Only perform a dry run (logging only) of duplicate file deletions. Dry Run setting is ignore when running this task.
+    defaultArgs:
+      mode: dryrun_delete_duplicates

plugins/DupFileManager/DupFileManager_config.py

@@ -0,0 +1,20 @@
+# Description: This is a Stash plugin which manages duplicate files.
+# By David Maisonave (aka Axter) Jul-2024 (https://www.axter.com/)
+# Get the latest developers version from following link: https://github.com/David-Maisonave/Axter-Stash/tree/main/plugins/DupFileManager
+config = {
+    # Define white list of preferential paths to determine which duplicate should be the primary.
+    "whitelist_paths": [],     #Example: "whitelist_paths": ['C:\SomeMediaPath\subpath', 'E:\YetAnotherPath\subpath', 'E:\YetAnotherPath\secondSubPath']
+    # Define black list to determine which duplicates should be deleted first.
+    "blacklist_paths": [],     #Example: "blacklist_paths": ['C:\SomeMediaPath\subpath', 'E:\YetAnotherPath\subpath', 'E:\YetAnotherPath\secondSubPath']
+    # Define ignore list to avoid specific directories. No action is taken on any file in the ignore list.
+    "ignore_paths": [],     #Example: "ignore_paths": ['C:\SomeMediaPath\subpath', 'E:\YetAnotherPath\subpath', 'E:\YetAnotherPath\secondSubPath']
+    # Keep empty to check all paths, or populate it with the only paths to check for duplicates
+    "onlyCheck_paths": [],     #Example: "onlyCheck_paths": ['C:\SomeMediaPath\subpath', 'E:\YetAnotherPath\subpath', 'E:\YetAnotherPath\secondSubPath']
+    # Alternative path to move duplicate files. Path needs to be in the same drive as the duplicate file.
+    "dup_path": "", #Example: "C:\TempDeleteFolder"
+    
+    # The following fields are ONLY used when running DupFileManager in script mode
+    "endpoint_Scheme" : "http", # Define endpoint to use when contacting the Stash server
+    "endpoint_Host" : "0.0.0.0", # Define endpoint to use when contacting the Stash server
+    "endpoint_Port" : 9999, # Define endpoint to use when contacting the Stash server
+}

plugins/DupFileManager/README.md

@@ -0,0 +1,40 @@
+# This Plugin is under construction!!!
+
+# DupFileManager: Ver 0.1.0 (By David Maisonave)
+DupFileManager is a [Stash](https://github.com/stashapp/stash) plugin which manages duplicate file in the Stash system.
+### Features
+- Can merge potential source in the duplicate file names for tag names, performers, and studios.
+  - Normally when Stash searches the file name for tag names, performers, and studios, it only does so using the primary file. This plugin scans the duplicate files to see if additional fields are available.
+- Delete duplicate file task with the following options:
+  - Options in plugin UI (Settings->Plugins->Plugins->[DupFileManager])
+    - Ignore reparse-points. By default, reparse-points are not deleted.
+    - Ignore symbolic links. By default, symbolic links are not deleted.
+    - Before deletion, merge potential source in the duplicate file names for tag names, performers, and studios.
+    - Optionally permanently duplicates or moved them to **trash can** / alternate folder.
+  - Options available via DupFileManager_config.py
+    - Use a white list of preferential directories to determine which duplicate should be the primary.
+    - Use a black list to determine which duplicates should be deleted first.
+    - Use an ignore list to avoid specific directories. No action is taken on any file in the ignore list.
+    - Target directories list. If this list is populated, only files under these directories are process. If list is empty, all files are processed (excluding those in ignore list).
+	- Alternative path to move duplicate files. Path needs to be in the same drive as the duplicate file. 
+	  - Example: "C:\TempDeleteFolder"
+
+### Using DupFileManager
+This Plugin is under construction!!!
+
+### Requirements
+`pip install stashapp-tools`
+`pip install pyYAML`
+
+### Installation
+- Follow **Requirements** instructions.
+- In the stash plugin directory (C:\Users\MyUserName\.stash\plugins), create a folder named **DupFileManager**.
+- Copy all the plugin files to this folder.(**C:\Users\MyUserName\\.stash\plugins\DupFileManager**).
+- Click the **[Reload Plugins]** button in Stash->Settings->Plugins->Plugins.
+
+That's it!!!
+
+### Options
+- Options are accessible in the GUI via Settings->Plugins->Plugins->[DupFileManager].
+- More options available in DupFileManager_config.py.
+

plugins/DupFileManager/requirements.txt

@@ -0,0 +1,4 @@
+stashapp-tools
+pyYAML
+watchdog
+requests

plugins/FileMonitor/README.md

@@ -1,4 +1,4 @@
-# FileMonitor: Ver 0.2.0 (By David Maisonave)
+# FileMonitor: Ver 0.3.0 (By David Maisonave)
 FileMonitor is a [Stash](https://github.com/stashapp/stash) plugin which updates Stash if any changes occurs in the Stash library paths.
 
 ### Using FileMonitor as a plugin
@@ -25,11 +25,12 @@ FileMonitor is a [Stash](https://github.com/stashapp/stash) plugin which updates
 - Follow **Requirements** instructions.
 - In the stash plugin directory (C:\Users\MyUserName\.stash\plugins), create a folder named **FileMonitor**.
 - Copy all the plugin files to this folder.(**C:\Users\MyUserName\\.stash\plugins\FileMonitor**).
-- Restart Stash.
+- Click the **[Reload Plugins]** button in Stash->Settings->Plugins->Plugins.
 
 That's it!!!
 
 ### Options
-- All options are accessible in the GUI via Settings->Plugins->Plugins->[FileMonitor].
+- All are accessible in the GUI via Settings->Plugins->Plugins->[FileMonitor].
+- More options available in filemonitor_config.py.
 
 

plugins/FileMonitor/filemonitor.py

@@ -2,19 +2,13 @@
 # By David Maisonave (aka Axter) Jul-2024 (https://www.axter.com/)
 # Get the latest developers version from following link: https://github.com/David-Maisonave/Axter-Stash/tree/main/plugins/FileMonitor
 # Note: To call this script outside of Stash, pass any argument. 
-#       Example:    python filemonitor.py foofoo
+#       Example:    python filemonitor.py start
 import os
 import sys
-import time
-import shutil
-import fileinput
-import hashlib
 import json
 from pathlib import Path
-import requests
 import logging
 from logging.handlers import RotatingFileHandler
-import stashapi.log as log # Importing stashapi.log as log for critical events ONLY
 from stashapi.stashapp import StashInterface
 from watchdog.observers import Observer # This is also needed for event attributes
 import watchdog  # pip install watchdog  # https://pythonhosted.org/watchdog/
@@ -26,18 +20,8 @@ from filemonitor_config import config # Import settings from filemonitor_config.
 # Constant global variables --------------------------------------------
 LOG_FILE_PATH = log_file_path = f"{Path(__file__).resolve().parent}\\{Path(__file__).stem}.log" 
 FORMAT = "[%(asctime)s - LN:%(lineno)s] %(message)s"
-PLUGIN_ARGS = False
 PLUGIN_ARGS_MODE = False
 PLUGIN_ID = Path(__file__).stem.lower()
-# GraphQL query to fetch all scenes
-QUERY_ALL_SCENES = """
-    query AllScenes {
-        allScenes {
-            id
-            updated_at
-        }
-    }
-"""
 RFH = RotatingFileHandler(
     filename=LOG_FILE_PATH, 
     mode='a',
@@ -73,9 +57,6 @@ try:
     if len(sys.argv) == 1:
         print(f"Attempting to read stdin. (len(sys.argv)={len(sys.argv)})", file=sys.stderr)
         StdInRead = sys.stdin.read()
-        # for line in fileinput.input():
-            # StdInRead = line
-            # break
     else:
         if len(sys.argv) > 1 and sys.argv[1].lower() == "stop":
             stopLibraryMonitoring = True
@@ -124,11 +105,10 @@ for item in STASHPATHSCONFIG:
 DRY_RUN = settings["zzdryRun"]
 dry_run_prefix = ''
 try:
-    PLUGIN_ARGS         = json_input['args']
     PLUGIN_ARGS_MODE    = json_input['args']["mode"]
 except:
     pass
-logger.info(f"\nStarting (runningInPluginMode={runningInPluginMode}) (debugTracing={debugTracing}) (DRY_RUN={DRY_RUN}) (PLUGIN_ARGS_MODE={PLUGIN_ARGS_MODE}) (PLUGIN_ARGS={PLUGIN_ARGS})************************************************")
+logger.info(f"\nStarting (runningInPluginMode={runningInPluginMode}) (debugTracing={debugTracing}) (DRY_RUN={DRY_RUN}) (PLUGIN_ARGS_MODE={PLUGIN_ARGS_MODE})************************************************")
 if debugTracing: logger.info(f"Debug Tracing (stash.get_configuration()={stash.get_configuration()})................")
 if debugTracing: logger.info("settings: %s " % (settings,))
 if debugTracing: logger.info(f"Debug Tracing (STASHCONFIGURATION={STASHCONFIGURATION})................")
@@ -260,6 +240,7 @@ def start_library_monitor():
 # Stops monitoring after triggered by the next file change.
 # ToDo: Add logic so it doesn't have to wait until the next file change
 def stop_library_monitor():
+    import time
     if debugTracing: logger.info("Opening shared memory map.")
     try:
         shm_a = shared_memory.SharedMemory(name="DavidMaisonaveAxter_FileMonitor", create=False, size=4)

plugins/FileMonitor/filemonitor.yml

@@ -1,8 +1,12 @@
 name: FileMonitor
 description: Monitors the Stash library folders, and updates Stash if any changes occurs in the Stash library paths.
-version: 0.2.0
+version: 0.3.0
 url: https://github.com/David-Maisonave/Axter-Stash/tree/main/plugins/FileMonitor
 settings:
+  onCreateCallDupFileManager:
+    displayName: Call DupFileManager on-Create
+    description: When enabled, if CREATE flag is triggered, DupFileManager task is called if the plugin is installed.
+    type: BOOLEAN
   recursiveDisabled:
     displayName: No Recursive
     description: Enable stop monitoring paths recursively.