From 27f8d5db9a3844c31576a1f65502740dbad19d45 Mon Sep 17 00:00:00 2001 From: David Maisonave <47364845+David-Maisonave@users.noreply.github.com> Date: Thu, 9 Jan 2025 04:20:26 -0500 Subject: [PATCH] Added support for multiple Docker Stash file monitoring --- .gitignore | 1 + plugins/FileMonitor/filemonitor.py | 49 +++++++++++++++++++++++ plugins/FileMonitor/filemonitor_config.py | 25 ++++++++++++ 3 files changed, 75 insertions(+) diff --git a/.gitignore b/.gitignore index 7cd8203..52a226e 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ desktop.ini *.7z *.zip *.swp +*_config_dev.py data.csv /boost /scintilla diff --git a/plugins/FileMonitor/filemonitor.py b/plugins/FileMonitor/filemonitor.py index 04fdd99..77adc38 100644 --- a/plugins/FileMonitor/filemonitor.py +++ b/plugins/FileMonitor/filemonitor.py @@ -103,10 +103,18 @@ if CREATE_SPECIAL_FILE_TO_EXIT and os.path.isfile(SPECIAL_FILE_NAME): fileExtTypes = stash.pluginConfig['fileExtTypes'].split(",") if stash.pluginConfig['fileExtTypes'] != "" else [] includePathChanges = stash.pluginConfig['includePathChanges'] if len(stash.pluginConfig['includePathChanges']) > 0 else stash.STASH_PATHS +includePathChanges = includePathChanges[:] # Make a copy of the list, and not a reference +hostIncludePathChanges = includePathChanges[:] excludePathChanges = stash.pluginConfig['excludePathChanges'] turnOnSchedulerDeleteDup = stash.pluginSettings['turnOnSchedulerDeleteDup'] NotInLibraryTagName = stash.pluginConfig['NotInLibraryTagName'] +filemonitor_config_dev_file = f"{stash.PLUGINS_PATH}{os.sep}FileMonitor{os.sep}filemonitor_config_dev.py" +if os.path.exists(filemonitor_config_dev_file): + stash.Log(f"Getting {filemonitor_config_dev_file} configuration settings.") + from filemonitor_config_dev import config_dev + config['dockers'] = config_dev['dockers'] + dockerMapVolumes = {} dockerReverseMapVolumes = {} dockerObservedPaths = {} @@ -125,6 +133,9 @@ if not parse_args.docker == None and len(parse_args.docker) > 0: for volume in data_loaded['services'][service]['volumes']: volSplit = volume.replace(":ro", "").split(":/") hostPath = volSplit[0] + # Do not scan Stash interanl working folders + if volSplit[1] == "root/.stash" or volSplit[1] == "metadata" or volSplit[1] == "cache" or volSplit[1] == "blobs" or volSplit[1] == "generated": + continue if volSplit[0].startswith("./../../"): dockerStashParentPath = pathlib.Path(dockerStashPath).resolve().parent hostPath = f"{pathlib.Path(dockerStashParentPath).resolve().parent}{hostPath[8:]}" @@ -150,6 +161,32 @@ if stash.IS_DOCKER and stash.PLUGIN_TASK_NAME != "stop_library_monitor" and not sys.exit(10) # ERROR_BAD_ENVIRONMENT: The environment is incorrect. # Alternate error: sys.exit(160) # ERROR_BAD_ARGUMENTS: One or more arguments are not correct. +dockerStashes = {} +for docker in stash.pluginConfig['dockers']: + stash.Log(f"Adding monitoring to Docker Stash {docker['GQL']}") + dockerStashes[docker['GQL']] = StashPluginHelper( + stash_url=docker['GQL'], + debugTracing=parse_args.trace, + settings=settings, + config=config, + logToErrSet=logToErrSet, + logToNormSet=8, + maxbytes=5*1024*1024, + apiKey=docker['apiKey'] + ) + for bindMount in docker['bindMounts']: + for key in bindMount: + if len(key) == 0: + continue + # Do not scan Stash interanl working folders + if bindMount[key] == "/root/.stash" or bindMount[key] == "/metadata" or bindMount[key] == "/cache" or bindMount[key] == "/blobs" or bindMount[key] == "/generated": + continue + stash.Log(f"Adding monitoring for host path '{key}' which is Docker mount path '{bindMount[key]}' for Stash {docker['GQL']}") + includePathChanges += [key] +stash.Log(f"This Stash instance GQL = {stash.STASH_URL}") +# for path in includePathChanges: + # stash.Log(f"[post] includePathChange = {path}") + if stash.DRY_RUN: stash.Log("Dry run mode is enabled.") stash.Trace(f"(SCAN_MODIFIED={SCAN_MODIFIED}) (SCAN_ON_ANY_EVENT={SCAN_ON_ANY_EVENT}) (RECURSIVE={RECURSIVE})") @@ -667,6 +704,9 @@ def start_library_monitor(): pathToObserve = pathToObserve.replace('/', os.sep) dockerObservedPaths[f"{pathToObserve}{os.sep}"] = path stash.Log(f"Observing {pathToObserve}") + if not os.path.exists(pathToObserve): + stash.Error(f"Skipping path '{pathToObserve}' because it does not exist!!!") + continue observer.schedule(event_handler, pathToObserve, recursive=RECURSIVE) observer.schedule(event_handler, SPECIAL_FILE_DIR, recursive=RECURSIVE) stash.Trace(f"Observing FileMonitor path {SPECIAL_FILE_DIR}") @@ -780,6 +820,15 @@ def start_library_monitor(): stash.Log(f"Converted Host-Path {HostTmpTargetPath} to Docker-Path {CpyTmpTargetPath}") TmpTargetPaths += [CpyTmpTargetPath] break + if len(stash.pluginConfig['dockers']) > 0: + for TmpTargetPath in TmpTargetPaths: + for docker in stash.pluginConfig['dockers']: + for bindMount in docker['bindMounts']: + for key in bindMount: + if TmpTargetPath.startswith(key): + stash.Log(f"Sending notification to Stash Docker {docker['GQL']} for file system change in path '{bindMount[key]}' which is host path {key}.") + dockerStashes[docker['GQL']].Log(f"File system change in path '{bindMount[key]}' which is host path {key}.") + dockerStashes[docker['GQL']].metadata_scan(paths=bindMount[key]) stash.Trace(f"[metadata_scan] Calling metadata_scan for paths '{TmpTargetPaths}'") lastScanJob['id'] = int(stash.metadata_scan(paths=TmpTargetPaths)) lastScanJob['TargetPaths'] = TmpTargetPaths diff --git a/plugins/FileMonitor/filemonitor_config.py b/plugins/FileMonitor/filemonitor_config.py index 98249d6..1466315 100644 --- a/plugins/FileMonitor/filemonitor_config.py +++ b/plugins/FileMonitor/filemonitor_config.py @@ -89,6 +89,31 @@ config = { "createSpecFileToExit": True, # Enable to delete special file immediately after it's created in stop process. "deleteSpecFileInStop": False, + # Docker notification from host machine + "dockers": [ + # Example configurations: + # {"GQL":"http://localhost:9995", "apiKey":"", "bindMounts":[ + # {r"C:\Users\admin3\AppData\Local\Docker\wsl\Stash3\data":"/data"}, + # {r"C:\Video":"/mnt/Video"}, + # ] + # }, + # {"GQL":"http://localhost:9997", "apiKey":"", "bindMounts":[ + # {r"C:\Users\admin3\AppData\Local\Docker\wsl\ManyMnt\data":"/data"}, + # {r"C:\Casting":"/external"}, + # {r"C:\Celeb":"/external2"}, + # {r"C:\Hentai":"/external3"}, + # {r"C:\Teens":"/external4"}, + # {r"C:\Temp":"/external5"}, + # {r"C:\Downloads":"/external6"}, + # ] + # }, + # {"GQL":"http://localhost:9994", "apiKey":"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1aWQiOiJheHRlciIsInN1YiI6IkFQSUtleSIsImlhdCI6MTczNDU0MDk3N30.4nZVLk3xikjJZfZ0JTPA_Fic8JvFx3DZe5U21ZDcC3c", "bindMounts":[ + # {r"C:\Users\admin3\AppData\Local\Docker\wsl\MyStashContainer\data":"/data"}, + # {r"E:\Vid":"/external"}, + # {r"E:\Downloads":"/external2"}, + # ] + # }, + ], # Below are place holders for **possible** future features. # !!! Not yet implemented !!!