chris/Axter-Stash
1
0
Fork 0
forked from Github/Axter-Stash
Code Pull Requests Activity

Too many changes to list

Browse Source
This commit is contained in:
David Maisonave
2024-09-13 10:10:37 -04:00
parent 5b34502963
commit 452c08df03
18 changed files with 1645 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
plugins/FileMonitor/filemonitor.py
View File

@@ -5,8 +5,7 @@
# Example: python filemonitor.py --url http://localhost:9999
import os, sys, time, pathlib, argparse, platform, traceback, logging
from StashPluginHelper import StashPluginHelper
import watchdog # pip install watchdog # https://pythonhosted.org/watchdog/
from watchdog.observers import Observer # This is also needed for event attributes
from StashPluginHelper import taskQueue
from threading import Lock, Condition
from multiprocessing import shared_memory
from filemonitor_config import config
@@ -25,7 +24,8 @@ STOP_RUNNING_SIG = 32
parser = argparse.ArgumentParser()
parser.add_argument('--url', '-u', dest='stash_url', type=str, help='Add Stash URL')
parser.add_argument('--trace', '-t', dest='trace', action='store_true', help='Enables debug trace mode.')
parser.add_argument('--stop', '-s', dest='stop', action='store_true', help='Stop (kill) a running FileMonitor task.')
parser.add_argument('--stop', '-s', dest='stop', action='store_true', help='Stop a running FileMonitor task.')
parser.add_argument('--kill_que', '-k', dest='kill_job_task_que', type=str, help='Kill job on Task Queue while running in service mode (command line mode).')
parser.add_argument('--restart', '-r', dest='restart', action='store_true', help='Restart FileMonitor.')
parser.add_argument('--silent', '--quit', '-q', dest='quit', action='store_true', help='Run in silent mode. No output to console or stderr. Use this when running from pythonw.exe')
parser.add_argument('--apikey', '-a', dest='apikey', type=str, help='API Key')
@@ -54,8 +54,10 @@ stash = StashPluginHelper(
maxbytes=5*1024*1024,
apiKey=parse_args.apikey
)
stash.Status(logLevel=logging.DEBUG)
stash.status(logLevel=logging.DEBUG)
stash.Log(f"\nStarting (__file__={__file__}) (stash.CALLED_AS_STASH_PLUGIN={stash.CALLED_AS_STASH_PLUGIN}) (stash.DEBUG_TRACING={stash.DEBUG_TRACING}) (stash.DRY_RUN={stash.DRY_RUN}) (stash.PLUGIN_TASK_NAME={stash.PLUGIN_TASK_NAME})************************************************")
stash.Trace(f"stash.JSON_INPUT={stash.JSON_INPUT}")
stash.modulesInstalled(["watchdog", "schedule", "requests"])
exitMsg = "Change success!!"
mutex = Lock()
@@ -86,6 +88,7 @@ fileExtTypes = stash.pluginConfig['fileExtTypes'].split(",") if stash.pluginConf
includePathChanges = stash.pluginConfig['includePathChanges'] if len(stash.pluginConfig['includePathChanges']) > 0 else stash.STASH_PATHS
excludePathChanges = stash.pluginConfig['excludePathChanges']
turnOnSchedulerDeleteDup = stash.pluginSettings['turnOnSchedulerDeleteDup']
NotInLibraryTagName = stash.pluginConfig['NotInLibraryTagName']
if stash.DRY_RUN:
stash.Log("Dry run mode is enabled.")
@@ -93,34 +96,50 @@ stash.Trace(f"(SCAN_MODIFIED={SCAN_MODIFIED}) (SCAN_ON_ANY_EVENT={SCAN_ON_ANY_EV
StartFileMonitorAsAPluginTaskName = "Monitor as a Plugin"
StartFileMonitorAsAServiceTaskName = "Start Library Monitor Service"
StartFileMonitorAsAPluginTaskID = "start_library_monitor"
StartFileMonitorAsAServiceTaskID = "start_library_monitor_service"
StopFileMonitorAsAPluginTaskID = "stop_library_monitor"
SYNC_LIBRARY_REMOVE = "sync_library_remove"
SYNC_LIBRARY_TAG = "sync_library_tag"
CLEAR_SYNC_LIBRARY_TAG = "clear_sync_tags_task"
FileMonitorPluginIsOnTaskQue = stash.CALLED_AS_STASH_PLUGIN
StopLibraryMonitorWaitingInTaskQueue = False
JobIdInTheQue = 0
def isJobWaitingToRun():
JobIdOf_StartAsAServiceTask = None
def isJobWaitingToRun(getJobIdOf_StartAsAServiceTask = False):
global StopLibraryMonitorWaitingInTaskQueue
global JobIdInTheQue
global JobIdOf_StartAsAServiceTask
global FileMonitorPluginIsOnTaskQue
FileMonitorPluginIsOnTaskQue = False
jobIsWaiting = False
taskQue = stash.job_queue()
for jobDetails in taskQue:
stash.Trace(f"(Job ID({jobDetails['id']})={jobDetails})")
if jobDetails['status'] == "READY":
if jobDetails['description'] == "Running plugin task: Stop Library Monitor":
StopLibraryMonitorWaitingInTaskQueue = True
JobIdInTheQue = jobDetails['id']
jobIsWaiting = True
elif jobDetails['status'] == "RUNNING" and jobDetails['description'].find(StartFileMonitorAsAPluginTaskName) > -1:
FileMonitorPluginIsOnTaskQue = True
if getJobIdOf_StartAsAServiceTask:
if jobDetails['status'] == "RUNNING" and jobDetails['description'].find(StartFileMonitorAsAServiceTaskName) > -1:
JobIdOf_StartAsAServiceTask = jobDetails['id']
stash.Trace(f"Found current running task '{jobDetails['description']}' with Job ID {JobIdOf_StartAsAServiceTask}")
return True
else:
if jobDetails['status'] == "READY":
if jobDetails['description'] == "Running plugin task: Stop Library Monitor":
StopLibraryMonitorWaitingInTaskQueue = True
JobIdInTheQue = jobDetails['id']
jobIsWaiting = True
elif jobDetails['status'] == "RUNNING" and jobDetails['description'].find(StartFileMonitorAsAPluginTaskName) > -1:
FileMonitorPluginIsOnTaskQue = True
JobIdInTheQue = 0
return jobIsWaiting
if stash.CALLED_AS_STASH_PLUGIN and stash.PLUGIN_TASK_NAME == StartFileMonitorAsAPluginTaskID:
if stash.PLUGIN_TASK_NAME == StartFileMonitorAsAPluginTaskID:
stash.Trace(f"isJobWaitingToRun() = {isJobWaitingToRun()})")
elif stash.PLUGIN_TASK_NAME == StartFileMonitorAsAServiceTaskID:
stash.Trace(f"isJobWaitingToRun() = {isJobWaitingToRun(True)})")
class StashScheduler: # Stash Scheduler
def __init__(self):
import schedule # pip install schedule # https://github.com/dbader/schedule
@@ -292,7 +311,7 @@ class StashScheduler: # Stash Scheduler
if 'args' in task and len(task['args']) > 0:
args = args + [task['args']]
stash.Log(f"Executing command arguments {args}.")
return f"Execute process PID = {stash.ExecuteProcess(args)}"
return f"Execute process PID = {stash.executeProcess(args)}"
else:
stash.Error(f"Can not run task '{task['task']}', because it's missing 'command' field.")
return None
@@ -307,7 +326,7 @@ class StashScheduler: # Stash Scheduler
detached = True
if 'detach' in task:
detached = task['detach']
return f"Python process PID = {stash.ExecutePythonScript(args, ExecDetach=detached)}"
return f"Python process PID = {stash.executePythonScript(args, ExecDetach=detached)}"
else:
stash.Error(f"Can not run task '{task['task']}', because it's missing 'script' field.")
return None
@@ -345,8 +364,8 @@ class StashScheduler: # Stash Scheduler
taskMode = task['taskMode']
if ('taskQue' in task and task['taskQue'] == False) or taskName == None:
stash.Log(f"Running plugin task pluginID={task['task']}, task mode = {taskMode}. {validDirMsg}")
# Asynchronous threading logic to call run_plugin, because it's a blocking call.
stash.run_plugin(plugin_id=task['task'], task_mode=taskMode, asyn=True)
# Asynchronous threading logic to call runPlugin, because it's a blocking call.
stash.runPlugin(plugin_id=task['task'], task_mode=taskMode, asyn=True)
return None
else:
stash.Trace(f"Adding to Task Queue plugin task pluginID={task['task']}, task name = {taskName}. {validDirMsg}")
@@ -362,11 +381,11 @@ class StashScheduler: # Stash Scheduler
except:
pass
stash.Error("Failed to get response from Stash.")
if platform.system() == "Windows":
if stash.IS_WINDOWS:
execPath = f"{pathlib.Path(stash.PLUGINS_PATH).resolve().parent}{os.sep}stash-win.exe"
elif platform.system() == "Darwin": # MacOS
elif stash.IS_MAC_OS:
execPath = f"{pathlib.Path(stash.PLUGINS_PATH).resolve().parent}{os.sep} stash-macos "
elif platform.system().lower().startswith("linux"):
elif stash.IS_LINUX:
# ToDo: Need to verify this method will work for (stash-linux-arm32v6, stash-linux-arm32v7, and stash-linux-arm64v8)
if platform.system().lower().find("32v6") > -1:
execPath = f"{pathlib.Path(stash.PLUGINS_PATH).resolve().parent}{os.sep}stash-linux-arm32v6"
@@ -376,7 +395,7 @@ class StashScheduler: # Stash Scheduler
execPath = f"{pathlib.Path(stash.PLUGINS_PATH).resolve().parent}{os.sep}stash-linux-arm64v8"
else:
execPath = f"{pathlib.Path(stash.PLUGINS_PATH).resolve().parent}{os.sep}stash-linux"
elif platform.system().lower().startswith("freebsd"):
elif stash.IS_FREEBSD:
execPath = f"{pathlib.Path(stash.PLUGINS_PATH).resolve().parent}{os.sep}stash-freebsd"
elif 'command' not in task or task['command'] == "":
stash.Error("Can not start Stash, because failed to determine platform OS. As a workaround, add 'command' field to this task.")
@@ -391,7 +410,7 @@ class StashScheduler: # Stash Scheduler
else:
stash.Error("Could not start Stash, because could not find executable Stash file '{execPath}'")
return None
result = f"Execute process PID = {stash.ExecuteProcess(args)}"
result = f"Execute process PID = {stash.executeProcess(args)}"
time.sleep(sleepAfterStart)
if "RunAfter" in task and len(task['RunAfter']) > 0:
for runAfterTask in task['RunAfter']:
@@ -456,6 +475,8 @@ lastScanJob = {
JOB_ENDED_STATUSES = ["FINISHED", "CANCELLED"]
def start_library_monitor():
from watchdog.observers import Observer # This is also needed for event attributes
import watchdog # pip install watchdog # https://pythonhosted.org/watchdog/
global shouldUpdate
global TargetPaths
global lastScanJob
@@ -624,7 +645,7 @@ def start_library_monitor():
TargetPaths = []
TmpTargetPaths = list(set(TmpTargetPaths))
if TmpTargetPaths != [] or lastScanJob['DelayedProcessTargetPaths'] != []:
stash.Log(f"Triggering Stash scan for path(s) {TmpTargetPaths}")
stash.Log(f"Triggering Stash scan for path(s) {TmpTargetPaths} and/or {lastScanJob['DelayedProcessTargetPaths']}")
if lastScanJob['DelayedProcessTargetPaths'] != [] or len(TmpTargetPaths) > 1 or TmpTargetPaths[0] != SPECIAL_FILE_DIR:
if not stash.DRY_RUN:
if lastScanJob['id'] > -1:
@@ -657,11 +678,15 @@ def start_library_monitor():
lastScanJob['DelayedProcessTargetPaths'].append(path)
stash.Trace(f"lastScanJob['DelayedProcessTargetPaths'] = {lastScanJob['DelayedProcessTargetPaths']}")
if lastScanJob['id'] == -1:
stash.Trace(f"Calling metadata_scan for paths '{TmpTargetPaths}'")
lastScanJob['id'] = int(stash.metadata_scan(paths=TmpTargetPaths))
lastScanJob['TargetPaths'] = TmpTargetPaths
lastScanJob['timeAddedToTaskQueue'] = time.time()
stash.Trace(f"metadata_scan JobId = {lastScanJob['id']}, Start-Time = {lastScanJob['timeAddedToTaskQueue']}, paths = {lastScanJob['TargetPaths']}")
taskqueue = taskQueue(stash.job_queue())
if taskqueue.tooManyScanOnTaskQueue(7):
stash.Log(f"[metadata_scan] Skipping updating Stash for paths '{TmpTargetPaths}', because too many scans on Task Queue.")
else:
stash.Trace(f"[metadata_scan] Calling metadata_scan for paths '{TmpTargetPaths}'")
lastScanJob['id'] = int(stash.metadata_scan(paths=TmpTargetPaths))
lastScanJob['TargetPaths'] = TmpTargetPaths
lastScanJob['timeAddedToTaskQueue'] = time.time()
stash.Trace(f"metadata_scan JobId = {lastScanJob['id']}, Start-Time = {lastScanJob['timeAddedToTaskQueue']}, paths = {lastScanJob['TargetPaths']}")
if RUN_CLEAN_AFTER_DELETE and RunCleanMetadata:
stash.metadata_clean(paths=TmpTargetPaths, dry_run=stash.DRY_RUN)
if RUN_GENERATE_CONTENT:
@@ -723,9 +748,81 @@ def start_library_monitor_service():
pass
stash.Trace("FileMonitor is not running, so it's safe to start it as a service.")
args = [f"{pathlib.Path(__file__).resolve().parent}{os.sep}filemonitor.py", '--url', f"{stash.STASH_URL}"]
if JobIdOf_StartAsAServiceTask != None:
args += ["-k", JobIdOf_StartAsAServiceTask]
if stash.API_KEY:
args = args + ["-a", stash.API_KEY]
stash.ExecutePythonScript(args)
args += ["-a", stash.API_KEY]
results = stash.executePythonScript(args)
stash.Trace(f"executePythonScript results='{results}'")
def synchronize_library(removeScene=False):
stash.startSpinningProcessBar()
scenes = stash.find_scenes(fragment='id tags {id name} files {path}')
qtyResults = len(scenes)
Qty = 0
stash.Log(f"count = {qtyResults}")
stash.stopSpinningProcessBar()
sceneIDs = stash.find_scenes(fragment='id files {path}')
for scene in scenes:
Qty += 1
stash.progressBar(Qty, qtyResults)
scenePartOfLibrary = False
for path in stash.STASH_PATHS:
if scene['files'][0]['path'].startswith(path):
scenePartOfLibrary = True
break
if scenePartOfLibrary == False:
stash.Log(f"Scene ID={scene['id']}; path={scene['files'][0]['path']} not part of Stash Library")
if removeScene:
stash.destroy_scene(scene['id'])
stash.Log(f"Removed Scene ID={scene['id']}; path={scene['files'][0]['path']}")
else:
stash.addTag(scene, NotInLibraryTagName)
stash.Trace(f"Tagged ({NotInLibraryTagName}) Scene ID={scene['id']}; path={scene['files'][0]['path']}")
def manageTagggedScenes(clearTag=True):
tagId = stash.find_tags(q=NotInLibraryTagName)
if len(tagId) > 0 and 'id' in tagId[0]:
tagId = tagId[0]['id']
else:
stash.Warn(f"Could not find tag ID for tag '{NotInLibraryTagName}'.")
return
QtyDup = 0
QtyRemoved = 0
QtyClearedTags = 0
QtyFailedQuery = 0
stash.Trace("#########################################################################")
stash.startSpinningProcessBar()
stash.Trace(f"Calling find_scenes with tagId={tagId}")
sceneIDs = stash.find_scenes(f={"tags": {"value":tagId, "modifier":"INCLUDES"}}, fragment='id')
stash.stopSpinningProcessBar()
qtyResults = len(sceneIDs)
stash.Trace(f"Found {qtyResults} scenes with tag ({NotInLibraryTagName}): sceneIDs = {sceneIDs}")
for sceneID in sceneIDs:
# stash.Trace(f"Getting scene data for scene ID {sceneID['id']}.")
QtyDup += 1
prgs = QtyDup / qtyResults
stash.progressBar(QtyDup, qtyResults)
scene = stash.find_scene(sceneID['id'])
if scene == None or len(scene) == 0:
stash.Warn(f"Could not get scene data for scene ID {sceneID['id']}.")
QtyFailedQuery += 1
continue
# stash.Trace(f"scene={scene}")
if clearTag:
tags = [int(item['id']) for item in scene["tags"] if item['id'] != tagId]
stash.TraceOnce(f"tagId={tagId}, len={len(tags)}, tags = {tags}")
dataDict = {'id' : scene['id']}
dataDict.update({'tag_ids' : tags})
stash.Log(f"Updating scene with {dataDict}")
stash.update_scene(dataDict)
# stash.removeTag(scene, NotInLibraryTagName)
QtyClearedTags += 1
else:
stash.destroy_scene(scene['id'])
stash.Log(f"Removed Scene ID={scene['id']}; path={scene['files'][0]['path']}")
QtyRemoved += 1
stash.Log(f"QtyDup={QtyDup}, QtyClearedTags={QtyClearedTags}, QtyRemoved={QtyRemoved}, QtyFailedQuery={QtyFailedQuery}")
runTypeID=0
runTypeName=["NothingToDo", "stop_library_monitor", "StartFileMonitorAsAServiceTaskID", "StartFileMonitorAsAPluginTaskID", "CommandLineStartLibMonitor"]
@@ -742,13 +839,29 @@ try:
elif stash.PLUGIN_TASK_NAME == StartFileMonitorAsAServiceTaskID:
runTypeID=2
start_library_monitor_service()
stash.Trace(f"{StartFileMonitorAsAServiceTaskID} EXIT")
stash.Trace(f"{StartFileMonitorAsAServiceTaskID} transitioning to service mode.")
elif stash.PLUGIN_TASK_NAME == StartFileMonitorAsAPluginTaskID:
runTypeID=3
start_library_monitor()
stash.Trace(f"{StartFileMonitorAsAPluginTaskID} EXIT")
elif stash.PLUGIN_TASK_NAME == SYNC_LIBRARY_REMOVE:
runTypeID=5
synchronize_library(removeScene=tRUE)
stash.Trace(f"{SYNC_LIBRARY_REMOVE} EXIT")
elif stash.PLUGIN_TASK_NAME == SYNC_LIBRARY_TAG:
runTypeID=6
synchronize_library()
stash.Trace(f"{SYNC_LIBRARY_TAG} EXIT")
elif stash.PLUGIN_TASK_NAME == CLEAR_SYNC_LIBRARY_TAG:
runTypeID=7
manageTagggedScenes()
stash.Trace(f"{CLEAR_SYNC_LIBRARY_TAG} EXIT")
elif not stash.CALLED_AS_STASH_PLUGIN:
runTypeID=4
if parse_args.kill_job_task_que != None and parse_args.kill_job_task_que != "":
# Removing the job from the Task Queue is really only needed for Linux, but it should be OK to do in general.
stash.Log(f"Removing job ID {parse_args.kill_job_task_que} from the Task Queue, because transitioning to service mode.")
stash.stop_job(parse_args.kill_job_task_que)
start_library_monitor()
stash.Trace("Command line FileMonitor EXIT")
else:
@@ -756,6 +869,7 @@ try:
except Exception as e:
tb = traceback.format_exc()
stash.Error(f"Exception while running FileMonitor. runType='{runTypeName[runTypeID]}'; Error: {e}\nTraceBack={tb}")
stash.log.exception('Got exception on main handler')
stash.Trace("\n*********************************\nEXITING ***********************\n*********************************")
# ToDo: Add option to add path to library if path not included when calling metadata_scan