From a6f3b352a362d4e119c45a632c7a3cd463c1319b Mon Sep 17 00:00:00 2001 From: David Maisonave <47364845+David-Maisonave@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:18:41 -0500 Subject: [PATCH] 1.0.0.3 ### 1.0.0.3 - Added option on report to merge all metadata missing in [**Duplicate to Keep**] files. - Added cookies to report so as to remember user options for Disable Complete Confirmation **[Disable Complete Confirmation]** and **[Disable Delete Confirmation]**. - This change was needed because sometimes the browser refuse to open local URL's with params on the URL. - Using cookies also allows check options status to stay the same after refresh. - Added code to [**Advance Duplicate File Deletion Menu**] to delete based on flags. --- plugins/DupFileManager/DupFileManager.py | 142 +++++++++++++----- plugins/DupFileManager/DupFileManager.yml | 2 +- .../DupFileManager_report_config.py | 72 ++++++--- plugins/DupFileManager/README.md | 3 +- plugins/DupFileManager/advance_options.html | 20 ++- .../DupFileManager/version_history/README.md | 6 + plugins/FileMonitor/StashPluginHelper.py | 21 ++- plugins/RenameFile/StashPluginHelper.py | 21 ++- 8 files changed, 218 insertions(+), 69 deletions(-) diff --git a/plugins/DupFileManager/DupFileManager.py b/plugins/DupFileManager/DupFileManager.py index 0260df8..57fa9b2 100644 --- a/plugins/DupFileManager/DupFileManager.py +++ b/plugins/DupFileManager/DupFileManager.py @@ -796,10 +796,12 @@ def writeRowToHtmlReport(fileHtmlReport, DupFile, DupFileToKeep, QtyTagForDel = # rename file fileHtmlReport.write(f"

") - fileHtmlReport.write("\n") + fileHtmlReport.write(f"\n") fragmentForSceneDetails = 'id tags {id name ignore_auto_tag} groups {group {name} } performers {name} galleries {id} files {path width height duration size video_codec bit_rate frame_rate} details ' htmlFileData = " paths {screenshot sprite " + htmlPreviewOrStream + "} " +mergeFieldData = " code director title rating100 date studio {id name} urls " +fragmentForSceneDetails += mergeFieldData + htmlFileData DuplicateCandidateForDeletionList = f"{htmlReportNameFolder}{os.sep}DuplicateCandidateForDeletionList.txt" def mangeDupFiles(merge=False, deleteDup=False, tagDuplicates=False, deleteBlacklistOnly=False, deleteLowerResAndDuration=False): @@ -835,10 +837,9 @@ def mangeDupFiles(merge=False, deleteDup=False, tagDuplicates=False, deleteBlack stash.Trace("#########################################################################") stash.Log(f"Waiting for find_duplicate_scenes_diff to return results; matchDupDistance={matchPhaseDistanceText}; significantTimeDiff={significantTimeDiff}", printTo=LOG_STASH_N_PLUGIN) stash.startSpinningProcessBar() - mergeFieldData = " code director title rating100 date studio {id name} movies {movie {id} } urls " if merge else "" if not createHtmlReport: htmlFileData = "" - DupFileSets = stash.find_duplicate_scenes(matchPhaseDistance, fragment= fragmentForSceneDetails + mergeFieldData + htmlFileData) + DupFileSets = stash.find_duplicate_scenes(matchPhaseDistance, fragment=fragmentForSceneDetails) stash.stopSpinningProcessBar() qtyResults = len(DupFileSets) stash.setProgressBarIter(qtyResults) @@ -1102,14 +1103,26 @@ def toJson(data): data = data.replace("\\\\\\\\", "\\\\") return json.loads(data) -def getAnAdvanceMenuOptionSelected(taskName, target, isTagOnlyScenes, isBlackList, isGrayList, isPinkList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater): +def getAnAdvanceMenuOptionSelected(taskName, target, isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater): stash.Log(f"Processing taskName = {taskName}, target = {target}") if "Blacklist" in taskName: - isBlackList = True + tagOrFlag = "Blacklist" if "Graylist" in taskName: - isGrayList = True + tagOrFlag = "Graylist" if "Pinklist" in taskName: - isPinkList = True + tagOrFlag = "Pinklist" + if "YellowFlag" in taskName: + tagOrFlag = "YellowFlag" + if "GreenFlag" in taskName: + tagOrFlag = "GreenFlag" + if "OrangeFlag" in taskName: + tagOrFlag = "OrangeFlag" + if "CyanFlag" in taskName: + tagOrFlag = "CyanFlag" + if "PinkFlag" in taskName: + tagOrFlag = "PinkFlag" + if "RedFlag" in taskName: + tagOrFlag = "RedFlag" if "Less" in taskName: compareToLess = True if "Greater" in taskName: @@ -1145,13 +1158,11 @@ def getAnAdvanceMenuOptionSelected(taskName, target, isTagOnlyScenes, isBlackLis isTagOnlyScenes = False elif "TagOnlyScenes" in taskName: isTagOnlyScenes = True - return isTagOnlyScenes, isBlackList, isGrayList, isPinkList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater + return isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater def getAdvanceMenuOptionSelected(advanceMenuOptionSelected): isTagOnlyScenes = False - isBlackList = False - isGrayList = False - isPinkList = False + tagOrFlag = None pathToDelete = "" sizeToDelete = -1 durationToDelete = -1 @@ -1169,10 +1180,10 @@ def getAdvanceMenuOptionSelected(advanceMenuOptionSelected): if "applyCombo" in stash.PLUGIN_TASK_NAME: jsonObject = toJson(stash.JSON_INPUT['args']['Target']) for taskName in jsonObject: - isTagOnlyScenes, isBlackList, isGrayList, isPinkList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater = getAnAdvanceMenuOptionSelected(taskName, jsonObject[taskName], isTagOnlyScenes, isBlackList, isGrayList, isPinkList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater) + isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater = getAnAdvanceMenuOptionSelected(taskName, jsonObject[taskName], isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater) else: - return getAnAdvanceMenuOptionSelected(stash.PLUGIN_TASK_NAME, stash.JSON_INPUT['args']['Target'], isTagOnlyScenes, isBlackList, isGrayList, isPinkList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, compareToLess, compareToGreater) - return isTagOnlyScenes, isBlackList, isGrayList, isPinkList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater + return getAnAdvanceMenuOptionSelected(stash.PLUGIN_TASK_NAME, stash.JSON_INPUT['args']['Target'], isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, compareToLess, compareToGreater) + return isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater def getScenesFromReport(): stash.Log(f"Getting candidates for deletion from file {DuplicateCandidateForDeletionList}.") @@ -1206,11 +1217,26 @@ def getFlaggedScenesFromReport(fileName, flagType): stash.Trace(f"Did not find flag {flagType}") return None -def getFlaggedScenes(ReportName = htmlReportName): +def getFlaggedScenes(flagType=None, ReportName = htmlReportName): flaggedScenes = [] - flagType = stash.JSON_INPUT['args']['Target'] - if flagType == "green": - flagType = "#00FF00" + if flagType == None: + flagType = stash.JSON_INPUT['args']['Target'] + if flagType == "green": + flagType = "#00FF00" + else: + if flagType == "YellowFlag": + flagType = "yellow" + if flagType == "GreenFlag": + flagType = "#00FF00" + if flagType == "OrangeFlag": + flagType = "orange" + if flagType == "CyanFlag": + flagType = "cyan" + if flagType == "PinkFlag": + flagType = "pink" + if flagType == "RedFlag": + flagType = "red" + stash.Debug(f"Searching for scenes with flag type {flagType}") if os.path.isfile(ReportName): @@ -1246,20 +1272,24 @@ def manageDuplicatesTaggedOrInReport(deleteScenes=False, clearTag=False, setGray if clearAllDupfileManagerTags: excludedTags = [duplicateMarkForDeletion, duplicateWhitelistTag, excludeDupFileDeleteTag, graylistMarkForDeletion, longerDurationLowerResolution] - isTagOnlyScenes, isBlackList, isGrayList, isPinkList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater = getAdvanceMenuOptionSelected(advanceMenuOptionSelected) + isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater = getAdvanceMenuOptionSelected(advanceMenuOptionSelected) if advanceMenuOptionSelected and deleteScenes and pathToDelete == "" and tagToDelete == "" and titleToDelete == "" and pathStrToDelete == "" and sizeToDelete == -1 and durationToDelete == -1 and resolutionToDelete == -1 and ratingToDelete == -1 and fileNotExistToDelete == False: stash.Error("Running advance menu option with no options enabled.") return flaggedScenes = None flagType = None - if checkFlagOption: - flaggedScenes, flagType = getFlaggedScenes() + if checkFlagOption or "Flag" in tagOrFlag: + if checkFlagOption: + flaggedScenes, flagType = getFlaggedScenes() + else: + checkFlagOption = True + flaggedScenes, flagType = getFlaggedScenes(tagOrFlag) if flaggedScenes == None or len(flaggedScenes) == 0: stash.Error(f"Early exit, because found no scenes with flag {flagType}.") return stash.Debug(f"Fournd {len(flaggedScenes)} scenes with flag {flagType}") - + QtyDup = 0 QtyDeleted = 0 QtyClearedTags = 0 @@ -1267,14 +1297,16 @@ def manageDuplicatesTaggedOrInReport(deleteScenes=False, clearTag=False, setGray QtyFailedQuery = 0 stash.Debug("#########################################################################") stash.startSpinningProcessBar() - if isTagOnlyScenes or (advanceMenuOptionSelected == False and checkFlagOption == False): + if advanceMenuOptionSelected == False and checkFlagOption == False: + isTagOnlyScenes = True + if isTagOnlyScenes: stash.Log(f"Getting candidates for deletion by using tag-ID {tagId} and tag-name {tagName}; isTagOnlyScenes={isTagOnlyScenes};advanceMenuOptionSelected={advanceMenuOptionSelected}") - scenes = stash.find_scenes(f={"tags": {"value":tagId, "modifier":"INCLUDES"}}, fragment='id tags {id name} files {path width height duration size video_codec bit_rate frame_rate} details title rating100') + scenes = stash.find_scenes(f={"tags": {"value":tagId, "modifier":"INCLUDES"}}, fragment=fragmentForSceneDetails) # Old setting 'id tags {id name} files {path width height duration size video_codec bit_rate frame_rate} details title rating100') else: scenes = getScenesFromReport() stash.stopSpinningProcessBar() qtyResults = len(scenes) - if isTagOnlyScenes or (advanceMenuOptionSelected == False and checkFlagOption == False): + if isTagOnlyScenes: stash.Log(f"Found {qtyResults} scenes with tag ({duplicateMarkForDeletion})") else: stash.Log(f"Found {qtyResults} scenes in report") @@ -1321,19 +1353,24 @@ def manageDuplicatesTaggedOrInReport(deleteScenes=False, clearTag=False, setGray elif deleteScenes: DupFileName = scene['files'][0]['path'] DupFileNameOnly = pathlib.Path(DupFileName).stem - if checkFlagOption: + if checkFlagOption and "Flag" not in tagOrFlag: if int(scene['id']) in flaggedScenes: stash.Log(f"Found {flagType} flagged candidate for deletion; Scene ID = {scene['id']}") else: continue elif advanceMenuOptionSelected: - if isBlackList: + if checkFlagOption: + if int(scene['id']) in flaggedScenes: + stash.Trace(f"Found {flagType} flag for Scene ID = {scene['id']}") + else: + continue + if tagOrFlag == "Blacklist": if not stash.startsWithInList(blacklist, scene['files'][0]['path']): continue - if isGrayList: + if tagOrFlag == "Graylist": if not stash.startsWithInList(graylist, scene['files'][0]['path']): continue - if isPinkList: + if tagOrFlag == "Pinklist": if not stash.startsWithInList(pinklist, scene['files'][0]['path']): continue if pathToDelete != "": @@ -1463,11 +1500,13 @@ def removeExcludeTag(): stash.Log(f"Done removing exclude tag from scene {scene}.") sys.stdout.write("{" + f"removeExcludeTag : 'complete', id: '{scene}'" + "}") -def getParseData(getSceneDetails1=True, getSceneDetails2=True): +def getParseData(getSceneDetails1=True, getSceneDetails2=True, checkIfNotSplitValue=False): if 'Target' not in stash.JSON_INPUT['args']: stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})") return None, None targetsSrc = stash.JSON_INPUT['args']['Target'] + if checkIfNotSplitValue and ":" not in targetsSrc: + return targetsSrc, None targets = targetsSrc.split(":") if len(targets) < 2: stash.Error(f"Could not get both targets from string {targetsSrc}") @@ -1482,12 +1521,45 @@ def getParseData(getSceneDetails1=True, getSceneDetails2=True): elif len(targets) > 2: target2 = target2 + targets[2] return target1, target2 - + +def mergeMetadataInThisFile(fileName): + stash.Debug(f"Checking report file '{fileName}' for yellow icons indicating missing metadata in DuplicateToKeep.") + lines = None + with open(fileName, 'r') as file: + lines = file.readlines() + for line in lines: + if "https://www.axter.com/images/stash/Yellow" in line: # FYI: This catches YellowGroup.png as well, even though group is not currently supported for merging + searchStrScene1 = "