# Description: This is a Stash plugin which manages duplicate files. # By David Maisonave (aka Axter) Jul-2024 (https://www.axter.com/) # Get the latest developers version from following link: https://github.com/David-Maisonave/Axter-Stash/tree/main/plugins/DupFileManager # Note: To call this script outside of Stash, pass argument --url # Example: python DupFileManager.py --url http://localhost:9999 -a try: import ModulesValidate ModulesValidate.modulesInstalled(["send2trash", "requests"], silent=True) except Exception as e: import traceback, sys tb = traceback.format_exc() print(f"ModulesValidate Exception. Error: {e}\nTraceBack={tb}", file=sys.stderr) import os, sys, time, pathlib, argparse, platform, shutil, traceback, logging, requests, json from datetime import datetime from StashPluginHelper import StashPluginHelper from stashapi.stash_types import PhashDistance from DupFileManager_config import config # Import config from DupFileManager_config.py from DupFileManager_report_config import report_config # ToDo: make sure the following line of code works config |= report_config parser = argparse.ArgumentParser() parser.add_argument('--url', '-u', dest='stash_url', type=str, help='Add Stash URL') parser.add_argument('--trace', '-t', dest='trace', action='store_true', help='Enables debug trace mode.') parser.add_argument('--add_dup_tag', '-a', dest='dup_tag', action='store_true', help='Set a tag to duplicate files.') parser.add_argument('--clear_dup_tag', '-c', dest='clear_tag', action='store_true', help='Clear duplicates of duplicate tags.') parser.add_argument('--del_tag_dup', '-d', dest='del_tag', action='store_true', help='Only delete scenes having DuplicateMarkForDeletion tag.') parser.add_argument('--remove_dup', '-r', dest='remove', action='store_true', help='Remove (delete) duplicate files.') parse_args = parser.parse_args() settings = { "matchDupDistance": 0, "mergeDupFilename": False, "whitelistDelDupInSameFolder": False, "zvWhitelist": "", "zwGraylist": "", "zxBlacklist": "", "zxPinklist": "", "zyMaxDupToProcess": 0, "zySwapHighRes": False, "zySwapLongLength": False, "zySwapBetterBitRate": False, "zySwapCodec": False, "zySwapBetterFrameRate": False, "zzDebug": False, "zzTracing": False, "zzdryRun": False, "zzObsoleteSettingsCheckVer2": False, # This is a hidden variable that is NOT displayed in the UI # Obsolete setting names "zWhitelist": "", "zxGraylist": "", "zyBlacklist": "", "zyMatchDupDistance": 0, "zSwapHighRes": False, "zSwapLongLength": False, "zSwapBetterBitRate": False, "zSwapCodec": False, "zSwapBetterFrameRate": False, } stash = StashPluginHelper( stash_url=parse_args.stash_url, debugTracing=parse_args.trace, settings=settings, config=config, maxbytes=10*1024*1024, DebugTraceFieldName="zzTracing", DebugFieldName="zzDebug", ) stash.convertToAscii = True dry_run = stash.Setting("zzdryRun") advanceMenuOptions = [ "applyCombo", "applyComboPinklist", "applyComboGraylist", "applyComboBlacklist", "pathToDelete", "pathToDeleteBlacklist", "sizeToDeleteLess", "sizeToDeleteGreater", "sizeToDeleteBlacklistLess", "sizeToDeleteBlacklistGreater", "durationToDeleteLess", "durationToDeleteGreater", "durationToDeleteBlacklistLess", "durationToDeleteBlacklistGreater", "commonResToDeleteLess", "commonResToDeleteEq", "commonResToDeleteGreater", "commonResToDeleteBlacklistLess", "commonResToDeleteBlacklistEq", "commonResToDeleteBlacklistGreater", "resolutionToDeleteLess", "resolutionToDeleteEq", "resolutionToDeleteGreater", "resolutionToDeleteBlacklistLess", "resolutionToDeleteBlacklistEq", "resolutionToDeleteBlacklistGreater", "ratingToDeleteLess", "ratingToDeleteEq", "ratingToDeleteGreater", "ratingToDeleteBlacklistLess", "ratingToDeleteBlacklistEq", "ratingToDeleteBlacklistGreater", "tagToDelete", "tagToDeleteBlacklist", "titleToDelete", "titleToDeleteBlacklist", "pathStrToDelete", "pathStrToDeleteBlacklist"] doJsonReturnModeTypes = ["tag_duplicates_task", "removeDupTag", "addExcludeTag", "removeExcludeTag", "mergeTags", "getLocalDupReportPath", "createDuplicateReportWithoutTagging", "deleteLocalDupReportHtmlFiles", "clear_duplicate_tags_task", "deleteAllDupFileManagerTags", "deleteBlackListTaggedDuplicatesTask", "deleteTaggedDuplicatesLwrResOrLwrDuration", "deleteBlackListTaggedDuplicatesLwrResOrLwrDuration", "create_duplicate_report_task"] doJsonReturnModeTypes += [advanceMenuOptions] doJsonReturn = False if len(sys.argv) < 2 and stash.PLUGIN_TASK_NAME in doJsonReturnModeTypes: doJsonReturn = True stash.log_to_norm = stash.LogTo.FILE elif stash.PLUGIN_TASK_NAME == "doEarlyExit": time.sleep(3) stash.Log("Doing early exit because of task name") time.sleep(3) exit(0) stash.Log("******************* Starting *******************") if len(sys.argv) > 1: stash.Log(f"argv = {sys.argv}") else: stash.Debug(f"No command line arguments. JSON_INPUT['args'] = {stash.JSON_INPUT['args']}; PLUGIN_TASK_NAME = {stash.PLUGIN_TASK_NAME}; argv = {sys.argv}") stash.status(logLevel=logging.DEBUG) obsoleteSettingsToConvert = {"zWhitelist" : "zvWhitelist", "zxGraylist" : "zwGraylist", "zyBlacklist" : "zxBlacklist", "zyMatchDupDistance" : "matchDupDistance", "zSwapHighRes" : "zySwapHighRes", "zSwapLongLength" : "zySwapLongLength", "zSwapBetterBitRate" : "zySwapBetterBitRate", "zSwapCodec" : "zySwapCodec", "zSwapBetterFrameRate" : "zySwapBetterFrameRate"} stash.replaceObsoleteSettings(obsoleteSettingsToConvert, "zzObsoleteSettingsCheckVer2") LOG_STASH_N_PLUGIN = stash.LogTo.STASH if stash.CALLED_AS_STASH_PLUGIN else stash.LogTo.CONSOLE + stash.LogTo.FILE listSeparator = stash.Setting('listSeparator', ',', notEmpty=True) addPrimaryDupPathToDetails = stash.Setting('addPrimaryDupPathToDetails') clearAllDupfileManagerTags = stash.Setting('clearAllDupfileManagerTags') doGeneratePhash = stash.Setting('doGeneratePhash') mergeDupFilename = stash.Setting('mergeDupFilename') moveToTrashCan = False if stash.Setting('permanentlyDelete') else True alternateTrashCanPath = stash.Setting('dup_path') whitelistDelDupInSameFolder = stash.Setting('whitelistDelDupInSameFolder') graylistTagging = stash.Setting('graylistTagging') maxDupToProcess = int(stash.Setting('zyMaxDupToProcess')) significantTimeDiff = float(stash.Setting('significantTimeDiff')) toRecycleBeforeSwap = stash.Setting('toRecycleBeforeSwap') cleanAfterDel = stash.Setting('cleanAfterDel') swapHighRes = stash.Setting('zySwapHighRes') swapLongLength = stash.Setting('zySwapLongLength') swapBetterBitRate = stash.Setting('zySwapBetterBitRate') swapCodec = stash.Setting('zySwapCodec') swapBetterFrameRate = stash.Setting('zySwapBetterFrameRate') favorLongerFileName = stash.Setting('favorLongerFileName') favorLargerFileSize = stash.Setting('favorLargerFileSize') favorBitRateChange = stash.Setting('favorBitRateChange') favorHighBitRate = stash.Setting('favorHighBitRate') favorFrameRateChange = stash.Setting('favorFrameRateChange') favorHigherFrameRate = stash.Setting('favorHigherFrameRate') favorCodecRanking = stash.Setting('favorCodecRanking') codecRankingSetToUse = stash.Setting('codecRankingSetToUse') if codecRankingSetToUse == 4: codecRanking = stash.Setting('codecRankingSet4') elif codecRankingSetToUse == 3: codecRanking = stash.Setting('codecRankingSet3') elif codecRankingSetToUse == 2: codecRanking = stash.Setting('codecRankingSet2') else: codecRanking = stash.Setting('codecRankingSet1') skipIfTagged = stash.Setting('skipIfTagged') killScanningPostProcess = stash.Setting('killScanningPostProcess') tagLongDurationLowRes = stash.Setting('tagLongDurationLowRes') bitRateIsImporantComp = stash.Setting('bitRateIsImporantComp') codecIsImporantComp = stash.Setting('codecIsImporantComp') excludeFromReportIfSignificantTimeDiff = False matchDupDistance = int(stash.Setting('matchDupDistance')) matchPhaseDistance = PhashDistance.EXACT matchPhaseDistanceText = "Exact Match" if (stash.PLUGIN_TASK_NAME == "tag_duplicates_task" or stash.PLUGIN_TASK_NAME == "create_duplicate_report_task") and 'Target' in stash.JSON_INPUT['args']: stash.enableProgressBar(False) if stash.JSON_INPUT['args']['Target'].startswith("0"): matchDupDistance = 0 elif stash.JSON_INPUT['args']['Target'].startswith("1"): matchDupDistance = 1 elif stash.JSON_INPUT['args']['Target'].startswith("2"): matchDupDistance = 2 elif stash.JSON_INPUT['args']['Target'].startswith("3"): matchDupDistance = 3 if stash.JSON_INPUT['args']['Target'].find(":") == 1: significantTimeDiff = float(stash.JSON_INPUT['args']['Target'][2:]) excludeFromReportIfSignificantTimeDiff = True if matchDupDistance == 1: matchPhaseDistance = PhashDistance.HIGH matchPhaseDistanceText = "High Match" elif matchDupDistance == 2: matchPhaseDistance = PhashDistance.MEDIUM matchPhaseDistanceText = "Medium Match" elif matchDupDistance == 3: matchPhaseDistance = PhashDistance.LOW matchPhaseDistanceText = "Low Match" # significantTimeDiff can not be higher than 1 and shouldn't be lower than .5 if significantTimeDiff > 1: significantTimeDiff = float(1.00) if significantTimeDiff < .25: significantTimeDiff = float(0.25) duplicateMarkForDeletion = stash.Setting('DupFileTag') if duplicateMarkForDeletion == "": duplicateMarkForDeletion = 'DuplicateMarkForDeletion' base1_duplicateMarkForDeletion = duplicateMarkForDeletion duplicateWhitelistTag = stash.Setting('DupWhiteListTag') if duplicateWhitelistTag == "": duplicateWhitelistTag = '_DuplicateWhitelistFile' excludeDupFileDeleteTag = stash.Setting('excludeDupFileDeleteTag') if excludeDupFileDeleteTag == "": excludeDupFileDeleteTag = '_ExcludeDuplicateMarkForDeletion' graylistMarkForDeletion = stash.Setting('graylistMarkForDeletion') if graylistMarkForDeletion == "": graylistMarkForDeletion = '_GraylistMarkForDeletion' longerDurationLowerResolution = stash.Setting('longerDurationLowerResolution') if longerDurationLowerResolution == "": longerDurationLowerResolution = '_LongerDurationLowerResolution' excludeMergeTags = [duplicateMarkForDeletion, duplicateWhitelistTag, excludeDupFileDeleteTag] if stash.Setting('underscoreDupFileTag') and not duplicateMarkForDeletion.startswith('_'): duplicateMarkForDeletionWithOutUnderscore = duplicateMarkForDeletion duplicateMarkForDeletion = "_" + duplicateMarkForDeletion if stash.renameTag(duplicateMarkForDeletionWithOutUnderscore, duplicateMarkForDeletion): stash.Log(f"Renamed tag {duplicateMarkForDeletionWithOutUnderscore} to {duplicateMarkForDeletion}") stash.Trace(f"Added underscore to {duplicateMarkForDeletionWithOutUnderscore} = {duplicateMarkForDeletion}") excludeMergeTags += [duplicateMarkForDeletion] else: stash.Trace(f"duplicateMarkForDeletion = {duplicateMarkForDeletion}") base2_duplicateMarkForDeletion = duplicateMarkForDeletion if stash.Setting('appendMatchDupDistance'): duplicateMarkForDeletion += f"_{matchDupDistance}" excludeMergeTags += [duplicateMarkForDeletion] stash.initMergeMetadata(excludeMergeTags) graylist = stash.Setting('zwGraylist').split(listSeparator) graylist = [item.lower() for item in graylist] if graylist == [""] : graylist = [] stash.Trace(f"graylist = {graylist}") whitelist = stash.Setting('zvWhitelist').split(listSeparator) whitelist = [item.lower() for item in whitelist] if whitelist == [""] : whitelist = [] stash.Trace(f"whitelist = {whitelist}") blacklist = stash.Setting('zxBlacklist').split(listSeparator) blacklist = [item.lower() for item in blacklist] if blacklist == [""] : blacklist = [] pinklist = stash.Setting('zxPinklist').split(listSeparator) pinklist = [item.lower() for item in pinklist] if pinklist == [""] : pinklist = [] stash.Trace(f"pinklist = {pinklist}") def realpath(path): """ get_symbolic_target for win """ try: import win32file f = win32file.CreateFile(path, win32file.GENERIC_READ, win32file.FILE_SHARE_READ, None, win32file.OPEN_EXISTING, win32file.FILE_FLAG_BACKUP_SEMANTICS, None) target = win32file.GetFinalPathNameByHandle(f, 0) # an above gives us something like u'\\\\?\\C:\\tmp\\scalarizr\\3.3.0.7978' return target.strip('\\\\?\\') except ImportError: handle = open_dir(path) target = get_symbolic_target(handle) check_closed(handle) return target def isReparsePoint(path): import win32api import win32con from parse_reparsepoint import Navigator FinalPathname = realpath(path) stash.Log(f"(path='{path}') (FinalPathname='{FinalPathname}')") if FinalPathname != path: stash.Log(f"Symbolic link '{path}'") return True if not os.path.isdir(path): path = os.path.dirname(path) return win32api.GetFileAttributes(path) & win32con.FILE_ATTRIBUTE_REPARSE_POINT def testReparsePointAndSymLink(merge=False, deleteDup=False): stash.Trace(f"Debug Tracing (platform.system()={platform.system()})") myTestPath1 = r"B:\V\V\Tip\POV - Holly Molly petite ginger anal slut - RedTube.mp4" # not a reparse point or symbolic link myTestPath2 = r"B:\_\SpecialSet\Amateur Anal Attempts\BRCC test studio name.m2ts" # reparse point myTestPath3 = r"B:\_\SpecialSet\Amateur Anal Attempts\Amateur Anal Attempts 4.mp4" #symbolic link myTestPath4 = r"E:\Stash\plugins\RenameFile\README.md" #symbolic link myTestPath5 = r"E:\_\David-Maisonave\Axter-Stash\plugins\RenameFile\README.md" #symbolic link myTestPath6 = r"E:\_\David-Maisonave\Axter-Stash\plugins\DeleteMe\Renamer\README.md" # not reparse point stash.Log(f"Testing '{myTestPath1}'") if isReparsePoint(myTestPath1): stash.Log(f"isSymLink '{myTestPath1}'") else: stash.Log(f"Not isSymLink '{myTestPath1}'") if isReparsePoint(myTestPath2): stash.Log(f"isSymLink '{myTestPath2}'") else: stash.Log(f"Not isSymLink '{myTestPath2}'") if isReparsePoint(myTestPath3): stash.Log(f"isSymLink '{myTestPath3}'") else: stash.Log(f"Not isSymLink '{myTestPath3}'") if isReparsePoint(myTestPath4): stash.Log(f"isSymLink '{myTestPath4}'") else: stash.Log(f"Not isSymLink '{myTestPath4}'") if isReparsePoint(myTestPath5): stash.Log(f"isSymLink '{myTestPath5}'") else: stash.Log(f"Not isSymLink '{myTestPath5}'") if isReparsePoint(myTestPath6): stash.Log(f"isSymLink '{myTestPath6}'") else: stash.Log(f"Not isSymLink '{myTestPath6}'") return detailPrefix = "BaseDup=" detailPostfix = "\n" def setTagId(tagName, sceneDetails, DupFileToKeep, TagReason="", ignoreAutoTag=False): details = "" ORG_DATA_DICT = {'id' : sceneDetails['id']} dataDict = ORG_DATA_DICT.copy() doAddTag = True if addPrimaryDupPathToDetails: BaseDupStr = f"{detailPrefix}{DupFileToKeep['files'][0]['path']}\n{stash.STASH_URL}/scenes/{DupFileToKeep['id']}\n{TagReason}(matchDupDistance={matchPhaseDistanceText})\n{detailPostfix}" if sceneDetails['details'] == "": details = BaseDupStr elif not sceneDetails['details'].startswith(detailPrefix): details = f"{BaseDupStr};\n{sceneDetails['details']}" for tag in sceneDetails['tags']: if tag['name'] == tagName: doAddTag = False break if doAddTag: stash.addTag(sceneDetails, tagName, ignoreAutoTag=ignoreAutoTag) if details != "": dataDict.update({'details' : details}) if dataDict != ORG_DATA_DICT: stash.updateScene(dataDict) stash.Trace(f"[setTagId] Updated {sceneDetails['files'][0]['path']} with metadata {dataDict} and tag {tagName}", toAscii=True) else: stash.Trace(f"[setTagId] Nothing to update {sceneDetails['files'][0]['path']} already has tag {tagName}.", toAscii=True) return doAddTag def setTagId_withRetry(tagName, sceneDetails, DupFileToKeep, TagReason="", ignoreAutoTag=False, retryCount = 12, sleepSecondsBetweenRetry = 5): errMsg = None for i in range(0, retryCount): try: if errMsg != None: stash.Warn(errMsg) return setTagId(tagName, sceneDetails, DupFileToKeep, TagReason, ignoreAutoTag) except (requests.exceptions.ConnectionError, ConnectionResetError): tb = traceback.format_exc() errMsg = f"[setTagId] Exception calling setTagId. Will retry; count({i}); Error: {e}\nTraceBack={tb}" except Exception as e: tb = traceback.format_exc() errMsg = f"[setTagId] Unknown exception calling setTagId. Will retry; count({i}); Error: {e}\nTraceBack={tb}" time.sleep(sleepSecondsBetweenRetry) def hasSameDir(path1, path2): if pathlib.Path(path1).resolve().parent == pathlib.Path(path2).resolve().parent: return True return False def sendToTrash(path): if not os.path.isfile(path): stash.Warn(f"File does not exist: {path}.", toAscii=True) return False try: from send2trash import send2trash # Requirement: pip install Send2Trash send2trash(path) return True except Exception as e: stash.Error(f"Failed to send file {path} to recycle bin. Error: {e}", toAscii=True) try: if os.path.isfile(path): os.remove(path) return True except Exception as e: stash.Error(f"Failed to delete file {path}. Error: {e}", toAscii=True) return False # If ckTimeDiff=False: Does durration2 have significant more time than durration1 def significantTimeDiffCheck(durration1, durration2, ckTimeDiff = False): # If ckTimeDiff=True: is time different significant in either direction. if not isinstance(durration1, int) and 'files' in durration1: durration1 = int(durration1['files'][0]['duration']) durration2 = int(durration2['files'][0]['duration']) timeDiff = getTimeDif(durration1, durration2) if ckTimeDiff and timeDiff > 1: timeDiff = getTimeDif(durration2, durration1) if timeDiff < significantTimeDiff: return True return False def getTimeDif(durration1, durration2): # Where durration1 is ecpected to be smaller than durration2 IE(45/60=.75) return durration1 / durration2 def isBetterVideo(scene1, scene2, swapCandidateCk = False): # is scene2 better than scene1 # Prioritize higher reslution over codec, bit rate, and frame rate if int(scene1['files'][0]['width']) * int(scene1['files'][0]['height']) > int(scene2['files'][0]['width']) * int(scene2['files'][0]['height']): return False if (favorBitRateChange and swapCandidateCk == False) or (swapCandidateCk and swapBetterBitRate): if (favorHighBitRate and int(scene2['files'][0]['bit_rate']) > int(scene1['files'][0]['bit_rate'])) or (not favorHighBitRate and int(scene2['files'][0]['bit_rate']) < int(scene1['files'][0]['bit_rate'])): stash.Trace(f"[isBetterVideo]:[favorHighBitRate={favorHighBitRate}] Better bit rate. {scene1['files'][0]['path']}={scene1['files'][0]['bit_rate']} v.s. {scene2['files'][0]['path']}={scene2['files'][0]['bit_rate']}") return True if (favorCodecRanking and swapCandidateCk == False) or (swapCandidateCk and swapCodec): scene1CodecRank = stash.indexStartsWithInList(codecRanking, scene1['files'][0]['video_codec']) scene2CodecRank = stash.indexStartsWithInList(codecRanking, scene2['files'][0]['video_codec']) if scene2CodecRank < scene1CodecRank: stash.Trace(f"[isBetterVideo] Better codec. {scene1['files'][0]['path']}={scene1['files'][0]['video_codec']}:Rank={scene1CodecRank} v.s. {scene2['files'][0]['path']}={scene2['files'][0]['video_codec']}:Rank={scene2CodecRank}") return True if (favorFrameRateChange and swapCandidateCk == False) or (swapCandidateCk and swapBetterFrameRate): if (favorHigherFrameRate and int(scene2['files'][0]['frame_rate']) > int(scene1['files'][0]['frame_rate'])) or (not favorHigherFrameRate and int(scene2['files'][0]['frame_rate']) < int(scene1['files'][0]['frame_rate'])): stash.Trace(f"[isBetterVideo]:[favorHigherFrameRate={favorHigherFrameRate}] Better frame rate. {scene1['files'][0]['path']}={scene1['files'][0]['frame_rate']} v.s. {scene2['files'][0]['path']}={scene2['files'][0]['frame_rate']}") return True return False def significantMoreTimeCompareToBetterVideo(scene1, scene2): # is scene2 better than scene1 if isinstance(scene1, int): scene1 = stash.find_scene(scene1) scene2 = stash.find_scene(scene2) if int(scene1['files'][0]['duration']) >= int(scene2['files'][0]['duration']): return False if int(scene1['files'][0]['width']) * int(scene1['files'][0]['height']) > int(scene2['files'][0]['width']) * int(scene2['files'][0]['height']): if significantTimeDiffCheck(scene1, scene2): if tagLongDurationLowRes: didAddTag = setTagId_withRetry(longerDurationLowerResolution, scene2, scene1, ignoreAutoTag=True) stash.Log(f"Tagged sene2 with tag {longerDurationLowerResolution}, because scene1 is better video, but it has significant less time ({getTimeDif(int(scene1['files'][0]['duration']), int(scene2['files'][0]['duration']))}%) compare to scene2; scene1={scene1['files'][0]['path']} (ID={scene1['id']})(duration={scene1['files'][0]['duration']}); scene2={scene2['files'][0]['path']} (ID={scene2['id']}) (duration={scene1['files'][0]['duration']}); didAddTag={didAddTag}") else: stash.Warn(f"Scene1 is better video, but it has significant less time ({getTimeDif(int(scene1['files'][0]['duration']), int(scene2['files'][0]['duration']))}%) compare to scene2; Scene1={scene1['files'][0]['path']} (ID={scene1['id']})(duration={scene1['files'][0]['duration']}); Scene2={scene2['files'][0]['path']} (ID={scene2['id']}) (duration={scene1['files'][0]['duration']})") return False return True def allThingsEqual(scene1, scene2): # If all important things are equal, return true if int(scene1['files'][0]['duration']) != int(scene2['files'][0]['duration']): return False if scene1['files'][0]['width'] != scene2['files'][0]['width']: return False if scene1['files'][0]['height'] != scene2['files'][0]['height']: return False if bitRateIsImporantComp and scene1['files'][0]['bit_rate'] != scene2['files'][0]['bit_rate']: return False if codecIsImporantComp and scene1['files'][0]['video_codec'] != scene2['files'][0]['video_codec']: return False return True def isSwapCandidate(DupFileToKeep, DupFile): # Don't move if both are in whitelist if stash.startsWithInList(whitelist, DupFileToKeep['files'][0]['path']) and stash.startsWithInList(whitelist, DupFile['files'][0]['path']): return False if swapHighRes and int(DupFileToKeep['files'][0]['width']) * int(DupFileToKeep['files'][0]['height']) > int(DupFile['files'][0]['width']) * int(DupFile['files'][0]['height']): if not significantTimeDiffCheck(DupFileToKeep, DupFile): return True else: stash.Warn(f"File '{DupFileToKeep['files'][0]['path']}' has a higher resolution than '{DupFile['files'][0]['path']}', but the duration is significantly shorter.", toAscii=True) if swapLongLength and int(DupFileToKeep['files'][0]['duration']) > int(DupFile['files'][0]['duration']): if int(DupFileToKeep['files'][0]['width']) >= int(DupFile['files'][0]['width']) or int(DupFileToKeep['files'][0]['height']) >= int(DupFile['files'][0]['height']): return True if isBetterVideo(DupFile, DupFileToKeep, swapCandidateCk=True): if not significantTimeDiffCheck(DupFileToKeep, DupFile): return True else: stash.Warn(f"File '{DupFileToKeep['files'][0]['path']}' has better codec/bit-rate than '{DupFile['files'][0]['path']}', but the duration is significantly shorter; DupFileToKeep-ID={DupFileToKeep['id']};DupFile-ID={DupFile['id']};BitRate {DupFileToKeep['files'][0]['bit_rate']} vs {DupFile['files'][0]['bit_rate']};Codec {DupFileToKeep['files'][0]['video_codec']} vs {DupFile['files'][0]['video_codec']};FrameRate {DupFileToKeep['files'][0]['frame_rate']} vs {DupFile['files'][0]['frame_rate']};", toAscii=True) return False dupWhitelistTagId = None def addDupWhitelistTag(): global dupWhitelistTagId stash.Trace(f"Adding tag duplicateWhitelistTag = {duplicateWhitelistTag}") descp = 'Tag added to duplicate scenes which are in the whitelist. This means there are two or more duplicates in the whitelist.' dupWhitelistTagId = stash.createTagId(duplicateWhitelistTag, descp, ignoreAutoTag=True) stash.Trace(f"dupWhitelistTagId={dupWhitelistTagId} name={duplicateWhitelistTag}") excludeDupFileDeleteTagId = None def addExcludeDupTag(): global excludeDupFileDeleteTagId stash.Trace(f"Adding tag excludeDupFileDeleteTag = {excludeDupFileDeleteTag}") descp = 'Excludes duplicate scene from DupFileManager tagging and deletion process. A scene having this tag will not get deleted by DupFileManager' excludeDupFileDeleteTagId = stash.createTagId(excludeDupFileDeleteTag, descp, ignoreAutoTag=True) stash.Trace(f"dupWhitelistTagId={excludeDupFileDeleteTagId} name={excludeDupFileDeleteTag}") def isTaggedExcluded(Scene): for tag in Scene['tags']: if tag['name'] == excludeDupFileDeleteTag: return True return False def isWorseKeepCandidate(DupFileToKeep, Scene): if not stash.startsWithInList(whitelist, Scene['files'][0]['path']) and stash.startsWithInList(whitelist, DupFileToKeep['files'][0]['path']): return True if not stash.startsWithInList(graylist, Scene['files'][0]['path']) and stash.startsWithInList(graylist, DupFileToKeep['files'][0]['path']): return True if not stash.startsWithInList(blacklist, DupFileToKeep['files'][0]['path']) and stash.startsWithInList(blacklist, Scene['files'][0]['path']): return True if stash.startsWithInList(graylist, Scene['files'][0]['path']) and stash.startsWithInList(graylist, DupFileToKeep['files'][0]['path']) and stash.indexStartsWithInList(graylist, DupFileToKeep['files'][0]['path']) < stash.indexStartsWithInList(graylist, Scene['files'][0]['path']): return True if stash.startsWithInList(blacklist, DupFileToKeep['files'][0]['path']) and stash.startsWithInList(blacklist, Scene['files'][0]['path']) and stash.indexStartsWithInList(blacklist, DupFileToKeep['files'][0]['path']) < stash.indexStartsWithInList(blacklist, Scene['files'][0]['path']): return True return False def killScanningJobs(): try: if killScanningPostProcess: stash.stopJobs(1, "Scanning...") except Exception as e: tb = traceback.format_exc() stash.Error(f"Exception while trying to kill scan jobs; Error: {e}\nTraceBack={tb}") def getPath(Scene, getParent = False): path = stash.asc2(Scene['files'][0]['path']) path = path.replace("'", "") path = path.replace("\\\\", "\\") if getParent: return pathlib.Path(path).resolve().parent return path def getHtmlReportTableRow(qtyResults, tagDuplicates): htmlReportPrefix = stash.Setting('htmlReportPrefix') htmlReportPrefix = htmlReportPrefix.replace('http://127.0.0.1:9999/graphql', stash.url) htmlReportPrefix = htmlReportPrefix.replace('http://localhost:9999/graphql', stash.url) if 'apiKey' in stash.STASH_CONFIGURATION and stash.STASH_CONFIGURATION['apiKey'] != "": htmlReportPrefix = htmlReportPrefix.replace('var apiKey = "";', f"var apiKey = \"{stash.STASH_CONFIGURATION['apiKey']}\";") if tagDuplicates == False: htmlReportPrefix = htmlReportPrefix.replace('
') fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write("
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") # ToDo: Add following buttons: # rename file if dupFileExist and tagDuplicates: fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write("
") if dupFileExist: fileHtmlReport.write('
') fileHtmlReport.write(f"
[Folder]
") fileHtmlReport.write(f"
[Play]
") fileHtmlReport.write("
") else: fileHtmlReport.write(fileDoesNotExistStr) DupToKeepMissingTag, DelCandidateMissingTag = doesDelCandidateHaveMetadataNotInDupToKeep(DupFile, DupFileToKeep, 'tags') if len(DupFile['tags']) > 0: if DupToKeepMissingTag: fileHtmlReport.write(htmlTagPrefix.replace(defaultColorTag, "YellowTag.png")) else: fileHtmlReport.write(htmlTagPrefix) for tag in DupFile['tags']: # if not tag['ignore_auto_tag']: fileHtmlReport.write(f"
{tag['name']}
") fileHtmlReport.write("") DupToKeepMissingPerformer, DelCandidateMissingPerformer = doesDelCandidateHaveMetadataNotInDupToKeep(DupFile, DupFileToKeep, 'performers') if len(DupFile['performers']) > 0: if DupToKeepMissingPerformer: fileHtmlReport.write(htmlPerformerPrefix.replace(defaultColorPerformer, "YellowHeadshot.png")) else: fileHtmlReport.write(htmlPerformerPrefix) for performer in DupFile['performers']: fileHtmlReport.write(f"
{performer['name']}
") fileHtmlReport.write("") DupToKeepMissingGallery, DelCandidateMissingGallery = doesDelCandidateHaveMetadataNotInDupToKeep(DupFile, DupFileToKeep, 'galleries', 'title') if len(DupFile['galleries']) > 0: if DupToKeepMissingGallery: fileHtmlReport.write(htmlGalleryPrefix.replace(defaultColorGalleries, "YellowGalleries.png")) else: fileHtmlReport.write(htmlGalleryPrefix) for gallery in DupFile['galleries']: gallery = stash.getGalleryName(gallery['id']) fileHtmlReport.write(f"
{gallery['title']}
") fileHtmlReport.write("") DupToKeepMissingGroup, DelCandidateMissingGroup = doesDelCandidateHaveMetadataNotInDupToKeep(DupFile, DupFileToKeep, 'groups') if len(DupFile['groups']) > 0: if DupToKeepMissingGroup: fileHtmlReport.write(htmlGroupPrefix.replace(defaultColorGroup, "YellowGroup.png")) else: fileHtmlReport.write(htmlGroupPrefix) for group in DupFile['groups']: fileHtmlReport.write(f"
{group['group']['name']}
") fileHtmlReport.write("") fileHtmlReport.write("

") videoPreview = f"" if htmlIncludeImagePreview: imagePreview = f"" fileHtmlReport.write(f"{getSceneID(DupFileToKeep)}
{videoPreview}{imagePreview}
") else: fileHtmlReport.write(f"{getSceneID(DupFileToKeep)}{videoPreview}") fileHtmlReport.write(f"{getSceneID(DupFileToKeep)}{getPath(DupFileToKeep)}") fileHtmlReport.write(f"

") fileHtmlReport.write(f"
ResDurrationBitRateCodecFrameRatesizeID
{DupFileToKeep['files'][0]['width']}x{DupFileToKeep['files'][0]['height']}{DupFileToKeep['files'][0]['duration']}{DupFileToKeep['files'][0]['bit_rate']}{DupFileToKeep['files'][0]['video_codec']}{DupFileToKeep['files'][0]['frame_rate']}{DupFileToKeep['files'][0]['size']}{DupFileToKeep['id']}
") fileHtmlReport.write('

') fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write("
") fileHtmlReport.write(f"
") if isTaggedExcluded(DupFileToKeep): fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write(f"
") fileHtmlReport.write("
") fileHtmlReport.write('
') fileHtmlReport.write(f"
[Folder]
") if toKeepFileExist: fileHtmlReport.write(f"
[Play]
") fileHtmlReport.write("
") if not toKeepFileExist: fileHtmlReport.write(fileDoesNotExistStr) if len(DupFileToKeep['tags']) > 0: if DelCandidateMissingTag: fileHtmlReport.write(htmlTagPrefix.replace(defaultColorTag, "RedTag.png")) else: fileHtmlReport.write(htmlTagPrefix) for tag in DupFileToKeep['tags']: # if not tag['ignore_auto_tag']: fileHtmlReport.write(f"
{tag['name']}
") fileHtmlReport.write("") if len(DupFileToKeep['performers']) > 0: if DelCandidateMissingPerformer: fileHtmlReport.write(htmlPerformerPrefix.replace(defaultColorPerformer, "PinkHeadshot.png")) else: fileHtmlReport.write(htmlPerformerPrefix) for performer in DupFileToKeep['performers']: fileHtmlReport.write(f"
{performer['name']}
") fileHtmlReport.write("") if len(DupFileToKeep['galleries']) > 0: if DelCandidateMissingGallery: fileHtmlReport.write(htmlGalleryPrefix.replace(defaultColorGalleries, "PinkGalleries.png")) else: fileHtmlReport.write(htmlGalleryPrefix) for gallery in DupFileToKeep['galleries']: gallery = stash.getGalleryName(gallery['id']) fileHtmlReport.write(f"
{gallery['title']}
") fileHtmlReport.write("") if len(DupFileToKeep['groups']) > 0: if DelCandidateMissingGroup: fileHtmlReport.write(htmlGroupPrefix.replace(defaultColorGroup, "PinkGroup.png")) else: fileHtmlReport.write(htmlGroupPrefix) for group in DupFileToKeep['groups']: fileHtmlReport.write(f"
{group['group']['name']}
") fileHtmlReport.write("") # ToDo: Add following buttons: # rename file fileHtmlReport.write(f"

") fileHtmlReport.write(f"\n") fragmentForSceneDetails = 'id tags {id name ignore_auto_tag} groups {group {name} } performers {name} galleries {id} files {path width height duration size video_codec bit_rate frame_rate} details ' htmlFileData = " paths {screenshot sprite " + htmlPreviewOrStream + "} " mergeFieldData = " code director title rating100 date studio {id name} urls " fragmentForSceneDetails += mergeFieldData + htmlFileData DuplicateCandidateForDeletionList = f"{htmlReportNameFolder}{os.sep}DuplicateCandidateForDeletionList.txt" def mangeDupFiles(merge=False, deleteDup=False, tagDuplicates=False, deleteBlacklistOnly=False, deleteLowerResAndDuration=False): global reasonDict global htmlFileData duplicateMarkForDeletion_descp = 'Tag added to duplicate scenes so-as to tag them for deletion.' stash.Trace(f"duplicateMarkForDeletion = {duplicateMarkForDeletion}") dupTagId = stash.createTagId(duplicateMarkForDeletion, duplicateMarkForDeletion_descp, ignoreAutoTag=True) stash.Trace(f"dupTagId={dupTagId} name={duplicateMarkForDeletion}") createHtmlReport = stash.Setting('createHtmlReport') htmlReportNameHomePage = htmlReportName htmlReportPaginate = stash.Setting('htmlReportPaginate') addDupWhitelistTag() addExcludeDupTag() QtyDupSet = 0 QtyDup = 0 QtyExactDup = 0 QtyAlmostDup = 0 QtyRealTimeDiff = 0 QtyTagForDel = 0 QtyTagForDelPaginate = 0 PaginateId = 0 QtyNewlyTag = 0 QtySkipForDel = 0 QtyExcludeForDel = 0 QtySwap = 0 QtyMerge = 0 QtyDeleted = 0 stash.Log("#########################################################################") stash.Trace("#########################################################################") stash.Log(f"Waiting for find_duplicate_scenes_diff to return results; matchDupDistance={matchPhaseDistanceText}; significantTimeDiff={significantTimeDiff}", printTo=LOG_STASH_N_PLUGIN) stash.startSpinningProcessBar() if not createHtmlReport: htmlFileData = "" DupFileSets = stash.find_duplicate_scenes(matchPhaseDistance, fragment=fragmentForSceneDetails) stash.stopSpinningProcessBar() qtyResults = len(DupFileSets) stash.setProgressBarIter(qtyResults) stash.Trace("#########################################################################") stash.Log(f"Found {qtyResults} duplicate sets...") fileHtmlReport = None if not os.path.isdir(htmlReportNameFolder): os.mkdir(htmlReportNameFolder) if not os.path.isdir(htmlReportNameFolder): stash.Error(f"Failed to create report directory {htmlReportNameFolder}.") return if createHtmlReport: deleteLocalDupReportHtmlFiles(False) fileHtmlReport = open(htmlReportName, "w") fileHtmlReport.write(f"{getHtmlReportTableRow(qtyResults, tagDuplicates)}\n") fileHtmlReport.write(f"{stash.Setting('htmlReportTable')}\n") htmlReportTableHeader = stash.Setting('htmlReportTableHeader') fileHtmlReport.write(f"{htmlReportTableRow}{htmlReportTableHeader}Scene{htmlReportTableHeader}Duplicate to Delete{htmlReportTableHeader}Scene-ToKeep{htmlReportTableHeader}Duplicate to Keep\n") fileDuplicateCandidateForDeletionList = open(DuplicateCandidateForDeletionList, "w") for DupFileSet in DupFileSets: # stash.Trace(f"DupFileSet={DupFileSet}", toAscii=True) QtyDupSet+=1 stash.progressBar(QtyDupSet, qtyResults) SepLine = "---------------------------" DupFileToKeep = None DupToCopyFrom = "" DupFileDetailList = [] for DupFile in DupFileSet: QtyDup+=1 Scene = DupFile if skipIfTagged and createHtmlReport == False and duplicateMarkForDeletion in Scene['tags']: stash.Trace(f"Skipping scene '{Scene['files'][0]['path']}' because already tagged with {duplicateMarkForDeletion}") continue stash.TraceOnce(f"Scene = {Scene}", toAscii=True) DupFileDetailList = DupFileDetailList + [Scene] if os.path.isfile(Scene['files'][0]['path']): if DupFileToKeep != None: if int(DupFileToKeep['files'][0]['duration']) == int(Scene['files'][0]['duration']): # Do not count fractions of a second as a difference QtyExactDup+=1 else: QtyAlmostDup+=1 SepLine = "***************************" if significantTimeDiffCheck(DupFileToKeep, Scene): QtyRealTimeDiff += 1 if int(DupFileToKeep['files'][0]['width']) * int(DupFileToKeep['files'][0]['height']) < int(Scene['files'][0]['width']) * int(Scene['files'][0]['height']): logReason(DupFileToKeep, Scene, f"resolution: {DupFileToKeep['files'][0]['width']}x{DupFileToKeep['files'][0]['height']} < {Scene['files'][0]['width']}x{Scene['files'][0]['height']}") DupFileToKeep = Scene elif significantMoreTimeCompareToBetterVideo(DupFileToKeep, Scene): if significantTimeDiffCheck(DupFileToKeep, Scene): theReason = f"significant-duration: {DupFileToKeep['files'][0]['duration']} < {Scene['files'][0]['duration']}" else: theReason = f"duration: {DupFileToKeep['files'][0]['duration']} < {Scene['files'][0]['duration']}" reasonKeyword = "significant-duration" if significantTimeDiffCheck(DupFileToKeep, Scene) else "duration" logReason(DupFileToKeep, Scene, theReason) DupFileToKeep = Scene elif isBetterVideo(DupFileToKeep, Scene): logReason(DupFileToKeep, Scene, f"codec,bit_rate, or frame_rate: {DupFileToKeep['files'][0]['video_codec']}, {DupFileToKeep['files'][0]['bit_rate']}, {DupFileToKeep['files'][0]['frame_rate']} : {Scene['files'][0]['video_codec']}, {Scene['files'][0]['bit_rate']}, {Scene['files'][0]['frame_rate']}") DupFileToKeep = Scene elif stash.startsWithInList(whitelist, Scene['files'][0]['path']) and not stash.startsWithInList(whitelist, DupFileToKeep['files'][0]['path']): logReason(DupFileToKeep, Scene, f"not whitelist vs whitelist") DupFileToKeep = Scene elif isTaggedExcluded(Scene) and not isTaggedExcluded(DupFileToKeep): logReason(DupFileToKeep, Scene, f"not ExcludeTag vs ExcludeTag") DupFileToKeep = Scene elif allThingsEqual(DupFileToKeep, Scene): # Only do below checks if all imporant things are equal. if stash.startsWithInList(blacklist, DupFileToKeep['files'][0]['path']) and not stash.startsWithInList(blacklist, Scene['files'][0]['path']): logReason(DupFileToKeep, Scene, f"blacklist vs not blacklist") DupFileToKeep = Scene elif stash.startsWithInList(blacklist, DupFileToKeep['files'][0]['path']) and stash.startsWithInList(blacklist, Scene['files'][0]['path']) and stash.indexStartsWithInList(blacklist, DupFileToKeep['files'][0]['path']) > stash.indexStartsWithInList(blacklist, Scene['files'][0]['path']): logReason(DupFileToKeep, Scene, f"blacklist-index {stash.indexStartsWithInList(blacklist, DupFileToKeep['files'][0]['path'])} > {stash.indexStartsWithInList(blacklist, Scene['files'][0]['path'])}") DupFileToKeep = Scene elif stash.startsWithInList(graylist, Scene['files'][0]['path']) and not stash.startsWithInList(graylist, DupFileToKeep['files'][0]['path']): logReason(DupFileToKeep, Scene, f"not graylist vs graylist") DupFileToKeep = Scene elif stash.startsWithInList(graylist, Scene['files'][0]['path']) and stash.startsWithInList(graylist, DupFileToKeep['files'][0]['path']) and stash.indexStartsWithInList(graylist, DupFileToKeep['files'][0]['path']) > stash.indexStartsWithInList(graylist, Scene['files'][0]['path']): logReason(DupFileToKeep, Scene, f"graylist-index {stash.indexStartsWithInList(graylist, DupFileToKeep['files'][0]['path'])} > {stash.indexStartsWithInList(graylist, Scene['files'][0]['path'])}") DupFileToKeep = Scene elif favorLongerFileName and len(DupFileToKeep['files'][0]['path']) < len(Scene['files'][0]['path']) and not isWorseKeepCandidate(DupFileToKeep, Scene): logReason(DupFileToKeep, Scene, f"path-len {len(DupFileToKeep['files'][0]['path'])} < {len(Scene['files'][0]['path'])}") DupFileToKeep = Scene elif favorLargerFileSize and int(DupFileToKeep['files'][0]['size']) < int(Scene['files'][0]['size']) and not isWorseKeepCandidate(DupFileToKeep, Scene): logReason(DupFileToKeep, Scene, f"size {DupFileToKeep['files'][0]['size']} < {Scene['files'][0]['size']}") DupFileToKeep = Scene elif not favorLongerFileName and len(DupFileToKeep['files'][0]['path']) > len(Scene['files'][0]['path']) and not isWorseKeepCandidate(DupFileToKeep, Scene): logReason(DupFileToKeep, Scene, f"path-len {len(DupFileToKeep['files'][0]['path'])} > {len(Scene['files'][0]['path'])}") DupFileToKeep = Scene elif not favorLargerFileSize and int(DupFileToKeep['files'][0]['size']) > int(Scene['files'][0]['size']) and not isWorseKeepCandidate(DupFileToKeep, Scene): logReason(DupFileToKeep, Scene, f"size {DupFileToKeep['files'][0]['size']} > {Scene['files'][0]['size']}") DupFileToKeep = Scene else: DupFileToKeep = Scene # stash.Trace(f"DupFileToKeep = {DupFileToKeep}") stash.Debug(f"KeepID={DupFileToKeep['id']}, ID={DupFile['id']} duration=({Scene['files'][0]['duration']}), Size=({Scene['files'][0]['size']}), Res=({Scene['files'][0]['width']} x {Scene['files'][0]['height']}) Name={Scene['files'][0]['path']}, KeepPath={DupFileToKeep['files'][0]['path']}", toAscii=True) else: stash.Error(f"Scene does NOT exist; path={Scene['files'][0]['path']}; ID={Scene['id']}") for DupFile in DupFileDetailList: if DupFileToKeep != None and DupFile['id'] != DupFileToKeep['id']: if merge: result = stash.mergeMetadata(DupFile, DupFileToKeep) if result != "Nothing To Merge": QtyMerge += 1 didAddTag = False if stash.startsWithInList(whitelist, DupFile['files'][0]['path']) and (not whitelistDelDupInSameFolder or not hasSameDir(DupFile['files'][0]['path'], DupFileToKeep['files'][0]['path'])): QtySkipForDel+=1 if isSwapCandidate(DupFileToKeep, DupFile): if merge: stash.mergeMetadata(DupFileToKeep, DupFile) if toRecycleBeforeSwap: sendToTrash(DupFile['files'][0]['path']) stash.Log(f"Moving better file '{DupFileToKeep['files'][0]['path']}' to '{DupFile['files'][0]['path']}'; SrcID={DupFileToKeep['id']};DescID={DupFile['id']};QtyDup={QtyDup};Set={QtyDupSet} of {qtyResults};QtySwap={QtySwap};QtySkipForDel={QtySkipForDel}", toAscii=True, printTo=LOG_STASH_N_PLUGIN) try: shutil.move(DupFileToKeep['files'][0]['path'], DupFile['files'][0]['path']) QtySwap+=1 except Exception as e: tb = traceback.format_exc() stash.Error(f"Exception while moving file '{DupFileToKeep['files'][0]['path']}' to '{DupFile['files'][0]['path']}; SrcID={DupFileToKeep['id']};DescID={DupFile['id']}'; Error: {e}\nTraceBack={tb}") DupFileToKeep = DupFile else: if dupWhitelistTagId and tagDuplicates: didAddTag = setTagId_withRetry(duplicateWhitelistTag, DupFile, DupFileToKeep, ignoreAutoTag=True) stash.Log(f"NOT processing duplicate, because it's in whitelist. '{DupFile['files'][0]['path']}';AddTagW={didAddTag};QtyDup={QtyDup};Set={QtyDupSet} of {qtyResults};QtySkipForDel={QtySkipForDel}", toAscii=True) else: if isTaggedExcluded(DupFile): QtyExcludeForDel+=1 stash.Log(f"Excluding file {DupFile['files'][0]['path']} because tagged for exclusion via tag {excludeDupFileDeleteTag};QtyDup={QtyDup};Set={QtyDupSet} of {qtyResults}") else: # ToDo: Add merge logic here if deleteDup: DupFileName = DupFile['files'][0]['path'] if not deleteBlacklistOnly or stash.startsWithInList(blacklist, DupFile['files'][0]['path']): if not deleteLowerResAndDuration or (isBetterVideo(DupFile, DupFileToKeep) and not significantMoreTimeCompareToBetterVideo(DupFileToKeep, DupFile)) or (significantMoreTimeCompareToBetterVideo(DupFile, DupFileToKeep) and not isBetterVideo(DupFileToKeep, DupFile)): QtyDeleted += 1 DupFileNameOnly = pathlib.Path(DupFileName).stem stash.Warn(f"Deleting duplicate '{DupFileName}';QtyDup={QtyDup};Set={QtyDupSet} of {qtyResults};QtyDeleted={QtyDeleted}", toAscii=True, printTo=LOG_STASH_N_PLUGIN) if alternateTrashCanPath != "": destPath = f"{alternateTrashCanPath }{os.sep}{DupFileNameOnly}" if os.path.isfile(destPath): destPath = f"{alternateTrashCanPath }{os.sep}_{time.time()}_{DupFileNameOnly}" shutil.move(DupFileName, destPath) elif moveToTrashCan: sendToTrash(DupFileName) if dry_run: result = f"dry_run enabled, but scene {DupFile['files'][0]['path']} would have been removed from stash with delete_file=True." stash.Log(result) else: stash.destroyScene(DupFile['id'], delete_file=True) updateDuplicateCandidateForDeletionList(DupFile['id'], removeScene = True) elif tagDuplicates or fileHtmlReport != None: if excludeFromReportIfSignificantTimeDiff and significantTimeDiffCheck(DupFile, DupFileToKeep, True): stash.Log(f"Skipping duplicate {DupFile['files'][0]['path']} (ID={DupFile['id']}), because of time difference greater than {significantTimeDiff} for file {DupFileToKeep['files'][0]['path']}.") continue QtyTagForDel+=1 QtyTagForDelPaginate+=1 didAddTag = False if tagDuplicates: didAddTag = setTagId_withRetry(duplicateMarkForDeletion, DupFile, DupFileToKeep, ignoreAutoTag=True) if fileHtmlReport != None: # ToDo: Add icons using github path # add copy button with copy icon # add move button with r-sqr icon # repace delete button with trashcan icon # add rename file code and button # add delete only from stash db code and button using DB delete icon stash.Debug(f"Adding scene {DupFile['id']} to HTML report.") writeRowToHtmlReport(fileHtmlReport, DupFile, DupFileToKeep, QtyTagForDel, tagDuplicates) fileDuplicateCandidateForDeletionList.write(json.dumps(DupFile) + "\n") if QtyTagForDelPaginate >= htmlReportPaginate: QtyTagForDelPaginate = 0 fileHtmlReport.write("\n") homeHtmReportLink = f"[Home]" prevHtmReportLink = "" if PaginateId > 0: if PaginateId > 1: prevHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId-1}.html") else: prevHtmReport = htmlReportNameHomePage prevHtmReportLink = f"[Prev]" nextHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId+1}.html") nextHtmReportLink = f"[Next]" fileHtmlReport.write(f"
{homeHtmReportLink}{prevHtmReportLink}{nextHtmReportLink}
") fileHtmlReport.write(f"{stash.Setting('htmlReportPostfix')}") fileHtmlReport.close() PaginateId+=1 fileHtmlReport = open(nextHtmReport, "w") fileHtmlReport.write(f"{getHtmlReportTableRow(qtyResults, tagDuplicates)}\n") if PaginateId > 1: prevHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId-1}.html") else: prevHtmReport = htmlReportNameHomePage prevHtmReportLink = f"[Prev]" if len(DupFileSets) > (QtyTagForDel + htmlReportPaginate): nextHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId+1}.html") nextHtmReportLink = f"[Next]" fileHtmlReport.write(f"
{homeHtmReportLink}{prevHtmReportLink}{nextHtmReportLink}
") else: stash.Debug(f"DupFileSets Qty = {len(DupFileSets)}; DupFileDetailList Qty = {len(DupFileDetailList)}; QtyTagForDel = {QtyTagForDel}; htmlReportPaginate = {htmlReportPaginate}; QtyTagForDel + htmlReportPaginate = {QtyTagForDel+htmlReportPaginate}") fileHtmlReport.write(f"
{homeHtmReportLink}{prevHtmReportLink}
") fileHtmlReport.write(f"{stash.Setting('htmlReportTable')}\n") fileHtmlReport.write(f"{htmlReportTableRow}{htmlReportTableHeader}Scene{htmlReportTableHeader}Duplicate to Delete{htmlReportTableHeader}Scene-ToKeep{htmlReportTableHeader}Duplicate to Keep\n") if tagDuplicates and graylistTagging and stash.startsWithInList(graylist, DupFile['files'][0]['path']): stash.addTag(DupFile, graylistMarkForDeletion, ignoreAutoTag=True) if didAddTag: QtyNewlyTag+=1 if QtyTagForDel == 1: stash.Log(f"Tagging duplicate {DupFile['files'][0]['path']} for deletion with tag {duplicateMarkForDeletion}", toAscii=True, printTo=LOG_STASH_N_PLUGIN) else: didAddTag = 1 if didAddTag else 0 stash.Log(f"Tagging duplicate {DupFile['files'][0]['path']} for deletion;AddTag={didAddTag};Qty={QtyDup};Set={QtyDupSet} of {qtyResults};NewlyTag={QtyNewlyTag};isTag={QtyTagForDel}", toAscii=True, printTo=LOG_STASH_N_PLUGIN) stash.Trace(SepLine) if maxDupToProcess > 0 and ((QtyTagForDel > maxDupToProcess) or (QtyTagForDel == 0 and QtyDup > maxDupToProcess)): break fileDuplicateCandidateForDeletionList.close() if fileHtmlReport != None: fileHtmlReport.write("\n") if PaginateId > 0: homeHtmReportLink = f"[Home]" if PaginateId > 1: prevHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId-1}.html") else: prevHtmReport = htmlReportNameHomePage prevHtmReportLink = f"[Prev]" fileHtmlReport.write(f"
{homeHtmReportLink}{prevHtmReportLink}
") fileHtmlReport.write(f"

Total Tagged for Deletion {QtyTagForDel}

\n") fileHtmlReport.write(f"{stash.Setting('htmlReportPostfix')}") fileHtmlReport.close() stash.Log(f"************************************************************", printTo = stash.LogTo.STASH) stash.Log(f"************************************************************", printTo = stash.LogTo.STASH) stash.Log(f"View Stash duplicate report using Stash->Settings->Tools->[Duplicate File Report]", printTo = stash.LogTo.STASH) stash.Log(f"************************************************************", printTo = stash.LogTo.STASH) stash.Log(f"************************************************************", printTo = stash.LogTo.STASH) stash.Debug("#####################################################") stash.Log(f"QtyDupSet={QtyDupSet}, QtyDup={QtyDup}, QtyDeleted={QtyDeleted}, QtySwap={QtySwap}, QtyTagForDel={QtyTagForDel}, QtySkipForDel={QtySkipForDel}, QtyExcludeForDel={QtyExcludeForDel}, QtyExactDup={QtyExactDup}, QtyAlmostDup={QtyAlmostDup}, QtyMerge={QtyMerge}, QtyRealTimeDiff={QtyRealTimeDiff}", printTo=LOG_STASH_N_PLUGIN) killScanningJobs() if cleanAfterDel and deleteDup: stash.Log("Adding clean jobs to the Task Queue", printTo=LOG_STASH_N_PLUGIN) stash.metadata_clean() stash.metadata_clean_generated() stash.optimise_database() if doGeneratePhash: stash.metadata_generate({"phashes": True}) sys.stdout.write("Report complete") def findCurrentTagId(tagNames): # tagNames = [i for n, i in enumerate(tagNames) if i not in tagNames[:n]] for tagName in tagNames: if tag := stash.find_tag(tagName): return tag['id'], tagName return "-1", None def toJson(data): # data = data.replace("'", '"') data = data.replace("\\", "\\\\") data = data.replace("\\\\\\\\", "\\\\") return json.loads(data) def getAnAdvanceMenuOptionSelected(taskName, target, isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater): stash.Log(f"Processing taskName = {taskName}, target = {target}") if "Blacklist" in taskName: tagOrFlag = "Blacklist" if "Graylist" in taskName: tagOrFlag = "Graylist" if "Pinklist" in taskName: tagOrFlag = "Pinklist" if "YellowFlag" in taskName: tagOrFlag = "YellowFlag" if "GreenFlag" in taskName: tagOrFlag = "GreenFlag" if "OrangeFlag" in taskName: tagOrFlag = "OrangeFlag" if "CyanFlag" in taskName: tagOrFlag = "CyanFlag" if "PinkFlag" in taskName: tagOrFlag = "PinkFlag" if "RedFlag" in taskName: tagOrFlag = "RedFlag" if "Less" in taskName: compareToLess = True if "Greater" in taskName: compareToGreater = True if ":TagOnlyScenes" in target: isTagOnlyScenes = True target = target.replace(":TagOnlyScenes","") if "pathToDelete" in taskName: pathToDelete = target.lower() elif "sizeToDelete" in taskName: sizeToDelete = int(target) elif "durationToDelete" in taskName: durationToDelete = int(target) elif "commonResToDelete" in taskName: resolutionToDelete = int(target) elif "resolutionToDelete" in taskName: resolutionToDelete = int(target) elif "ratingToDelete" in taskName: ratingToDelete = int(target) * 20 elif "tagToDelete" in taskName: tagToDelete = target.lower() elif "titleToDelete" in taskName: titleToDelete = target.lower() elif "pathStrToDelete" in taskName: pathStrToDelete = target.lower() elif "fileNotExistToDelete" in taskName: fileNotExistToDelete = True if target == "Tagged": isTagOnlyScenes = True else: isTagOnlyScenes = False elif "TagOnlyScenes" in taskName: isTagOnlyScenes = True return isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater def getAdvanceMenuOptionSelected(advanceMenuOptionSelected): isTagOnlyScenes = False tagOrFlag = None pathToDelete = "" sizeToDelete = -1 durationToDelete = -1 resolutionToDelete = -1 ratingToDelete = -1 tagToDelete = "" titleToDelete = "" pathStrToDelete = "" fileNotExistToDelete = False compareToLess = False compareToGreater = False if advanceMenuOptionSelected: stash.enableProgressBar(False) if 'Target' in stash.JSON_INPUT['args']: if "applyCombo" in stash.PLUGIN_TASK_NAME: jsonObject = toJson(stash.JSON_INPUT['args']['Target']) for taskName in jsonObject: isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater = getAnAdvanceMenuOptionSelected(taskName, jsonObject[taskName], isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater) else: return getAnAdvanceMenuOptionSelected(stash.PLUGIN_TASK_NAME, stash.JSON_INPUT['args']['Target'], isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, compareToLess, compareToGreater) return isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater def getScenesFromReport(): stash.Log(f"Getting candidates for deletion from file {DuplicateCandidateForDeletionList}.") scenes = [] lines = None with open(DuplicateCandidateForDeletionList, 'r') as file: lines = file.readlines() for line in lines: scenes += [json.loads(line)] return scenes deleteSceneFlagBgColor = "#646464" def getFlaggedScenesFromReport(fileName, flagType): stash.Debug(f"Searching for flag type {flagType} in file {fileName}") flaggedScenes = [] lines = None with open(fileName, 'r') as file: lines = file.readlines() stash.Trace(f"line count = {len(lines)}") for line in lines: if line.startswith(f".ID_") and flagType in line and deleteSceneFlagBgColor not in line: id = int(line[4:line.index("{")]) stash.Debug(f"Found scene id = {id} with flag {flagType}") flaggedScenes +=[id] stash.Trace(f"flaggedScenes count = {len(flaggedScenes)}") elif line.startswith(""): if len(flaggedScenes) > 0: return flaggedScenes break stash.Trace(f"Did not find flag {flagType}") return None def getFlaggedScenes(flagType=None, ReportName = htmlReportName): flaggedScenes = [] if flagType == None: flagType = stash.JSON_INPUT['args']['Target'] if flagType == "green": flagType = "#00FF00" else: if flagType == "YellowFlag": flagType = "yellow" if flagType == "GreenFlag": flagType = "#00FF00" if flagType == "OrangeFlag": flagType = "orange" if flagType == "CyanFlag": flagType = "cyan" if flagType == "PinkFlag": flagType = "pink" if flagType == "RedFlag": flagType = "red" stash.Debug(f"Searching for scenes with flag type {flagType}") if os.path.isfile(ReportName): results = getFlaggedScenesFromReport(ReportName,flagType) if results != None: flaggedScenes += results stash.Trace(f"flaggedScenes count = {len(flaggedScenes)}") for x in range(2, 9999): fileName = ReportName.replace(".html", f"_{x-1}.html") stash.Debug(f"Checking if file '{fileName}' exist.") if not os.path.isfile(fileName): break results = getFlaggedScenesFromReport(fileName,flagType) if results != None: flaggedScenes += results stash.Trace(f"flaggedScenes count = {len(flaggedScenes)}") else: stash.Log(f"Report file does not exist: {ReportName}") return flaggedScenes, flagType # ////////////////////////////////////////////////////////////////////////////// # ////////////////////////////////////////////////////////////////////////////// def manageDuplicatesTaggedOrInReport(deleteScenes=False, clearTag=False, setGrayListTag=False, tagId=-1, advanceMenuOptionSelected=False, checkFlagOption=False): tagName = None if tagId == -1: tagId, tagName = findCurrentTagId([duplicateMarkForDeletion, base1_duplicateMarkForDeletion, base2_duplicateMarkForDeletion, 'DuplicateMarkForDeletion', '_DuplicateMarkForDeletion']) if int(tagId) < 0: stash.Warn(f"Could not find tag ID for tag '{duplicateMarkForDeletion}'.") return excludedTags = [duplicateMarkForDeletion] if clearAllDupfileManagerTags: excludedTags = [duplicateMarkForDeletion, duplicateWhitelistTag, excludeDupFileDeleteTag, graylistMarkForDeletion, longerDurationLowerResolution] isTagOnlyScenes, tagOrFlag, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater = getAdvanceMenuOptionSelected(advanceMenuOptionSelected) if advanceMenuOptionSelected and deleteScenes and pathToDelete == "" and tagToDelete == "" and titleToDelete == "" and pathStrToDelete == "" and sizeToDelete == -1 and durationToDelete == -1 and resolutionToDelete == -1 and ratingToDelete == -1 and fileNotExistToDelete == False: stash.Error("Running advance menu option with no options enabled.") return flaggedScenes = None flagType = None if checkFlagOption or (tagOrFlag != None and "Flag" in tagOrFlag): if checkFlagOption: flaggedScenes, flagType = getFlaggedScenes() else: checkFlagOption = True flaggedScenes, flagType = getFlaggedScenes(tagOrFlag) if flaggedScenes == None or len(flaggedScenes) == 0: stash.Error(f"Early exit, because found no scenes with flag {flagType}.") return stash.Debug(f"Fournd {len(flaggedScenes)} scenes with flag {flagType}") QtyDup = 0 QtyDeleted = 0 QtyClearedTags = 0 QtySetGraylistTag = 0 QtyFailedQuery = 0 stash.Debug("#########################################################################") stash.startSpinningProcessBar() if advanceMenuOptionSelected == False and checkFlagOption == False: isTagOnlyScenes = True if isTagOnlyScenes: stash.Log(f"Getting candidates for deletion by using tag-ID {tagId} and tag-name {tagName}; isTagOnlyScenes={isTagOnlyScenes};advanceMenuOptionSelected={advanceMenuOptionSelected}") scenes = stash.find_scenes(f={"tags": {"value":tagId, "modifier":"INCLUDES"}}, fragment=fragmentForSceneDetails) # Old setting 'id tags {id name} files {path width height duration size video_codec bit_rate frame_rate} details title rating100') else: scenes = getScenesFromReport() stash.stopSpinningProcessBar() qtyResults = len(scenes) if isTagOnlyScenes: stash.Log(f"Found {qtyResults} scenes with tag ({duplicateMarkForDeletion})") else: stash.Log(f"Found {qtyResults} scenes in report") stash.setProgressBarIter(qtyResults) for scene in scenes: QtyDup += 1 stash.progressBar(QtyDup, qtyResults) # scene = stash.find_scene(sceneID['id']) # if scene == None or len(scene) == 0: # stash.Warn(f"Could not get scene data for scene ID {scene['id']}.") # QtyFailedQuery += 1 # continue # stash.Trace(f"scene={scene}") if clearTag: QtyClearedTags += 1 # ToDo: Add logic to exclude graylistMarkForDeletion tags = [int(item['id']) for item in scene["tags"] if item['name'] not in excludedTags] # if clearAllDupfileManagerTags: # tags = [] # for tag in scene["tags"]: # if tag['name'] in excludedTags: # continue # tags += [int(tag['id'])] stash.TraceOnce(f"tagId={tagId}, len={len(tags)}, tags = {tags}") dataDict = {'id' : scene['id']} if addPrimaryDupPathToDetails: sceneDetails = scene['details'] if sceneDetails.find(detailPrefix) == 0 and sceneDetails.find(detailPostfix) > 1: Pos1 = sceneDetails.find(detailPrefix) Pos2 = sceneDetails.find(detailPostfix) sceneDetails = sceneDetails[0:Pos1] + sceneDetails[Pos2 + len(detailPostfix):] dataDict.update({'details' : sceneDetails}) dataDict.update({'tag_ids' : tags}) stash.Log(f"Updating scene with {dataDict};QtyClearedTags={QtyClearedTags};Count={QtyDup} of {qtyResults}") stash.updateScene(dataDict) # stash.removeTag(scene, duplicateMarkForDeletion) elif setGrayListTag: if stash.startsWithInList(graylist, scene['files'][0]['path']): QtySetGraylistTag+=1 if stash.addTag(scene, graylistMarkForDeletion, ignoreAutoTag=True): stash.Log(f"Added tag {graylistMarkForDeletion} to scene {scene['files'][0]['path']};QtySetGraylistTag={QtySetGraylistTag};Count={QtyDup} of {qtyResults}") else: stash.Trace(f"Scene already had tag {graylistMarkForDeletion}; {scene['files'][0]['path']}") elif deleteScenes: DupFileName = scene['files'][0]['path'] DupFileNameOnly = pathlib.Path(DupFileName).stem if checkFlagOption and (tagOrFlag == None or "Flag" not in tagOrFlag): if int(scene['id']) in flaggedScenes: stash.Log(f"Found {flagType} flagged candidate for deletion; Scene ID = {scene['id']}") else: continue elif advanceMenuOptionSelected: if checkFlagOption: if int(scene['id']) in flaggedScenes: stash.Trace(f"Found {flagType} flag for Scene ID = {scene['id']}") else: continue if tagOrFlag == "Blacklist": if not stash.startsWithInList(blacklist, scene['files'][0]['path']): continue if tagOrFlag == "Graylist": if not stash.startsWithInList(graylist, scene['files'][0]['path']): continue if tagOrFlag == "Pinklist": if not stash.startsWithInList(pinklist, scene['files'][0]['path']): continue if pathToDelete != "": if not DupFileName.lower().startswith(pathToDelete): stash.Debug(f"Skipping file {DupFileName} because it does not start with {pathToDelete}.") continue if pathStrToDelete != "": if not pathStrToDelete in DupFileName.lower(): stash.Debug(f"Skipping file {DupFileName} because it does not contain value {pathStrToDelete}.") continue if sizeToDelete != -1: compareTo = int(scene['files'][0]['size']) if compareToLess: if not (compareTo < sizeToDelete): continue elif compareToGreater: if not (compareTo > sizeToDelete): continue else: if not compareTo == sizeToDelete: continue if durationToDelete != -1: compareTo = int(scene['files'][0]['duration']) if compareToLess: if not (compareTo < durationToDelete): continue elif compareToGreater: if not (compareTo > durationToDelete): continue else: if not compareTo == durationToDelete: continue if resolutionToDelete != -1: compareTo = int(scene['files'][0]['width']) * int(scene['files'][0]['height']) if compareToLess: if not (compareTo < resolutionToDelete): continue elif compareToGreater: if not (compareTo > resolutionToDelete): continue else: if not compareTo == resolutionToDelete: continue if ratingToDelete != -1: if scene['rating100'] == "None": compareTo = 0 else: compareTo = int(scene['rating100']) if compareToLess: if not (compareTo < resolutionToDelete): continue elif compareToGreater: if not (compareTo > resolutionToDelete): continue else: if not compareTo == resolutionToDelete: continue if titleToDelete != "": if not titleToDelete in scene['title'].lower(): stash.Debug(f"Skipping file {DupFileName} because it does not contain value {titleToDelete} in title ({scene['title']}).") continue if tagToDelete != "": doProcessThis = False for tag in scene['tags']: if tag['name'].lower() == tagToDelete: doProcessThis = True break if doProcessThis == False: continue if fileNotExistToDelete: if os.path.isfile(scene['files'][0]['path']): continue stash.Log(f"Deleting duplicate '{DupFileName}'", toAscii=True, printTo=LOG_STASH_N_PLUGIN) if alternateTrashCanPath != "": destPath = f"{alternateTrashCanPath }{os.sep}{DupFileNameOnly}" if os.path.isfile(destPath): destPath = f"{alternateTrashCanPath }{os.sep}_{time.time()}_{DupFileNameOnly}" shutil.move(DupFileName, destPath) elif moveToTrashCan: sendToTrash(DupFileName) result = deleteScene(scene=scene['id'], deleteFile=True, writeToStdOut=False) QtyDeleted += 1 stash.Debug(f"destroyScene result={result} for file {DupFileName};QtyDeleted={QtyDeleted};Count={QtyDup} of {qtyResults}", toAscii=True) else: stash.Error("manageDuplicatesTaggedOrInReport called with invlaid input arguments. Doing early exit.") return stash.Debug("#####################################################") stash.Log(f"QtyDup={QtyDup}, QtyClearedTags={QtyClearedTags}, QtySetGraylistTag={QtySetGraylistTag}, QtyDeleted={QtyDeleted}, QtyFailedQuery={QtyFailedQuery}", printTo=LOG_STASH_N_PLUGIN) killScanningJobs() if deleteScenes and not advanceMenuOptionSelected: if cleanAfterDel: stash.Log("Adding clean jobs to the Task Queue", printTo=LOG_STASH_N_PLUGIN) stash.metadata_clean() stash.metadata_clean_generated() stash.optimise_database() def removeDupTag(): if 'Target' not in stash.JSON_INPUT['args']: stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})") return scene = stash.JSON_INPUT['args']['Target'] stash.Log(f"Processing scene ID# {scene}") stash.removeTag(scene, duplicateMarkForDeletion) stash.Log(f"Done removing tag from scene {scene}.") jsonReturn = "{'removeDupTag' : 'complete', 'id': '" + f"{scene}" + "'}" stash.Log(f"Sending json value {jsonReturn}") sys.stdout.write(jsonReturn) def addExcludeTag(): if 'Target' not in stash.JSON_INPUT['args']: stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})") return scene = stash.JSON_INPUT['args']['Target'] stash.Log(f"Processing scene ID# {scene}") stash.addTag(scene, excludeDupFileDeleteTag) stash.Log(f"Done adding exclude tag to scene {scene}.") sys.stdout.write("{" + f"addExcludeTag : 'complete', id: '{scene}'" + "}") def removeExcludeTag(): if 'Target' not in stash.JSON_INPUT['args']: stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})") return scene = stash.JSON_INPUT['args']['Target'] stash.Log(f"Processing scene ID# {scene}") stash.removeTag(scene, excludeDupFileDeleteTag) stash.Log(f"Done removing exclude tag from scene {scene}.") sys.stdout.write("{" + f"removeExcludeTag : 'complete', id: '{scene}'" + "}") def getParseData(getSceneDetails1=True, getSceneDetails2=True, checkIfNotSplitValue=False): if 'Target' not in stash.JSON_INPUT['args']: stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})") return None, None targetsSrc = stash.JSON_INPUT['args']['Target'] if checkIfNotSplitValue and ":" not in targetsSrc: return targetsSrc, None targets = targetsSrc.split(":") if len(targets) < 2: stash.Error(f"Could not get both targets from string {targetsSrc}") return None, None stash.Log(f"Parsed targets {targets[0]} and {targets[1]}") target1 = targets[0] target2 = targets[1] if getSceneDetails1: target1 = stash.find_scene(int(target1)) if getSceneDetails2: target2 = stash.find_scene(int(target2)) elif len(targets) > 2: target2 = target2 + targets[2] return target1, target2 def mergeMetadataInThisFile(fileName): stash.Debug(f"Checking report file '{fileName}' for yellow icons indicating missing metadata in DuplicateToKeep.") lines = None with open(fileName, 'r') as file: lines = file.readlines() for line in lines: if "https://www.axter.com/images/stash/Yellow" in line: # FYI: This catches YellowGroup.png as well, even though group is not currently supported for merging searchStrScene1 = "