forked from Github/Axter-Stash
1441 lines
90 KiB
Python
1441 lines
90 KiB
Python
# Description: This is a Stash plugin which manages duplicate files.
|
|
# By David Maisonave (aka Axter) Jul-2024 (https://www.axter.com/)
|
|
# Get the latest developers version from following link: https://github.com/David-Maisonave/Axter-Stash/tree/main/plugins/DupFileManager
|
|
# Note: To call this script outside of Stash, pass argument --url
|
|
# Example: python DupFileManager.py --url http://localhost:9999 -a
|
|
try:
|
|
import ModulesValidate
|
|
ModulesValidate.modulesInstalled(["send2trash", "requests"], silent=True)
|
|
except Exception as e:
|
|
import traceback, sys
|
|
tb = traceback.format_exc()
|
|
print(f"ModulesValidate Exception. Error: {e}\nTraceBack={tb}", file=sys.stderr)
|
|
import os, sys, time, pathlib, argparse, platform, shutil, traceback, logging, requests
|
|
from datetime import datetime
|
|
from StashPluginHelper import StashPluginHelper
|
|
from stashapi.stash_types import PhashDistance
|
|
from DupFileManager_config import config # Import config from DupFileManager_config.py
|
|
from DupFileManager_report_config import report_config
|
|
|
|
# ToDo: make sure the following line of code works
|
|
config += report_config
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--url', '-u', dest='stash_url', type=str, help='Add Stash URL')
|
|
parser.add_argument('--trace', '-t', dest='trace', action='store_true', help='Enables debug trace mode.')
|
|
parser.add_argument('--add_dup_tag', '-a', dest='dup_tag', action='store_true', help='Set a tag to duplicate files.')
|
|
parser.add_argument('--clear_dup_tag', '-c', dest='clear_tag', action='store_true', help='Clear duplicates of duplicate tags.')
|
|
parser.add_argument('--del_tag_dup', '-d', dest='del_tag', action='store_true', help='Only delete scenes having DuplicateMarkForDeletion tag.')
|
|
parser.add_argument('--remove_dup', '-r', dest='remove', action='store_true', help='Remove (delete) duplicate files.')
|
|
parse_args = parser.parse_args()
|
|
|
|
settings = {
|
|
"matchDupDistance": 0,
|
|
"mergeDupFilename": False,
|
|
"whitelistDelDupInSameFolder": False,
|
|
"zvWhitelist": "",
|
|
"zwGraylist": "",
|
|
"zxBlacklist": "",
|
|
"zyMaxDupToProcess": 0,
|
|
"zySwapHighRes": False,
|
|
"zySwapLongLength": False,
|
|
"zySwapBetterBitRate": False,
|
|
"zySwapCodec": False,
|
|
"zySwapBetterFrameRate": False,
|
|
"zzDebug": False,
|
|
"zzTracing": False,
|
|
|
|
"zzObsoleteSettingsCheckVer2": False, # This is a hidden variable that is NOT displayed in the UI
|
|
|
|
# Obsolete setting names
|
|
"zWhitelist": "",
|
|
"zxGraylist": "",
|
|
"zyBlacklist": "",
|
|
"zyMatchDupDistance": 0,
|
|
"zSwapHighRes": False,
|
|
"zSwapLongLength": False,
|
|
"zSwapBetterBitRate": False,
|
|
"zSwapCodec": False,
|
|
"zSwapBetterFrameRate": False,
|
|
}
|
|
stash = StashPluginHelper(
|
|
stash_url=parse_args.stash_url,
|
|
debugTracing=parse_args.trace,
|
|
settings=settings,
|
|
config=config,
|
|
maxbytes=10*1024*1024,
|
|
DebugTraceFieldName="zzTracing",
|
|
DebugFieldName="zzDebug",
|
|
)
|
|
stash.convertToAscii = True
|
|
|
|
advanceMenuOptions = [ "applyCombo", "applyComboBlacklist", "pathToDelete", "pathToDeleteBlacklist", "sizeToDeleteLess", "sizeToDeleteGreater", "sizeToDeleteBlacklistLess", "sizeToDeleteBlacklistGreater", "durationToDeleteLess", "durationToDeleteGreater", "durationToDeleteBlacklistLess", "durationToDeleteBlacklistGreater",
|
|
"commonResToDeleteLess", "commonResToDeleteEq", "commonResToDeleteGreater", "commonResToDeleteBlacklistLess", "commonResToDeleteBlacklistEq", "commonResToDeleteBlacklistGreater", "resolutionToDeleteLess", "resolutionToDeleteEq", "resolutionToDeleteGreater",
|
|
"resolutionToDeleteBlacklistLess", "resolutionToDeleteBlacklistEq", "resolutionToDeleteBlacklistGreater", "ratingToDeleteLess", "ratingToDeleteEq", "ratingToDeleteGreater", "ratingToDeleteBlacklistLess", "ratingToDeleteBlacklistEq", "ratingToDeleteBlacklistGreater",
|
|
"tagToDelete", "tagToDeleteBlacklist", "titleToDelete", "titleToDeleteBlacklist", "pathStrToDelete", "pathStrToDeleteBlacklist"]
|
|
|
|
doJsonReturnModeTypes = ["tag_duplicates_task", "removeDupTag", "addExcludeTag", "removeExcludeTag", "mergeTags", "getLocalDupReportPath",
|
|
"createDuplicateReportWithoutTagging", "deleteLocalDupReportHtmlFiles", "clear_duplicate_tags_task",
|
|
"deleteAllDupFileManagerTags", "deleteBlackListTaggedDuplicatesTask", "deleteTaggedDuplicatesLwrResOrLwrDuration",
|
|
"deleteBlackListTaggedDuplicatesLwrResOrLwrDuration"]
|
|
doJsonReturnModeTypes += [advanceMenuOptions]
|
|
doJsonReturn = False
|
|
if len(sys.argv) < 2 and stash.PLUGIN_TASK_NAME in doJsonReturnModeTypes:
|
|
doJsonReturn = True
|
|
stash.log_to_norm = stash.LogTo.FILE
|
|
elif stash.PLUGIN_TASK_NAME == "doEarlyExit":
|
|
time.sleep(3)
|
|
stash.Log("Doing early exit because of task name")
|
|
time.sleep(3)
|
|
exit(0)
|
|
|
|
stash.Log("******************* Starting *******************")
|
|
if len(sys.argv) > 1:
|
|
stash.Log(f"argv = {sys.argv}")
|
|
else:
|
|
stash.Debug(f"No command line arguments. JSON_INPUT['args'] = {stash.JSON_INPUT['args']}; PLUGIN_TASK_NAME = {stash.PLUGIN_TASK_NAME}; argv = {sys.argv}")
|
|
stash.status(logLevel=logging.DEBUG)
|
|
|
|
obsoleteSettingsToConvert = {"zWhitelist" : "zvWhitelist", "zxGraylist" : "zwGraylist", "zyBlacklist" : "zxBlacklist", "zyMatchDupDistance" : "matchDupDistance", "zSwapHighRes" : "zySwapHighRes", "zSwapLongLength" : "zySwapLongLength", "zSwapBetterBitRate" : "zySwapBetterBitRate", "zSwapCodec" : "zySwapCodec", "zSwapBetterFrameRate" : "zySwapBetterFrameRate"}
|
|
stash.replaceObsoleteSettings(obsoleteSettingsToConvert, "zzObsoleteSettingsCheckVer2")
|
|
|
|
|
|
LOG_STASH_N_PLUGIN = stash.LogTo.STASH if stash.CALLED_AS_STASH_PLUGIN else stash.LogTo.CONSOLE + stash.LogTo.FILE
|
|
listSeparator = stash.Setting('listSeparator', ',', notEmpty=True)
|
|
addPrimaryDupPathToDetails = stash.Setting('addPrimaryDupPathToDetails')
|
|
clearAllDupfileManagerTags = stash.Setting('clearAllDupfileManagerTags')
|
|
doGeneratePhash = stash.Setting('doGeneratePhash')
|
|
mergeDupFilename = stash.Setting('mergeDupFilename')
|
|
moveToTrashCan = False if stash.Setting('permanentlyDelete') else True
|
|
alternateTrashCanPath = stash.Setting('dup_path')
|
|
whitelistDelDupInSameFolder = stash.Setting('whitelistDelDupInSameFolder')
|
|
graylistTagging = stash.Setting('graylistTagging')
|
|
maxDupToProcess = int(stash.Setting('zyMaxDupToProcess'))
|
|
significantTimeDiff = float(stash.Setting('significantTimeDiff'))
|
|
toRecycleBeforeSwap = stash.Setting('toRecycleBeforeSwap')
|
|
cleanAfterDel = stash.Setting('cleanAfterDel')
|
|
|
|
swapHighRes = stash.Setting('zySwapHighRes')
|
|
swapLongLength = stash.Setting('zySwapLongLength')
|
|
swapBetterBitRate = stash.Setting('zySwapBetterBitRate')
|
|
swapCodec = stash.Setting('zySwapCodec')
|
|
swapBetterFrameRate = stash.Setting('zySwapBetterFrameRate')
|
|
favorLongerFileName = stash.Setting('favorLongerFileName')
|
|
favorLargerFileSize = stash.Setting('favorLargerFileSize')
|
|
favorBitRateChange = stash.Setting('favorBitRateChange')
|
|
favorHighBitRate = stash.Setting('favorHighBitRate')
|
|
favorFrameRateChange = stash.Setting('favorFrameRateChange')
|
|
favorHigherFrameRate = stash.Setting('favorHigherFrameRate')
|
|
favorCodecRanking = stash.Setting('favorCodecRanking')
|
|
codecRankingSetToUse = stash.Setting('codecRankingSetToUse')
|
|
if codecRankingSetToUse == 4:
|
|
codecRanking = stash.Setting('codecRankingSet4')
|
|
elif codecRankingSetToUse == 3:
|
|
codecRanking = stash.Setting('codecRankingSet3')
|
|
elif codecRankingSetToUse == 2:
|
|
codecRanking = stash.Setting('codecRankingSet2')
|
|
else:
|
|
codecRanking = stash.Setting('codecRankingSet1')
|
|
skipIfTagged = stash.Setting('skipIfTagged')
|
|
killScanningPostProcess = stash.Setting('killScanningPostProcess')
|
|
tagLongDurationLowRes = stash.Setting('tagLongDurationLowRes')
|
|
bitRateIsImporantComp = stash.Setting('bitRateIsImporantComp')
|
|
codecIsImporantComp = stash.Setting('codecIsImporantComp')
|
|
|
|
excludeFromReportIfSignificantTimeDiff = False
|
|
|
|
matchDupDistance = int(stash.Setting('matchDupDistance'))
|
|
matchPhaseDistance = PhashDistance.EXACT
|
|
matchPhaseDistanceText = "Exact Match"
|
|
if stash.PLUGIN_TASK_NAME == "tag_duplicates_task" and 'Target' in stash.JSON_INPUT['args']:
|
|
if stash.JSON_INPUT['args']['Target'].startswith("0"):
|
|
matchDupDistance = 0
|
|
elif stash.JSON_INPUT['args']['Target'].startswith("1"):
|
|
matchDupDistance = 1
|
|
elif stash.JSON_INPUT['args']['Target'].startswith("2"):
|
|
matchDupDistance = 2
|
|
elif stash.JSON_INPUT['args']['Target'].startswith("3"):
|
|
matchDupDistance = 3
|
|
|
|
if stash.JSON_INPUT['args']['Target'].find(":") == 1:
|
|
significantTimeDiff = float(stash.JSON_INPUT['args']['Target'][2:])
|
|
excludeFromReportIfSignificantTimeDiff = True
|
|
|
|
if matchDupDistance == 1:
|
|
matchPhaseDistance = PhashDistance.HIGH
|
|
matchPhaseDistanceText = "High Match"
|
|
elif matchDupDistance == 2:
|
|
matchPhaseDistance = PhashDistance.MEDIUM
|
|
matchPhaseDistanceText = "Medium Match"
|
|
elif matchDupDistance == 3:
|
|
matchPhaseDistance = PhashDistance.LOW
|
|
matchPhaseDistanceText = "Low Match"
|
|
|
|
# significantTimeDiff can not be higher than 1 and shouldn't be lower than .5
|
|
if significantTimeDiff > 1:
|
|
significantTimeDiff = float(1.00)
|
|
if significantTimeDiff < .25:
|
|
significantTimeDiff = float(0.25)
|
|
|
|
|
|
duplicateMarkForDeletion = stash.Setting('DupFileTag')
|
|
if duplicateMarkForDeletion == "":
|
|
duplicateMarkForDeletion = 'DuplicateMarkForDeletion'
|
|
|
|
base1_duplicateMarkForDeletion = duplicateMarkForDeletion
|
|
|
|
duplicateWhitelistTag = stash.Setting('DupWhiteListTag')
|
|
if duplicateWhitelistTag == "":
|
|
duplicateWhitelistTag = '_DuplicateWhitelistFile'
|
|
|
|
excludeDupFileDeleteTag = stash.Setting('excludeDupFileDeleteTag')
|
|
if excludeDupFileDeleteTag == "":
|
|
excludeDupFileDeleteTag = '_ExcludeDuplicateMarkForDeletion'
|
|
|
|
graylistMarkForDeletion = stash.Setting('graylistMarkForDeletion')
|
|
if graylistMarkForDeletion == "":
|
|
graylistMarkForDeletion = '_GraylistMarkForDeletion'
|
|
|
|
longerDurationLowerResolution = stash.Setting('longerDurationLowerResolution')
|
|
if longerDurationLowerResolution == "":
|
|
longerDurationLowerResolution = '_LongerDurationLowerResolution'
|
|
|
|
excludeMergeTags = [duplicateMarkForDeletion, duplicateWhitelistTag, excludeDupFileDeleteTag]
|
|
|
|
if stash.Setting('underscoreDupFileTag') and not duplicateMarkForDeletion.startswith('_'):
|
|
duplicateMarkForDeletionWithOutUnderscore = duplicateMarkForDeletion
|
|
duplicateMarkForDeletion = "_" + duplicateMarkForDeletion
|
|
if stash.renameTag(duplicateMarkForDeletionWithOutUnderscore, duplicateMarkForDeletion):
|
|
stash.Log(f"Renamed tag {duplicateMarkForDeletionWithOutUnderscore} to {duplicateMarkForDeletion}")
|
|
stash.Trace(f"Added underscore to {duplicateMarkForDeletionWithOutUnderscore} = {duplicateMarkForDeletion}")
|
|
excludeMergeTags += [duplicateMarkForDeletion]
|
|
else:
|
|
stash.Trace(f"duplicateMarkForDeletion = {duplicateMarkForDeletion}")
|
|
|
|
base2_duplicateMarkForDeletion = duplicateMarkForDeletion
|
|
|
|
if stash.Setting('appendMatchDupDistance'):
|
|
duplicateMarkForDeletion += f"_{matchDupDistance}"
|
|
excludeMergeTags += [duplicateMarkForDeletion]
|
|
|
|
stash.initMergeMetadata(excludeMergeTags)
|
|
|
|
graylist = stash.Setting('zwGraylist').split(listSeparator)
|
|
graylist = [item.lower() for item in graylist]
|
|
if graylist == [""] : graylist = []
|
|
stash.Trace(f"graylist = {graylist}")
|
|
whitelist = stash.Setting('zvWhitelist').split(listSeparator)
|
|
whitelist = [item.lower() for item in whitelist]
|
|
if whitelist == [""] : whitelist = []
|
|
stash.Trace(f"whitelist = {whitelist}")
|
|
blacklist = stash.Setting('zxBlacklist').split(listSeparator)
|
|
blacklist = [item.lower() for item in blacklist]
|
|
if blacklist == [""] : blacklist = []
|
|
stash.Trace(f"blacklist = {blacklist}")
|
|
|
|
def realpath(path):
|
|
"""
|
|
get_symbolic_target for win
|
|
"""
|
|
try:
|
|
import win32file
|
|
f = win32file.CreateFile(path, win32file.GENERIC_READ,
|
|
win32file.FILE_SHARE_READ, None,
|
|
win32file.OPEN_EXISTING,
|
|
win32file.FILE_FLAG_BACKUP_SEMANTICS, None)
|
|
target = win32file.GetFinalPathNameByHandle(f, 0)
|
|
# an above gives us something like u'\\\\?\\C:\\tmp\\scalarizr\\3.3.0.7978'
|
|
return target.strip('\\\\?\\')
|
|
except ImportError:
|
|
handle = open_dir(path)
|
|
target = get_symbolic_target(handle)
|
|
check_closed(handle)
|
|
return target
|
|
|
|
def isReparsePoint(path):
|
|
import win32api
|
|
import win32con
|
|
from parse_reparsepoint import Navigator
|
|
FinalPathname = realpath(path)
|
|
stash.Log(f"(path='{path}') (FinalPathname='{FinalPathname}')")
|
|
if FinalPathname != path:
|
|
stash.Log(f"Symbolic link '{path}'")
|
|
return True
|
|
if not os.path.isdir(path):
|
|
path = os.path.dirname(path)
|
|
return win32api.GetFileAttributes(path) & win32con.FILE_ATTRIBUTE_REPARSE_POINT
|
|
|
|
def testReparsePointAndSymLink(merge=False, deleteDup=False):
|
|
stash.Trace(f"Debug Tracing (platform.system()={platform.system()})")
|
|
myTestPath1 = r"B:\V\V\Tip\POV - Holly Molly petite ginger anal slut - RedTube.mp4" # not a reparse point or symbolic link
|
|
myTestPath2 = r"B:\_\SpecialSet\Amateur Anal Attempts\BRCC test studio name.m2ts" # reparse point
|
|
myTestPath3 = r"B:\_\SpecialSet\Amateur Anal Attempts\Amateur Anal Attempts 4.mp4" #symbolic link
|
|
myTestPath4 = r"E:\Stash\plugins\RenameFile\README.md" #symbolic link
|
|
myTestPath5 = r"E:\_\David-Maisonave\Axter-Stash\plugins\RenameFile\README.md" #symbolic link
|
|
myTestPath6 = r"E:\_\David-Maisonave\Axter-Stash\plugins\DeleteMe\Renamer\README.md" # not reparse point
|
|
stash.Log(f"Testing '{myTestPath1}'")
|
|
if isReparsePoint(myTestPath1):
|
|
stash.Log(f"isSymLink '{myTestPath1}'")
|
|
else:
|
|
stash.Log(f"Not isSymLink '{myTestPath1}'")
|
|
|
|
if isReparsePoint(myTestPath2):
|
|
stash.Log(f"isSymLink '{myTestPath2}'")
|
|
else:
|
|
stash.Log(f"Not isSymLink '{myTestPath2}'")
|
|
|
|
if isReparsePoint(myTestPath3):
|
|
stash.Log(f"isSymLink '{myTestPath3}'")
|
|
else:
|
|
stash.Log(f"Not isSymLink '{myTestPath3}'")
|
|
|
|
if isReparsePoint(myTestPath4):
|
|
stash.Log(f"isSymLink '{myTestPath4}'")
|
|
else:
|
|
stash.Log(f"Not isSymLink '{myTestPath4}'")
|
|
|
|
if isReparsePoint(myTestPath5):
|
|
stash.Log(f"isSymLink '{myTestPath5}'")
|
|
else:
|
|
stash.Log(f"Not isSymLink '{myTestPath5}'")
|
|
|
|
if isReparsePoint(myTestPath6):
|
|
stash.Log(f"isSymLink '{myTestPath6}'")
|
|
else:
|
|
stash.Log(f"Not isSymLink '{myTestPath6}'")
|
|
return
|
|
|
|
detailPrefix = "BaseDup="
|
|
detailPostfix = "<BaseDup>\n"
|
|
|
|
def setTagId(tagName, sceneDetails, DupFileToKeep, TagReason="", ignoreAutoTag=False):
|
|
details = ""
|
|
ORG_DATA_DICT = {'id' : sceneDetails['id']}
|
|
dataDict = ORG_DATA_DICT.copy()
|
|
doAddTag = True
|
|
if addPrimaryDupPathToDetails:
|
|
BaseDupStr = f"{detailPrefix}{DupFileToKeep['files'][0]['path']}\n{stash.STASH_URL}/scenes/{DupFileToKeep['id']}\n{TagReason}(matchDupDistance={matchPhaseDistanceText})\n{detailPostfix}"
|
|
if sceneDetails['details'] == "":
|
|
details = BaseDupStr
|
|
elif not sceneDetails['details'].startswith(detailPrefix):
|
|
details = f"{BaseDupStr};\n{sceneDetails['details']}"
|
|
for tag in sceneDetails['tags']:
|
|
if tag['name'] == tagName:
|
|
doAddTag = False
|
|
break
|
|
if doAddTag:
|
|
stash.addTag(sceneDetails, tagName, ignoreAutoTag=ignoreAutoTag)
|
|
if details != "":
|
|
dataDict.update({'details' : details})
|
|
if dataDict != ORG_DATA_DICT:
|
|
stash.updateScene(dataDict)
|
|
stash.Trace(f"[setTagId] Updated {sceneDetails['files'][0]['path']} with metadata {dataDict} and tag {tagName}", toAscii=True)
|
|
else:
|
|
stash.Trace(f"[setTagId] Nothing to update {sceneDetails['files'][0]['path']} already has tag {tagName}.", toAscii=True)
|
|
return doAddTag
|
|
|
|
def setTagId_withRetry(tagName, sceneDetails, DupFileToKeep, TagReason="", ignoreAutoTag=False, retryCount = 12, sleepSecondsBetweenRetry = 5):
|
|
errMsg = None
|
|
for i in range(0, retryCount):
|
|
try:
|
|
if errMsg != None:
|
|
stash.Warn(errMsg)
|
|
return setTagId(tagName, sceneDetails, DupFileToKeep, TagReason, ignoreAutoTag)
|
|
except (requests.exceptions.ConnectionError, ConnectionResetError):
|
|
tb = traceback.format_exc()
|
|
errMsg = f"[setTagId] Exception calling setTagId. Will retry; count({i}); Error: {e}\nTraceBack={tb}"
|
|
except Exception as e:
|
|
tb = traceback.format_exc()
|
|
errMsg = f"[setTagId] Unknown exception calling setTagId. Will retry; count({i}); Error: {e}\nTraceBack={tb}"
|
|
time.sleep(sleepSecondsBetweenRetry)
|
|
|
|
def hasSameDir(path1, path2):
|
|
if pathlib.Path(path1).resolve().parent == pathlib.Path(path2).resolve().parent:
|
|
return True
|
|
return False
|
|
|
|
def sendToTrash(path):
|
|
if not os.path.isfile(path):
|
|
stash.Warn(f"File does not exist: {path}.", toAscii=True)
|
|
return False
|
|
try:
|
|
from send2trash import send2trash # Requirement: pip install Send2Trash
|
|
send2trash(path)
|
|
return True
|
|
except Exception as e:
|
|
stash.Error(f"Failed to send file {path} to recycle bin. Error: {e}", toAscii=True)
|
|
try:
|
|
if os.path.isfile(path):
|
|
os.remove(path)
|
|
return True
|
|
except Exception as e:
|
|
stash.Error(f"Failed to delete file {path}. Error: {e}", toAscii=True)
|
|
return False
|
|
# If ckTimeDiff=False: Does durration2 have significant more time than durration1
|
|
def significantTimeDiffCheck(durration1, durration2, ckTimeDiff = False): # If ckTimeDiff=True: is time different significant in either direction.
|
|
if not isinstance(durration1, int) and 'files' in durration1:
|
|
durration1 = int(durration1['files'][0]['duration'])
|
|
durration2 = int(durration2['files'][0]['duration'])
|
|
timeDiff = getTimeDif(durration1, durration2)
|
|
if ckTimeDiff and timeDiff > 1:
|
|
timeDiff = getTimeDif(durration2, durration1)
|
|
if timeDiff < significantTimeDiff:
|
|
return True
|
|
return False
|
|
|
|
def getTimeDif(durration1, durration2): # Where durration1 is ecpected to be smaller than durration2 IE(45/60=.75)
|
|
return durration1 / durration2
|
|
|
|
def isBetterVideo(scene1, scene2, swapCandidateCk = False): # is scene2 better than scene1
|
|
# Prioritize higher reslution over codec, bit rate, and frame rate
|
|
if int(scene1['files'][0]['width']) * int(scene1['files'][0]['height']) > int(scene2['files'][0]['width']) * int(scene2['files'][0]['height']):
|
|
return False
|
|
if (favorBitRateChange and swapCandidateCk == False) or (swapCandidateCk and swapBetterBitRate):
|
|
if (favorHighBitRate and int(scene2['files'][0]['bit_rate']) > int(scene1['files'][0]['bit_rate'])) or (not favorHighBitRate and int(scene2['files'][0]['bit_rate']) < int(scene1['files'][0]['bit_rate'])):
|
|
stash.Trace(f"[isBetterVideo]:[favorHighBitRate={favorHighBitRate}] Better bit rate. {scene1['files'][0]['path']}={scene1['files'][0]['bit_rate']} v.s. {scene2['files'][0]['path']}={scene2['files'][0]['bit_rate']}")
|
|
return True
|
|
if (favorCodecRanking and swapCandidateCk == False) or (swapCandidateCk and swapCodec):
|
|
scene1CodecRank = stash.indexStartsWithInList(codecRanking, scene1['files'][0]['video_codec'])
|
|
scene2CodecRank = stash.indexStartsWithInList(codecRanking, scene2['files'][0]['video_codec'])
|
|
if scene2CodecRank < scene1CodecRank:
|
|
stash.Trace(f"[isBetterVideo] Better codec. {scene1['files'][0]['path']}={scene1['files'][0]['video_codec']}:Rank={scene1CodecRank} v.s. {scene2['files'][0]['path']}={scene2['files'][0]['video_codec']}:Rank={scene2CodecRank}")
|
|
return True
|
|
if (favorFrameRateChange and swapCandidateCk == False) or (swapCandidateCk and swapBetterFrameRate):
|
|
if (favorHigherFrameRate and int(scene2['files'][0]['frame_rate']) > int(scene1['files'][0]['frame_rate'])) or (not favorHigherFrameRate and int(scene2['files'][0]['frame_rate']) < int(scene1['files'][0]['frame_rate'])):
|
|
stash.Trace(f"[isBetterVideo]:[favorHigherFrameRate={favorHigherFrameRate}] Better frame rate. {scene1['files'][0]['path']}={scene1['files'][0]['frame_rate']} v.s. {scene2['files'][0]['path']}={scene2['files'][0]['frame_rate']}")
|
|
return True
|
|
return False
|
|
|
|
def significantMoreTimeCompareToBetterVideo(scene1, scene2): # is scene2 better than scene1
|
|
if isinstance(scene1, int):
|
|
scene1 = stash.find_scene(scene1)
|
|
scene2 = stash.find_scene(scene2)
|
|
if int(scene1['files'][0]['duration']) >= int(scene2['files'][0]['duration']):
|
|
return False
|
|
if int(scene1['files'][0]['width']) * int(scene1['files'][0]['height']) > int(scene2['files'][0]['width']) * int(scene2['files'][0]['height']):
|
|
if significantTimeDiffCheck(scene1, scene2):
|
|
if tagLongDurationLowRes:
|
|
didAddTag = setTagId_withRetry(longerDurationLowerResolution, scene2, scene1, ignoreAutoTag=True)
|
|
stash.Log(f"Tagged sene2 with tag {longerDurationLowerResolution}, because scene1 is better video, but it has significant less time ({getTimeDif(int(scene1['files'][0]['duration']), int(scene2['files'][0]['duration']))}%) compare to scene2; scene1={scene1['files'][0]['path']} (ID={scene1['id']})(duration={scene1['files'][0]['duration']}); scene2={scene2['files'][0]['path']} (ID={scene2['id']}) (duration={scene1['files'][0]['duration']}); didAddTag={didAddTag}")
|
|
else:
|
|
stash.Warn(f"Scene1 is better video, but it has significant less time ({getTimeDif(int(scene1['files'][0]['duration']), int(scene2['files'][0]['duration']))}%) compare to scene2; Scene1={scene1['files'][0]['path']} (ID={scene1['id']})(duration={scene1['files'][0]['duration']}); Scene2={scene2['files'][0]['path']} (ID={scene2['id']}) (duration={scene1['files'][0]['duration']})")
|
|
return False
|
|
return True
|
|
|
|
def allThingsEqual(scene1, scene2): # If all important things are equal, return true
|
|
if int(scene1['files'][0]['duration']) != int(scene2['files'][0]['duration']):
|
|
return False
|
|
if scene1['files'][0]['width'] != scene2['files'][0]['width']:
|
|
return False
|
|
if scene1['files'][0]['height'] != scene2['files'][0]['height']:
|
|
return False
|
|
if bitRateIsImporantComp and scene1['files'][0]['bit_rate'] != scene2['files'][0]['bit_rate']:
|
|
return False
|
|
if codecIsImporantComp and scene1['files'][0]['video_codec'] != scene2['files'][0]['video_codec']:
|
|
return False
|
|
return True
|
|
|
|
def isSwapCandidate(DupFileToKeep, DupFile):
|
|
# Don't move if both are in whitelist
|
|
if stash.startsWithInList(whitelist, DupFileToKeep['files'][0]['path']) and stash.startsWithInList(whitelist, DupFile['files'][0]['path']):
|
|
return False
|
|
if swapHighRes and int(DupFileToKeep['files'][0]['width']) * int(DupFileToKeep['files'][0]['height']) > int(DupFile['files'][0]['width']) * int(DupFile['files'][0]['height']):
|
|
if not significantTimeDiffCheck(DupFileToKeep, DupFile):
|
|
return True
|
|
else:
|
|
stash.Warn(f"File '{DupFileToKeep['files'][0]['path']}' has a higher resolution than '{DupFile['files'][0]['path']}', but the duration is significantly shorter.", toAscii=True)
|
|
if swapLongLength and int(DupFileToKeep['files'][0]['duration']) > int(DupFile['files'][0]['duration']):
|
|
if int(DupFileToKeep['files'][0]['width']) >= int(DupFile['files'][0]['width']) or int(DupFileToKeep['files'][0]['height']) >= int(DupFile['files'][0]['height']):
|
|
return True
|
|
if isBetterVideo(DupFile, DupFileToKeep, swapCandidateCk=True):
|
|
if not significantTimeDiffCheck(DupFileToKeep, DupFile):
|
|
return True
|
|
else:
|
|
stash.Warn(f"File '{DupFileToKeep['files'][0]['path']}' has better codec/bit-rate than '{DupFile['files'][0]['path']}', but the duration is significantly shorter; DupFileToKeep-ID={DupFileToKeep['id']};DupFile-ID={DupFile['id']};BitRate {DupFileToKeep['files'][0]['bit_rate']} vs {DupFile['files'][0]['bit_rate']};Codec {DupFileToKeep['files'][0]['video_codec']} vs {DupFile['files'][0]['video_codec']};FrameRate {DupFileToKeep['files'][0]['frame_rate']} vs {DupFile['files'][0]['frame_rate']};", toAscii=True)
|
|
return False
|
|
|
|
dupWhitelistTagId = None
|
|
def addDupWhitelistTag():
|
|
global dupWhitelistTagId
|
|
stash.Trace(f"Adding tag duplicateWhitelistTag = {duplicateWhitelistTag}")
|
|
descp = 'Tag added to duplicate scenes which are in the whitelist. This means there are two or more duplicates in the whitelist.'
|
|
dupWhitelistTagId = stash.createTagId(duplicateWhitelistTag, descp, ignoreAutoTag=True)
|
|
stash.Trace(f"dupWhitelistTagId={dupWhitelistTagId} name={duplicateWhitelistTag}")
|
|
|
|
excludeDupFileDeleteTagId = None
|
|
def addExcludeDupTag():
|
|
global excludeDupFileDeleteTagId
|
|
stash.Trace(f"Adding tag excludeDupFileDeleteTag = {excludeDupFileDeleteTag}")
|
|
descp = 'Excludes duplicate scene from DupFileManager tagging and deletion process. A scene having this tag will not get deleted by DupFileManager'
|
|
excludeDupFileDeleteTagId = stash.createTagId(excludeDupFileDeleteTag, descp, ignoreAutoTag=True)
|
|
stash.Trace(f"dupWhitelistTagId={excludeDupFileDeleteTagId} name={excludeDupFileDeleteTag}")
|
|
|
|
def isTaggedExcluded(Scene):
|
|
for tag in Scene['tags']:
|
|
if tag['name'] == excludeDupFileDeleteTag:
|
|
return True
|
|
return False
|
|
|
|
def isWorseKeepCandidate(DupFileToKeep, Scene):
|
|
if not stash.startsWithInList(whitelist, Scene['files'][0]['path']) and stash.startsWithInList(whitelist, DupFileToKeep['files'][0]['path']):
|
|
return True
|
|
if not stash.startsWithInList(graylist, Scene['files'][0]['path']) and stash.startsWithInList(graylist, DupFileToKeep['files'][0]['path']):
|
|
return True
|
|
if not stash.startsWithInList(blacklist, DupFileToKeep['files'][0]['path']) and stash.startsWithInList(blacklist, Scene['files'][0]['path']):
|
|
return True
|
|
|
|
if stash.startsWithInList(graylist, Scene['files'][0]['path']) and stash.startsWithInList(graylist, DupFileToKeep['files'][0]['path']) and stash.indexStartsWithInList(graylist, DupFileToKeep['files'][0]['path']) < stash.indexStartsWithInList(graylist, Scene['files'][0]['path']):
|
|
return True
|
|
if stash.startsWithInList(blacklist, DupFileToKeep['files'][0]['path']) and stash.startsWithInList(blacklist, Scene['files'][0]['path']) and stash.indexStartsWithInList(blacklist, DupFileToKeep['files'][0]['path']) < stash.indexStartsWithInList(blacklist, Scene['files'][0]['path']):
|
|
return True
|
|
return False
|
|
|
|
def killScanningJobs():
|
|
try:
|
|
if killScanningPostProcess:
|
|
stash.stopJobs(1, "Scanning...")
|
|
except Exception as e:
|
|
tb = traceback.format_exc()
|
|
stash.Error(f"Exception while trying to kill scan jobs; Error: {e}\nTraceBack={tb}")
|
|
|
|
def getPath(Scene, getParent = False):
|
|
path = stash.asc2(Scene['files'][0]['path'])
|
|
path = path.replace("'", "")
|
|
path = path.replace("\\\\", "\\")
|
|
if getParent:
|
|
return pathlib.Path(path).resolve().parent
|
|
return path
|
|
|
|
def getHtmlReportTableRow(qtyResults, tagDuplicates):
|
|
htmlReportPrefix = stash.Setting('htmlReportPrefix')
|
|
htmlReportPrefix = htmlReportPrefix.replace('http://127.0.0.1:9999/graphql', stash.url)
|
|
htmlReportPrefix = htmlReportPrefix.replace('http://localhost:9999/graphql', stash.url)
|
|
if tagDuplicates == False:
|
|
htmlReportPrefix = htmlReportPrefix.replace('<td><button id="AdvanceMenu"', '<td hidden><button id="AdvanceMenu"')
|
|
htmlReportPrefix = htmlReportPrefix.replace('(QtyPlaceHolder)', f'{qtyResults}')
|
|
htmlReportPrefix = htmlReportPrefix.replace('(MatchTypePlaceHolder)', f'(Match Type = {matchPhaseDistanceText})')
|
|
htmlReportPrefix = htmlReportPrefix.replace('(DateCreatedPlaceHolder)', datetime.now().strftime("%d-%b-%Y, %H:%M:%S"))
|
|
return htmlReportPrefix
|
|
|
|
htmlReportTableData = stash.Setting('htmlReportTableData')
|
|
htmlDetailDiffTextColor = stash.Setting('htmlDetailDiffTextColor')
|
|
htmlSupperHighlight = stash.Setting('htmlSupperHighlight')
|
|
htmlLowerHighlight = stash.Setting('htmlLowerHighlight')
|
|
def getColor(Scene1, Scene2, ifScene1HigherChangeColor = False, roundUpNumber = False, qtyDiff=0):
|
|
if (Scene1 == Scene2) or (roundUpNumber and int(Scene1) == int(Scene2)):
|
|
return ""
|
|
if ifScene1HigherChangeColor and int(Scene1) > int(Scene2):
|
|
if (int(Scene1) - int(Scene2)) > qtyDiff:
|
|
return f' style="color:{htmlDetailDiffTextColor};background-color:{htmlSupperHighlight};"'
|
|
return f' style="color:{htmlDetailDiffTextColor};background-color:{htmlLowerHighlight};"'
|
|
return f' style="color:{htmlDetailDiffTextColor};"'
|
|
|
|
def getRes(Scene):
|
|
return int(Scene['files'][0]['width']) * int(Scene['files'][0]['height'])
|
|
|
|
reasonDict = {}
|
|
|
|
def logReason(DupFileToKeep, Scene, reason):
|
|
global reasonDict
|
|
reasonDict[Scene['id']] = reason
|
|
reasonDict[DupFileToKeep['id']] = reason
|
|
stash.Debug(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason={reason}")
|
|
|
|
|
|
def getSceneID(scene)
|
|
return htmlReportTableData.replace("<td", f"<td class=\"ID_{scene}\" ")
|
|
|
|
htmlReportNameFolder = f"{stash.PLUGINS_PATH}{os.sep}DupFileManager{os.sep}report"
|
|
htmlReportName = f"{htmlReportNameFolder}{os.sep}{stash.Setting('htmlReportName')}"
|
|
|
|
def mangeDupFiles(merge=False, deleteDup=False, tagDuplicates=False, deleteBlacklistOnly=False, deleteLowerResAndDuration=False):
|
|
global reasonDict
|
|
duplicateMarkForDeletion_descp = 'Tag added to duplicate scenes so-as to tag them for deletion.'
|
|
stash.Trace(f"duplicateMarkForDeletion = {duplicateMarkForDeletion}")
|
|
dupTagId = stash.createTagId(duplicateMarkForDeletion, duplicateMarkForDeletion_descp, ignoreAutoTag=True)
|
|
stash.Trace(f"dupTagId={dupTagId} name={duplicateMarkForDeletion}")
|
|
createHtmlReport = stash.Setting('createHtmlReport')
|
|
previewOrStream = "stream" if stash.Setting('streamOverPreview') else "preview"
|
|
htmlIncludeImagePreview = stash.Setting('htmlIncludeImagePreview')
|
|
htmlImagePreviewPopupSize = stash.Setting('htmlImagePreviewPopupSize')
|
|
htmlReportNameHomePage = htmlReportName
|
|
htmlReportTableRow = stash.Setting('htmlReportTableRow')
|
|
|
|
htmlReportVideoPreview = stash.Setting('htmlReportVideoPreview')
|
|
htmlHighlightTimeDiff = stash.Setting('htmlHighlightTimeDiff')
|
|
htmlReportPaginate = stash.Setting('htmlReportPaginate')
|
|
|
|
addDupWhitelistTag()
|
|
addExcludeDupTag()
|
|
|
|
QtyDupSet = 0
|
|
QtyDup = 0
|
|
QtyExactDup = 0
|
|
QtyAlmostDup = 0
|
|
QtyRealTimeDiff = 0
|
|
QtyTagForDel = 0
|
|
QtyTagForDelPaginate = 0
|
|
PaginateId = 0
|
|
QtyNewlyTag = 0
|
|
QtySkipForDel = 0
|
|
QtyExcludeForDel = 0
|
|
QtySwap = 0
|
|
QtyMerge = 0
|
|
QtyDeleted = 0
|
|
stash.Log("#########################################################################")
|
|
stash.Trace("#########################################################################")
|
|
stash.Log(f"Waiting for find_duplicate_scenes_diff to return results; matchDupDistance={matchPhaseDistanceText}; significantTimeDiff={significantTimeDiff}", printTo=LOG_STASH_N_PLUGIN)
|
|
stash.startSpinningProcessBar()
|
|
htmlFileData = " paths {screenshot sprite " + previewOrStream + "} " if createHtmlReport else ""
|
|
mergeFieldData = " code director title rating100 date studio {id} movies {movie {id} } galleries {id} performers {id} urls " if merge else ""
|
|
DupFileSets = stash.find_duplicate_scenes(matchPhaseDistance, fragment='id tags {id name} files {path width height duration size video_codec bit_rate frame_rate} details ' + mergeFieldData + htmlFileData)
|
|
stash.stopSpinningProcessBar()
|
|
qtyResults = len(DupFileSets)
|
|
stash.setProgressBarIter(qtyResults)
|
|
stash.Trace("#########################################################################")
|
|
stash.Log(f"Found {qtyResults} duplicate sets...")
|
|
fileHtmlReport = None
|
|
if createHtmlReport:
|
|
if not os.path.isdir(htmlReportNameFolder):
|
|
os.mkdir(htmlReportNameFolder)
|
|
if not os.path.isdir(htmlReportNameFolder):
|
|
stash.Error(f"Failed to create report directory {htmlReportNameFolder}.")
|
|
return
|
|
deleteLocalDupReportHtmlFiles(False)
|
|
fileHtmlReport = open(htmlReportName, "w")
|
|
fileHtmlReport.write(f"{getHtmlReportTableRow(qtyResults, tagDuplicates)}\n")
|
|
fileHtmlReport.write(f"{stash.Setting('htmlReportTable')}\n")
|
|
htmlReportTableHeader = stash.Setting('htmlReportTableHeader')
|
|
fileHtmlReport.write(f"{htmlReportTableRow}{htmlReportTableHeader}Scene</th>{htmlReportTableHeader}Duplicate to Delete</th>{htmlReportTableHeader}Scene-ToKeep</th>{htmlReportTableHeader}Duplicate to Keep</th></tr>\n")
|
|
|
|
for DupFileSet in DupFileSets:
|
|
# stash.Trace(f"DupFileSet={DupFileSet}", toAscii=True)
|
|
QtyDupSet+=1
|
|
stash.progressBar(QtyDupSet, qtyResults)
|
|
SepLine = "---------------------------"
|
|
DupFileToKeep = None
|
|
DupToCopyFrom = ""
|
|
DupFileDetailList = []
|
|
for DupFile in DupFileSet:
|
|
QtyDup+=1
|
|
Scene = DupFile
|
|
if skipIfTagged and createHtmlReport == False and duplicateMarkForDeletion in Scene['tags']:
|
|
stash.Trace(f"Skipping scene '{Scene['files'][0]['path']}' because already tagged with {duplicateMarkForDeletion}")
|
|
continue
|
|
stash.TraceOnce(f"Scene = {Scene}", toAscii=True)
|
|
DupFileDetailList = DupFileDetailList + [Scene]
|
|
if os.path.isfile(Scene['files'][0]['path']):
|
|
if DupFileToKeep != None:
|
|
if int(DupFileToKeep['files'][0]['duration']) == int(Scene['files'][0]['duration']): # Do not count fractions of a second as a difference
|
|
QtyExactDup+=1
|
|
else:
|
|
QtyAlmostDup+=1
|
|
SepLine = "***************************"
|
|
if significantTimeDiffCheck(DupFileToKeep, Scene):
|
|
QtyRealTimeDiff += 1
|
|
|
|
if int(DupFileToKeep['files'][0]['width']) * int(DupFileToKeep['files'][0]['height']) < int(Scene['files'][0]['width']) * int(Scene['files'][0]['height']):
|
|
logReason(DupFileToKeep, Scene, f"resolution: {DupFileToKeep['files'][0]['width']}x{DupFileToKeep['files'][0]['height']} < {Scene['files'][0]['width']}x{Scene['files'][0]['height']}")
|
|
DupFileToKeep = Scene
|
|
elif significantMoreTimeCompareToBetterVideo(DupFileToKeep, Scene):
|
|
if significantTimeDiffCheck(DupFileToKeep, Scene):
|
|
theReason = f"significant-duration: <b style='color:red;background-color:bright green;'>{DupFileToKeep['files'][0]['duration']} < {Scene['files'][0]['duration']}</b>"
|
|
else:
|
|
theReason = f"duration: {DupFileToKeep['files'][0]['duration']} < {Scene['files'][0]['duration']}"
|
|
reasonKeyword = "significant-duration" if significantTimeDiffCheck(DupFileToKeep, Scene) else "duration"
|
|
logReason(DupFileToKeep, Scene, theReason)
|
|
DupFileToKeep = Scene
|
|
elif isBetterVideo(DupFileToKeep, Scene):
|
|
logReason(DupFileToKeep, Scene, f"codec,bit_rate, or frame_rate: {DupFileToKeep['files'][0]['video_codec']}, {DupFileToKeep['files'][0]['bit_rate']}, {DupFileToKeep['files'][0]['frame_rate']} : {Scene['files'][0]['video_codec']}, {Scene['files'][0]['bit_rate']}, {Scene['files'][0]['frame_rate']}")
|
|
DupFileToKeep = Scene
|
|
elif stash.startsWithInList(whitelist, Scene['files'][0]['path']) and not stash.startsWithInList(whitelist, DupFileToKeep['files'][0]['path']):
|
|
logReason(DupFileToKeep, Scene, f"not whitelist vs whitelist")
|
|
DupFileToKeep = Scene
|
|
elif isTaggedExcluded(Scene) and not isTaggedExcluded(DupFileToKeep):
|
|
logReason(DupFileToKeep, Scene, f"not ExcludeTag vs ExcludeTag")
|
|
DupFileToKeep = Scene
|
|
elif allThingsEqual(DupFileToKeep, Scene):
|
|
# Only do below checks if all imporant things are equal.
|
|
if stash.startsWithInList(blacklist, DupFileToKeep['files'][0]['path']) and not stash.startsWithInList(blacklist, Scene['files'][0]['path']):
|
|
logReason(DupFileToKeep, Scene, f"blacklist vs not blacklist")
|
|
DupFileToKeep = Scene
|
|
elif stash.startsWithInList(blacklist, DupFileToKeep['files'][0]['path']) and stash.startsWithInList(blacklist, Scene['files'][0]['path']) and stash.indexStartsWithInList(blacklist, DupFileToKeep['files'][0]['path']) > stash.indexStartsWithInList(blacklist, Scene['files'][0]['path']):
|
|
logReason(DupFileToKeep, Scene, f"blacklist-index {stash.indexStartsWithInList(blacklist, DupFileToKeep['files'][0]['path'])} > {stash.indexStartsWithInList(blacklist, Scene['files'][0]['path'])}")
|
|
DupFileToKeep = Scene
|
|
elif stash.startsWithInList(graylist, Scene['files'][0]['path']) and not stash.startsWithInList(graylist, DupFileToKeep['files'][0]['path']):
|
|
logReason(DupFileToKeep, Scene, f"not graylist vs graylist")
|
|
DupFileToKeep = Scene
|
|
elif stash.startsWithInList(graylist, Scene['files'][0]['path']) and stash.startsWithInList(graylist, DupFileToKeep['files'][0]['path']) and stash.indexStartsWithInList(graylist, DupFileToKeep['files'][0]['path']) > stash.indexStartsWithInList(graylist, Scene['files'][0]['path']):
|
|
logReason(DupFileToKeep, Scene, f"graylist-index {stash.indexStartsWithInList(graylist, DupFileToKeep['files'][0]['path'])} > {stash.indexStartsWithInList(graylist, Scene['files'][0]['path'])}")
|
|
DupFileToKeep = Scene
|
|
elif favorLongerFileName and len(DupFileToKeep['files'][0]['path']) < len(Scene['files'][0]['path']) and not isWorseKeepCandidate(DupFileToKeep, Scene):
|
|
logReason(DupFileToKeep, Scene, f"path-len {len(DupFileToKeep['files'][0]['path'])} < {len(Scene['files'][0]['path'])}")
|
|
DupFileToKeep = Scene
|
|
elif favorLargerFileSize and int(DupFileToKeep['files'][0]['size']) < int(Scene['files'][0]['size']) and not isWorseKeepCandidate(DupFileToKeep, Scene):
|
|
logReason(DupFileToKeep, Scene, f"size {DupFileToKeep['files'][0]['size']} < {Scene['files'][0]['size']}")
|
|
DupFileToKeep = Scene
|
|
elif not favorLongerFileName and len(DupFileToKeep['files'][0]['path']) > len(Scene['files'][0]['path']) and not isWorseKeepCandidate(DupFileToKeep, Scene):
|
|
logReason(DupFileToKeep, Scene, f"path-len {len(DupFileToKeep['files'][0]['path'])} > {len(Scene['files'][0]['path'])}")
|
|
DupFileToKeep = Scene
|
|
elif not favorLargerFileSize and int(DupFileToKeep['files'][0]['size']) > int(Scene['files'][0]['size']) and not isWorseKeepCandidate(DupFileToKeep, Scene):
|
|
logReason(DupFileToKeep, Scene, f"size {DupFileToKeep['files'][0]['size']} > {Scene['files'][0]['size']}")
|
|
DupFileToKeep = Scene
|
|
else:
|
|
DupFileToKeep = Scene
|
|
# stash.Trace(f"DupFileToKeep = {DupFileToKeep}")
|
|
stash.Debug(f"KeepID={DupFileToKeep['id']}, ID={DupFile['id']} duration=({Scene['files'][0]['duration']}), Size=({Scene['files'][0]['size']}), Res=({Scene['files'][0]['width']} x {Scene['files'][0]['height']}) Name={Scene['files'][0]['path']}, KeepPath={DupFileToKeep['files'][0]['path']}", toAscii=True)
|
|
else:
|
|
stash.Error(f"Scene does NOT exist; path={Scene['files'][0]['path']}; ID={Scene['id']}")
|
|
|
|
for DupFile in DupFileDetailList:
|
|
if DupFileToKeep != None and DupFile['id'] != DupFileToKeep['id']:
|
|
if merge:
|
|
result = stash.mergeMetadata(DupFile, DupFileToKeep)
|
|
if result != "Nothing To Merge":
|
|
QtyMerge += 1
|
|
didAddTag = False
|
|
if stash.startsWithInList(whitelist, DupFile['files'][0]['path']) and (not whitelistDelDupInSameFolder or not hasSameDir(DupFile['files'][0]['path'], DupFileToKeep['files'][0]['path'])):
|
|
QtySkipForDel+=1
|
|
if isSwapCandidate(DupFileToKeep, DupFile):
|
|
if merge:
|
|
stash.mergeMetadata(DupFileToKeep, DupFile)
|
|
if toRecycleBeforeSwap:
|
|
sendToTrash(DupFile['files'][0]['path'])
|
|
stash.Log(f"Moving better file '{DupFileToKeep['files'][0]['path']}' to '{DupFile['files'][0]['path']}'; SrcID={DupFileToKeep['id']};DescID={DupFile['id']};QtyDup={QtyDup};Set={QtyDupSet} of {qtyResults};QtySwap={QtySwap};QtySkipForDel={QtySkipForDel}", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
|
|
try:
|
|
shutil.move(DupFileToKeep['files'][0]['path'], DupFile['files'][0]['path'])
|
|
QtySwap+=1
|
|
except Exception as e:
|
|
tb = traceback.format_exc()
|
|
stash.Error(f"Exception while moving file '{DupFileToKeep['files'][0]['path']}' to '{DupFile['files'][0]['path']}; SrcID={DupFileToKeep['id']};DescID={DupFile['id']}'; Error: {e}\nTraceBack={tb}")
|
|
DupFileToKeep = DupFile
|
|
else:
|
|
if dupWhitelistTagId and tagDuplicates:
|
|
didAddTag = setTagId_withRetry(duplicateWhitelistTag, DupFile, DupFileToKeep, ignoreAutoTag=True)
|
|
stash.Log(f"NOT processing duplicate, because it's in whitelist. '{DupFile['files'][0]['path']}';AddTagW={didAddTag};QtyDup={QtyDup};Set={QtyDupSet} of {qtyResults};QtySkipForDel={QtySkipForDel}", toAscii=True)
|
|
else:
|
|
if isTaggedExcluded(DupFile):
|
|
QtyExcludeForDel+=1
|
|
stash.Log(f"Excluding file {DupFile['files'][0]['path']} because tagged for exclusion via tag {excludeDupFileDeleteTag};QtyDup={QtyDup};Set={QtyDupSet} of {qtyResults}")
|
|
else:
|
|
# ToDo: Add merge logic here
|
|
if deleteDup:
|
|
DupFileName = DupFile['files'][0]['path']
|
|
if not deleteBlacklistOnly or stash.startsWithInList(blacklist, DupFile['files'][0]['path']):
|
|
if not deleteLowerResAndDuration or (isBetterVideo(DupFile, DupFileToKeep) and not significantMoreTimeCompareToBetterVideo(DupFileToKeep, DupFile)) or (significantMoreTimeCompareToBetterVideo(DupFile, DupFileToKeep) and not isBetterVideo(DupFileToKeep, DupFile)):
|
|
QtyDeleted += 1
|
|
DupFileNameOnly = pathlib.Path(DupFileName).stem
|
|
stash.Warn(f"Deleting duplicate '{DupFileName}';QtyDup={QtyDup};Set={QtyDupSet} of {qtyResults};QtyDeleted={QtyDeleted}", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
|
|
if alternateTrashCanPath != "":
|
|
destPath = f"{alternateTrashCanPath }{os.sep}{DupFileNameOnly}"
|
|
if os.path.isfile(destPath):
|
|
destPath = f"{alternateTrashCanPath }{os.sep}_{time.time()}_{DupFileNameOnly}"
|
|
shutil.move(DupFileName, destPath)
|
|
elif moveToTrashCan:
|
|
sendToTrash(DupFileName)
|
|
stash.destroyScene(DupFile['id'], delete_file=True)
|
|
elif tagDuplicates or fileHtmlReport != None:
|
|
if excludeFromReportIfSignificantTimeDiff and significantTimeDiffCheck(DupFile, DupFileToKeep, True):
|
|
stash.Log(f"Skipping duplicate {DupFile['files'][0]['path']} (ID={DupFile['id']}), because of time difference greater than {significantTimeDiff} for file {DupFileToKeep['files'][0]['path']}.")
|
|
continue
|
|
QtyTagForDel+=1
|
|
QtyTagForDelPaginate+=1
|
|
didAddTag = False
|
|
if tagDuplicates:
|
|
didAddTag = setTagId_withRetry(duplicateMarkForDeletion, DupFile, DupFileToKeep, ignoreAutoTag=True)
|
|
if fileHtmlReport != None:
|
|
# ToDo: Add icons using github path
|
|
# add copy button with copy icon
|
|
# add move button with r-sqr icon
|
|
# repace delete button with trashcan icon
|
|
# add rename file code and button
|
|
# add delete only from stash db code and button using DB delete icon
|
|
stash.Debug(f"Adding scene {DupFile['id']} to HTML report.")
|
|
dupFileExist = True if os.path.isfile(DupFile['files'][0]['path']) else False
|
|
toKeepFileExist = True if os.path.isfile(DupFileToKeep['files'][0]['path']) else False
|
|
|
|
fileHtmlReport.write(f"{htmlReportTableRow}")
|
|
videoPreview = f"<video {htmlReportVideoPreview} poster=\"{DupFile['paths']['screenshot']}\"><source src=\"{DupFile['paths'][previewOrStream]}\" type=\"video/mp4\"></video>"
|
|
if htmlIncludeImagePreview:
|
|
imagePreview = f"<ul><li><img src=\"{DupFile['paths']['sprite']}\" alt=\"\" width=\"140\"><span class=\"large\"><img src=\"{DupFile['paths']['sprite']}\" class=\"large-image\" alt=\"\" width=\"{htmlImagePreviewPopupSize}\"></span></li></ul>"
|
|
fileHtmlReport.write(f"{getSceneID(DupFile['id'])}<table><tr><td>{videoPreview}</td><td>{imagePreview}</td></tr></table></td>")
|
|
else:
|
|
fileHtmlReport.write(f"{getSceneID(DupFile['id'])}{videoPreview}</td>")
|
|
fileHtmlReport.write(f"{getSceneID(DupFile['id'])}<a href=\"{stash.STASH_URL}/scenes/{DupFile['id']}\" target=\"_blank\" rel=\"noopener noreferrer\">{getPath(DupFile)}</a>")
|
|
fileHtmlReport.write(f"<p><table><tr class=\"scene-details\"><th>Res</th><th>Durration</th><th>BitRate</th><th>Codec</th><th>FrameRate</th><th>size</th><th>ID</th><th>index</th></tr>")
|
|
fileHtmlReport.write(f"<tr class=\"scene-details\"><td {getColor(getRes(DupFile), getRes(DupFileToKeep), True)}>{DupFile['files'][0]['width']}x{DupFile['files'][0]['height']}</td><td {getColor(DupFile['files'][0]['duration'], DupFileToKeep['files'][0]['duration'], True, True, htmlHighlightTimeDiff)}>{DupFile['files'][0]['duration']}</td><td {getColor(DupFile['files'][0]['bit_rate'], DupFileToKeep['files'][0]['bit_rate'])}>{DupFile['files'][0]['bit_rate']}</td><td {getColor(DupFile['files'][0]['video_codec'], DupFileToKeep['files'][0]['video_codec'])}>{DupFile['files'][0]['video_codec']}</td><td {getColor(DupFile['files'][0]['frame_rate'], DupFileToKeep['files'][0]['frame_rate'])}>{DupFile['files'][0]['frame_rate']}</td><td {getColor(DupFile['files'][0]['size'], DupFileToKeep['files'][0]['size'])}>{DupFile['files'][0]['size']}</td><td>{DupFile['id']}</td><td>{QtyTagForDel}</td></tr>")
|
|
|
|
if DupFile['id'] in reasonDict:
|
|
fileHtmlReport.write(f"<tr class=\"reason-details\"><td colspan='8'>Reason: {reasonDict[DupFile['id']]}</td></tr>")
|
|
# elif DupFileToKeep['id'] in reasonDict:
|
|
# fileHtmlReport.write(f"<tr class=\"reason-details\"><td colspan='8'>Reason: {reasonDict[DupFileToKeep['id']]}</td></tr>")
|
|
elif int(DupFileToKeep['files'][0]['width']) * int(DupFileToKeep['files'][0]['height']) > int(DupFile['files'][0]['width']) * int(DupFile['files'][0]['height']):
|
|
fileHtmlReport.write(f"<tr class=\"reason-details\"><td colspan='8'>Reason: Resolution {DupFile['files'][0]['width']}x{DupFile['files'][0]['height']} < {DupFileToKeep['files'][0]['width']}x{DupFileToKeep['files'][0]['height']}</td></tr>")
|
|
elif significantMoreTimeCompareToBetterVideo(DupFile, DupFileToKeep):
|
|
if significantTimeDiffCheck(DupFile, DupFileToKeep):
|
|
theReason = f"Significant-Duration: <b style='color:red;background-color:neon green;'>{DupFile['files'][0]['duration']} < {DupFileToKeep['files'][0]['duration']}</b>"
|
|
else:
|
|
theReason = f"Duration: {DupFile['files'][0]['duration']} < {DupFileToKeep['files'][0]['duration']}"
|
|
fileHtmlReport.write(f"<tr class=\"reason-details\"><td colspan='8'>Reason: {theReason}</td></tr>")
|
|
elif isBetterVideo(DupFile, DupFileToKeep):
|
|
fileHtmlReport.write(f"<tr class=\"reason-details\"><td colspan='8'>Reason: Better Video</td></tr>")
|
|
elif stash.startsWithInList(DupFileToKeep, DupFile['files'][0]['path']) and not stash.startsWithInList(whitelist, DupFile['files'][0]['path']):
|
|
fileHtmlReport.write(f"<tr class=\"reason-details\"><td colspan='8'>Reason: not whitelist vs whitelist</td></tr>")
|
|
elif isTaggedExcluded(DupFileToKeep) and not isTaggedExcluded(DupFile):
|
|
fileHtmlReport.write(f"<tr class=\"reason-details\"><td colspan='8'>Reason: not ExcludeTag vs ExcludeTag</td></tr>")
|
|
|
|
fileHtmlReport.write("</table>")
|
|
fileHtmlReport.write(f"<button class=\"link-button\" title=\"Delete file and remove scene from stash\" value=\"deleteScene\" id=\"{DupFile['id']}\">[Delete]</button>")
|
|
fileHtmlReport.write(f"<button class=\"link-button\" title=\"Remove scene from stash only. Do NOT delete file.\" value=\"removeScene\" id=\"{DupFile['id']}\">[Remove]</button>")
|
|
fileHtmlReport.write(f"<button class=\"link-button\" title=\"Copy duplicate to file-to-keep.\" value=\"copyScene\" id=\"{DupFile['id']}:{DupFileToKeep['id']}\">[Copy]</button>")
|
|
fileHtmlReport.write(f"<button class=\"link-button\" title=\"Replace file-to-keep with this duplicate, and copy metadata from this duplicate to file-to-keep.\" value=\"moveScene\" id=\"{DupFile['id']}:{DupFileToKeep['id']}\">[Move]</button>")
|
|
fileHtmlReport.write(f"<button class=\"link-button\" title=\"Replace file-to-keep file name with this duplicate file name.\" value=\"renameScene\" id=\"{DupFileToKeep['id']}:{pathlib.Path(DupFile['file']['path']).stem}\">[CpyName]</button>")
|
|
# ToDo: Add following buttons:
|
|
# rename file
|
|
if dupFileExist and tagDuplicates:
|
|
fileHtmlReport.write(f"<button class=\"link-button\" title=\"Remove duplicate tag from scene.\" value=\"removeDupTag\" id=\"{DupFile['id']}\">[-Tag]</button>")
|
|
fileHtmlReport.write(f"<button class=\"link-button\" title=\"Add exclude tag to scene. This will exclude scene from deletion via deletion tag\" value=\"addExcludeTag\" id=\"{DupFile['id']}\">[+Exclude]</button>")
|
|
fileHtmlReport.write(f"<button class=\"link-button\" title=\"Merge duplicate scene tags with ToKeep scene tags\" value=\"mergeTags\" id=\"{DupFile['id']}:{DupFileToKeep['id']}\">[Merge Tags]</button>")
|
|
if dupFileExist:
|
|
fileHtmlReport.write(f"<a class=\"link-items\" title=\"Open folder\" href=\"file://{getPath(DupFile, True)}\">[Folder]</a>")
|
|
fileHtmlReport.write(f"<a class=\"link-items\" title=\"Play file locally\" href=\"file://{getPath(DupFile)}\">[Play]</a>")
|
|
else:
|
|
fileHtmlReport.write("<b style='color:red;'>[File NOT Exist]<b>")
|
|
fileHtmlReport.write("</p></td>")
|
|
|
|
videoPreview = f"<video {htmlReportVideoPreview} poster=\"{DupFileToKeep['paths']['screenshot']}\"><source src=\"{DupFileToKeep['paths'][previewOrStream]}\" type=\"video/mp4\"></video>"
|
|
if htmlIncludeImagePreview:
|
|
imagePreview = f"<ul><li><img src=\"{DupFileToKeep['paths']['sprite']}\" alt=\"\" width=\"140\"><span class=\"large\"><img src=\"{DupFileToKeep['paths']['sprite']}\" class=\"large-image\" alt=\"\" width=\"{htmlImagePreviewPopupSize}\"></span></li></ul>"
|
|
fileHtmlReport.write(f"{getSceneID(DupFileToKeep['id'])}<table><tr><td>{videoPreview}</td><td>{imagePreview}</td></tr></table></td>")
|
|
else:
|
|
fileHtmlReport.write(f"{getSceneID(DupFileToKeep['id'])}{videoPreview}</td>")
|
|
fileHtmlReport.write(f"{getSceneID(DupFileToKeep['id'])}<a href=\"{stash.STASH_URL}/scenes/{DupFileToKeep['id']}\" target=\"_blank\" rel=\"noopener noreferrer\">{getPath(DupFileToKeep)}</a>")
|
|
fileHtmlReport.write(f"<p><table><tr class=\"scene-details\"><th>Res</th><th>Durration</th><th>BitRate</th><th>Codec</th><th>FrameRate</th><th>size</th><th>ID</th></tr>")
|
|
fileHtmlReport.write(f"<tr class=\"scene-details\"><td>{DupFileToKeep['files'][0]['width']}x{DupFileToKeep['files'][0]['height']}</td><td>{DupFileToKeep['files'][0]['duration']}</td><td>{DupFileToKeep['files'][0]['bit_rate']}</td><td>{DupFileToKeep['files'][0]['video_codec']}</td><td>{DupFileToKeep['files'][0]['frame_rate']}</td><td>{DupFileToKeep['files'][0]['size']}</td><td>{DupFileToKeep['id']}</td></tr></table>")
|
|
fileHtmlReport.write(f"<button class=\"link-button\" title=\"Delete [DupFileToKeep] and remove scene from stash\" value=\"deleteScene\" id=\"{DupFileToKeep['id']}\">[Delete]</button>")
|
|
fileHtmlReport.write(f"<button class=\"link-button\" title=\"Remove scene from stash only. Do NOT delete file.\" value=\"removeScene\" id=\"{DupFileToKeep['id']}\">[Remove]</button>")
|
|
fileHtmlReport.write(f"<button class=\"link-button\" title=\"Rename file-to-keep.\" value=\"newName\" id=\"{DupFileToKeep['id']}:{pathlib.Path(DupFileToKeep['file']['path']).stem}\">[Rename]</button>")
|
|
if isTaggedExcluded(DupFileToKeep):
|
|
fileHtmlReport.write(f"<button class=\"link-button\" title=\"Remove exclude scene from deletion tag\" value=\"removeExcludeTag\" id=\"{DupFileToKeep['id']}\">[-Exclude]</button>")
|
|
fileHtmlReport.write(f"<a class=\"link-items\" title=\"Open folder\" href=\"file://{getPath(DupFileToKeep, True)}\">[Folder]</a>")
|
|
if toKeepFileExist:
|
|
fileHtmlReport.write(f"<a class=\"link-items\" title=\"Play file locally\" href=\"file://{getPath(DupFileToKeep)}\">[Play]</a>")
|
|
else:
|
|
fileHtmlReport.write("<b style='color:red;'>[File NOT Exist]<b>")
|
|
# ToDo: Add following buttons:
|
|
# rename file
|
|
fileHtmlReport.write(f"</p></td>")
|
|
|
|
fileHtmlReport.write("</tr>\n")
|
|
|
|
if QtyTagForDelPaginate >= htmlReportPaginate:
|
|
QtyTagForDelPaginate = 0
|
|
fileHtmlReport.write("</table>\n")
|
|
homeHtmReportLink = f"<a class=\"link-items\" title=\"Home Page\" href=\"file://{htmlReportNameHomePage}\">[Home]</a>"
|
|
prevHtmReportLink = ""
|
|
if PaginateId > 0:
|
|
if PaginateId > 1:
|
|
prevHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId-1}.html")
|
|
else:
|
|
prevHtmReport = htmlReportNameHomePage
|
|
prevHtmReportLink = f"<a class=\"link-items\" title=\"Previous Page\" href=\"file://{prevHtmReport}\">[Prev]</a>"
|
|
nextHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId+1}.html")
|
|
nextHtmReportLink = f"<a class=\"link-items\" title=\"Next Page\" href=\"file://{nextHtmReport}\">[Next]</a>"
|
|
fileHtmlReport.write(f"<center><table><tr><td>{homeHtmReportLink}</td><td>{prevHtmReportLink}</td><td>{nextHtmReportLink}</td></tr></table></center>")
|
|
fileHtmlReport.write(f"{stash.Setting('htmlReportPostfix')}")
|
|
fileHtmlReport.close()
|
|
PaginateId+=1
|
|
fileHtmlReport = open(nextHtmReport, "w")
|
|
fileHtmlReport.write(f"{getHtmlReportTableRow(qtyResults, tagDuplicates)}\n")
|
|
if PaginateId > 1:
|
|
prevHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId-1}.html")
|
|
else:
|
|
prevHtmReport = htmlReportNameHomePage
|
|
prevHtmReportLink = f"<a class=\"link-items\" title=\"Previous Page\" href=\"file://{prevHtmReport}\">[Prev]</a>"
|
|
if len(DupFileSets) > (QtyTagForDel + htmlReportPaginate):
|
|
nextHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId+1}.html")
|
|
nextHtmReportLink = f"<a class=\"link-items\" title=\"Next Page\" href=\"file://{nextHtmReport}\">[Next]</a>"
|
|
fileHtmlReport.write(f"<center><table><tr><td>{homeHtmReportLink}</td><td>{prevHtmReportLink}</td><td>{nextHtmReportLink}</td></tr></table></center>")
|
|
else:
|
|
stash.Debug(f"DupFileSets Qty = {len(DupFileSets)}; DupFileDetailList Qty = {len(DupFileDetailList)}; QtyTagForDel = {QtyTagForDel}; htmlReportPaginate = {htmlReportPaginate}; QtyTagForDel + htmlReportPaginate = {QtyTagForDel+htmlReportPaginate}")
|
|
fileHtmlReport.write(f"<center><table><tr><td>{homeHtmReportLink}</td><td>{prevHtmReportLink}</td></tr></table></center>")
|
|
fileHtmlReport.write(f"{stash.Setting('htmlReportTable')}\n")
|
|
fileHtmlReport.write(f"{htmlReportTableRow}{htmlReportTableHeader}Scene</th>{htmlReportTableHeader}Duplicate to Delete</th>{htmlReportTableHeader}Scene-ToKeep</th>{htmlReportTableHeader}Duplicate to Keep</th></tr>\n")
|
|
|
|
if tagDuplicates and graylistTagging and stash.startsWithInList(graylist, DupFile['files'][0]['path']):
|
|
stash.addTag(DupFile, graylistMarkForDeletion, ignoreAutoTag=True)
|
|
if didAddTag:
|
|
QtyNewlyTag+=1
|
|
if QtyTagForDel == 1:
|
|
stash.Log(f"Tagging duplicate {DupFile['files'][0]['path']} for deletion with tag {duplicateMarkForDeletion}", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
|
|
else:
|
|
didAddTag = 1 if didAddTag else 0
|
|
stash.Log(f"Tagging duplicate {DupFile['files'][0]['path']} for deletion;AddTag={didAddTag};Qty={QtyDup};Set={QtyDupSet} of {qtyResults};NewlyTag={QtyNewlyTag};isTag={QtyTagForDel}", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
|
|
stash.Trace(SepLine)
|
|
if maxDupToProcess > 0 and ((QtyTagForDel > maxDupToProcess) or (QtyTagForDel == 0 and QtyDup > maxDupToProcess)):
|
|
break
|
|
|
|
if fileHtmlReport != None:
|
|
fileHtmlReport.write("</table>\n")
|
|
if PaginateId > 0:
|
|
homeHtmReportLink = f"<a class=\"link-items\" title=\"Home Page\" href=\"file://{htmlReportNameHomePage}\">[Home]</a>"
|
|
if PaginateId > 1:
|
|
prevHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId-1}.html")
|
|
else:
|
|
prevHtmReport = htmlReportNameHomePage
|
|
prevHtmReportLink = f"<a class=\"link-items\" title=\"Previous Page\" href=\"file://{prevHtmReport}\">[Prev]</a>"
|
|
fileHtmlReport.write(f"<center><table><tr><td>{homeHtmReportLink}</td><td>{prevHtmReportLink}</td></tr></table></center>")
|
|
fileHtmlReport.write(f"<h2>Total Tagged for Deletion {QtyTagForDel}</h2>\n")
|
|
fileHtmlReport.write(f"{stash.Setting('htmlReportPostfix')}")
|
|
fileHtmlReport.close()
|
|
stash.Log(f"************************************************************", printTo = stash.LogTo.STASH)
|
|
stash.Log(f"************************************************************", printTo = stash.LogTo.STASH)
|
|
stash.Log(f"View Stash duplicate report using Stash->Settings->Tools->[Duplicate File Report]", printTo = stash.LogTo.STASH)
|
|
stash.Log(f"************************************************************", printTo = stash.LogTo.STASH)
|
|
stash.Log(f"************************************************************", printTo = stash.LogTo.STASH)
|
|
|
|
|
|
stash.Debug("#####################################################")
|
|
stash.Log(f"QtyDupSet={QtyDupSet}, QtyDup={QtyDup}, QtyDeleted={QtyDeleted}, QtySwap={QtySwap}, QtyTagForDel={QtyTagForDel}, QtySkipForDel={QtySkipForDel}, QtyExcludeForDel={QtyExcludeForDel}, QtyExactDup={QtyExactDup}, QtyAlmostDup={QtyAlmostDup}, QtyMerge={QtyMerge}, QtyRealTimeDiff={QtyRealTimeDiff}", printTo=LOG_STASH_N_PLUGIN)
|
|
killScanningJobs()
|
|
if cleanAfterDel and deleteDup:
|
|
stash.Log("Adding clean jobs to the Task Queue", printTo=LOG_STASH_N_PLUGIN)
|
|
stash.metadata_clean()
|
|
stash.metadata_clean_generated()
|
|
stash.optimise_database()
|
|
if doGeneratePhash:
|
|
stash.metadata_generate({"phashes": True})
|
|
sys.stdout.write("Report complete")
|
|
|
|
def findCurrentTagId(tagNames):
|
|
# tagNames = [i for n, i in enumerate(tagNames) if i not in tagNames[:n]]
|
|
for tagName in tagNames:
|
|
tagId = stash.find_tags(q=tagName)
|
|
if len(tagId) > 0 and 'id' in tagId[0]:
|
|
stash.Debug(f"Using tag name {tagName} with Tag ID {tagId[0]['id']}")
|
|
return tagId[0]['id']
|
|
return "-1"
|
|
|
|
def toJson(data):
|
|
import json
|
|
# data = data.replace("'", '"')
|
|
data = data.replace("\\", "\\\\")
|
|
data = data.replace("\\\\\\\\", "\\\\")
|
|
return json.loads(data)
|
|
|
|
def getAnAdvanceMenuOptionSelected(taskName, target, isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater):
|
|
stash.Log(f"Processing taskName = {taskName}, target = {target}")
|
|
if "Blacklist" in taskName:
|
|
isBlackList = True
|
|
if "Less" in taskName:
|
|
compareToLess = True
|
|
if "Greater" in taskName:
|
|
compareToGreater = True
|
|
|
|
if "pathToDelete" in taskName:
|
|
pathToDelete = target.lower()
|
|
elif "sizeToDelete" in taskName:
|
|
sizeToDelete = int(target)
|
|
elif "durationToDelete" in taskName:
|
|
durationToDelete = int(target)
|
|
elif "commonResToDelete" in taskName:
|
|
resolutionToDelete = int(target)
|
|
elif "resolutionToDelete" in taskName:
|
|
resolutionToDelete = int(target)
|
|
elif "ratingToDelete" in taskName:
|
|
ratingToDelete = int(target) * 20
|
|
elif "tagToDelete" in taskName:
|
|
tagToDelete = target.lower()
|
|
elif "titleToDelete" in taskName:
|
|
titleToDelete = target.lower()
|
|
elif "pathStrToDelete" in taskName:
|
|
pathStrToDelete = target.lower()
|
|
elif "fileNotExistToDelete" in taskName:
|
|
fileNotExistToDelete = True
|
|
return isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater
|
|
|
|
def getAdvanceMenuOptionSelected(advanceMenuOptionSelected):
|
|
isBlackList = False
|
|
pathToDelete = ""
|
|
sizeToDelete = -1
|
|
durationToDelete = -1
|
|
resolutionToDelete = -1
|
|
ratingToDelete = -1
|
|
tagToDelete = ""
|
|
titleToDelete = ""
|
|
pathStrToDelete = ""
|
|
fileNotExistToDelete = False
|
|
compareToLess = False
|
|
compareToGreater = False
|
|
if advanceMenuOptionSelected:
|
|
if 'Target' in stash.JSON_INPUT['args']:
|
|
if "applyCombo" in stash.PLUGIN_TASK_NAME:
|
|
jsonObject = toJson(stash.JSON_INPUT['args']['Target'])
|
|
for taskName in jsonObject:
|
|
isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater = getAnAdvanceMenuOptionSelected(taskName, jsonObject[taskName], isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, compareToLess, compareToGreater)
|
|
else:
|
|
return getAnAdvanceMenuOptionSelected(stash.PLUGIN_TASK_NAME, stash.JSON_INPUT['args']['Target'], isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, compareToLess, compareToGreater)
|
|
return isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater
|
|
|
|
# //////////////////////////////////////////////////////////////////////////////
|
|
# //////////////////////////////////////////////////////////////////////////////
|
|
def manageTagggedDuplicates(deleteScenes=False, clearTag=False, setGrayListTag=False, tagId=-1, advanceMenuOptionSelected=False):
|
|
if tagId == -1:
|
|
tagId = findCurrentTagId([duplicateMarkForDeletion, base1_duplicateMarkForDeletion, base2_duplicateMarkForDeletion, 'DuplicateMarkForDeletion', '_DuplicateMarkForDeletion'])
|
|
if int(tagId) < 0:
|
|
stash.Warn(f"Could not find tag ID for tag '{duplicateMarkForDeletion}'.")
|
|
return
|
|
|
|
excludedTags = [duplicateMarkForDeletion]
|
|
if clearAllDupfileManagerTags:
|
|
excludedTags = [duplicateMarkForDeletion, duplicateWhitelistTag, excludeDupFileDeleteTag, graylistMarkForDeletion, longerDurationLowerResolution]
|
|
|
|
isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater = getAdvanceMenuOptionSelected(advanceMenuOptionSelected)
|
|
if advanceMenuOptionSelected and deleteScenes and pathToDelete == "" and tagToDelete == "" and titleToDelete == "" and pathStrToDelete == "" and sizeToDelete == -1 and durationToDelete == -1 and resolutionToDelete == -1 and ratingToDelete == -1 and fileNotExistToDelete == False:
|
|
stash.Error("Running advance menu option with no options enabled.")
|
|
return
|
|
|
|
QtyDup = 0
|
|
QtyDeleted = 0
|
|
QtyClearedTags = 0
|
|
QtySetGraylistTag = 0
|
|
QtyFailedQuery = 0
|
|
stash.Debug("#########################################################################")
|
|
stash.startSpinningProcessBar()
|
|
scenes = stash.find_scenes(f={"tags": {"value":tagId, "modifier":"INCLUDES"}}, fragment='id tags {id name} files {path width height duration size video_codec bit_rate frame_rate} details title rating100')
|
|
stash.stopSpinningProcessBar()
|
|
qtyResults = len(scenes)
|
|
stash.Log(f"Found {qtyResults} scenes with tag ({duplicateMarkForDeletion})")
|
|
stash.setProgressBarIter(qtyResults)
|
|
for scene in scenes:
|
|
QtyDup += 1
|
|
stash.progressBar(QtyDup, qtyResults)
|
|
# scene = stash.find_scene(sceneID['id'])
|
|
# if scene == None or len(scene) == 0:
|
|
# stash.Warn(f"Could not get scene data for scene ID {scene['id']}.")
|
|
# QtyFailedQuery += 1
|
|
# continue
|
|
# stash.Trace(f"scene={scene}")
|
|
if clearTag:
|
|
QtyClearedTags += 1
|
|
# ToDo: Add logic to exclude graylistMarkForDeletion
|
|
tags = [int(item['id']) for item in scene["tags"] if item['name'] not in excludedTags]
|
|
# if clearAllDupfileManagerTags:
|
|
# tags = []
|
|
# for tag in scene["tags"]:
|
|
# if tag['name'] in excludedTags:
|
|
# continue
|
|
# tags += [int(tag['id'])]
|
|
stash.TraceOnce(f"tagId={tagId}, len={len(tags)}, tags = {tags}")
|
|
dataDict = {'id' : scene['id']}
|
|
if addPrimaryDupPathToDetails:
|
|
sceneDetails = scene['details']
|
|
if sceneDetails.find(detailPrefix) == 0 and sceneDetails.find(detailPostfix) > 1:
|
|
Pos1 = sceneDetails.find(detailPrefix)
|
|
Pos2 = sceneDetails.find(detailPostfix)
|
|
sceneDetails = sceneDetails[0:Pos1] + sceneDetails[Pos2 + len(detailPostfix):]
|
|
dataDict.update({'details' : sceneDetails})
|
|
dataDict.update({'tag_ids' : tags})
|
|
stash.Log(f"Updating scene with {dataDict};QtyClearedTags={QtyClearedTags};Count={QtyDup} of {qtyResults}")
|
|
stash.updateScene(dataDict)
|
|
# stash.removeTag(scene, duplicateMarkForDeletion)
|
|
elif setGrayListTag:
|
|
if stash.startsWithInList(graylist, scene['files'][0]['path']):
|
|
QtySetGraylistTag+=1
|
|
if stash.addTag(scene, graylistMarkForDeletion, ignoreAutoTag=True):
|
|
stash.Log(f"Added tag {graylistMarkForDeletion} to scene {scene['files'][0]['path']};QtySetGraylistTag={QtySetGraylistTag};Count={QtyDup} of {qtyResults}")
|
|
else:
|
|
stash.Trace(f"Scene already had tag {graylistMarkForDeletion}; {scene['files'][0]['path']}")
|
|
elif deleteScenes:
|
|
DupFileName = scene['files'][0]['path']
|
|
DupFileNameOnly = pathlib.Path(DupFileName).stem
|
|
if advanceMenuOptionSelected:
|
|
if isBlackList:
|
|
if not stash.startsWithInList(blacklist, scene['files'][0]['path']):
|
|
continue
|
|
if pathToDelete != "":
|
|
if not DupFileName.lower().startswith(pathToDelete):
|
|
stash.Debug(f"Skipping file {DupFileName} because it does not start with {pathToDelete}.")
|
|
continue
|
|
if pathStrToDelete != "":
|
|
if not pathStrToDelete in DupFileName.lower():
|
|
stash.Debug(f"Skipping file {DupFileName} because it does not contain value {pathStrToDelete}.")
|
|
continue
|
|
if sizeToDelete != -1:
|
|
compareTo = int(scene['files'][0]['size'])
|
|
if compareToLess:
|
|
if not (compareTo < sizeToDelete):
|
|
continue
|
|
elif compareToGreater:
|
|
if not (compareTo > sizeToDelete):
|
|
continue
|
|
else:
|
|
if not compareTo == sizeToDelete:
|
|
continue
|
|
if durationToDelete != -1:
|
|
compareTo = int(scene['files'][0]['duration'])
|
|
if compareToLess:
|
|
if not (compareTo < durationToDelete):
|
|
continue
|
|
elif compareToGreater:
|
|
if not (compareTo > durationToDelete):
|
|
continue
|
|
else:
|
|
if not compareTo == durationToDelete:
|
|
continue
|
|
if resolutionToDelete != -1:
|
|
compareTo = int(scene['files'][0]['width']) * int(scene['files'][0]['height'])
|
|
if compareToLess:
|
|
if not (compareTo < resolutionToDelete):
|
|
continue
|
|
elif compareToGreater:
|
|
if not (compareTo > resolutionToDelete):
|
|
continue
|
|
else:
|
|
if not compareTo == resolutionToDelete:
|
|
continue
|
|
if ratingToDelete != -1:
|
|
if scene['rating100'] == "None":
|
|
compareTo = 0
|
|
else:
|
|
compareTo = int(scene['rating100'])
|
|
if compareToLess:
|
|
if not (compareTo < resolutionToDelete):
|
|
continue
|
|
elif compareToGreater:
|
|
if not (compareTo > resolutionToDelete):
|
|
continue
|
|
else:
|
|
if not compareTo == resolutionToDelete:
|
|
continue
|
|
if titleToDelete != "":
|
|
if not titleToDelete in scene['title'].lower():
|
|
stash.Debug(f"Skipping file {DupFileName} because it does not contain value {titleToDelete} in title ({scene['title']}).")
|
|
continue
|
|
if tagToDelete != "":
|
|
doProcessThis = False
|
|
for tag in scene['tags']:
|
|
if tag['name'].lower() == tagToDelete:
|
|
doProcessThis = True
|
|
break
|
|
if doProcessThis == False:
|
|
continue
|
|
if fileNotExistToDelete:
|
|
if os.path.isfile(scene['files'][0]['path']):
|
|
continue
|
|
stash.Warn(f"Deleting duplicate '{DupFileName}'", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
|
|
if alternateTrashCanPath != "":
|
|
destPath = f"{alternateTrashCanPath }{os.sep}{DupFileNameOnly}"
|
|
if os.path.isfile(destPath):
|
|
destPath = f"{alternateTrashCanPath }{os.sep}_{time.time()}_{DupFileNameOnly}"
|
|
shutil.move(DupFileName, destPath)
|
|
elif moveToTrashCan:
|
|
sendToTrash(DupFileName)
|
|
result = stash.destroyScene(scene['id'], delete_file=True)
|
|
QtyDeleted += 1
|
|
stash.Debug(f"destroyScene result={result} for file {DupFileName};QtyDeleted={QtyDeleted};Count={QtyDup} of {qtyResults}", toAscii=True)
|
|
else:
|
|
stash.Error("manageTagggedDuplicates called with invlaid input arguments. Doing early exit.")
|
|
return
|
|
stash.Debug("#####################################################")
|
|
stash.Log(f"QtyDup={QtyDup}, QtyClearedTags={QtyClearedTags}, QtySetGraylistTag={QtySetGraylistTag}, QtyDeleted={QtyDeleted}, QtyFailedQuery={QtyFailedQuery}", printTo=LOG_STASH_N_PLUGIN)
|
|
killScanningJobs()
|
|
if deleteScenes and not advanceMenuOptionSelected:
|
|
if cleanAfterDel:
|
|
stash.Log("Adding clean jobs to the Task Queue", printTo=LOG_STASH_N_PLUGIN)
|
|
stash.metadata_clean()
|
|
stash.metadata_clean_generated()
|
|
stash.optimise_database()
|
|
|
|
|
|
def removeDupTag():
|
|
if 'Target' not in stash.JSON_INPUT['args']:
|
|
stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})")
|
|
return
|
|
scene = stash.JSON_INPUT['args']['Target']
|
|
stash.Log(f"Processing scene ID# {scene}")
|
|
stash.removeTag(scene, duplicateMarkForDeletion)
|
|
stash.Log(f"Done removing tag from scene {scene}.")
|
|
jsonReturn = "{'removeDupTag' : 'complete', 'id': '" + f"{scene}" + "'}"
|
|
stash.Log(f"Sending json value {jsonReturn}")
|
|
sys.stdout.write(jsonReturn)
|
|
|
|
def addExcludeTag():
|
|
if 'Target' not in stash.JSON_INPUT['args']:
|
|
stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})")
|
|
return
|
|
scene = stash.JSON_INPUT['args']['Target']
|
|
stash.Log(f"Processing scene ID# {scene}")
|
|
stash.addTag(scene, excludeDupFileDeleteTag)
|
|
stash.Log(f"Done adding exclude tag to scene {scene}.")
|
|
sys.stdout.write("{" + f"addExcludeTag : 'complete', id: '{scene}'" + "}")
|
|
|
|
def removeExcludeTag():
|
|
if 'Target' not in stash.JSON_INPUT['args']:
|
|
stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})")
|
|
return
|
|
scene = stash.JSON_INPUT['args']['Target']
|
|
stash.Log(f"Processing scene ID# {scene}")
|
|
stash.removeTag(scene, excludeDupFileDeleteTag)
|
|
stash.Log(f"Done removing exclude tag from scene {scene}.")
|
|
sys.stdout.write("{" + f"removeExcludeTag : 'complete', id: '{scene}'" + "}")
|
|
|
|
def getParseData(getSceneDetails1=True, getSceneDetails2=True):
|
|
if 'Target' not in stash.JSON_INPUT['args']:
|
|
stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})")
|
|
return None, None
|
|
targetsSrc = stash.JSON_INPUT['args']['Target']
|
|
targets = targetsSrc.split(":")
|
|
if len(targets) < 2:
|
|
stash.Error(f"Could not get both targets from string {targetsSrc}")
|
|
return None, None
|
|
stash.Log(f"Parsed targets {targets[0]} and {targets[1]}")
|
|
target1 = targets[0]
|
|
target2 = targets[1]
|
|
if getSceneDetails1:
|
|
target1 = stash.find_scene(int(target1))
|
|
if getSceneDetails2:
|
|
target2 = stash.find_scene(int(target2))
|
|
elif len(targets) > 2:
|
|
target2 = target2 + targets[2]
|
|
return target1, target2
|
|
|
|
|
|
def mergeTags():
|
|
scene1, scene2 = getParseData()
|
|
if scene1 == None or scene2 == None:
|
|
sys.stdout.write("{" + f"mergeTags : 'failed', id1: '{scene1}', id2: '{scene2}'" + "}")
|
|
return
|
|
stash.mergeMetadata(scene1, scene2)
|
|
stash.Log(f"Done merging scenes for scene {scene1['id']} and scene {scene2['id']}")
|
|
sys.stdout.write("{" + f"mergeTags : 'complete', id1: '{scene1['id']}', id2: '{scene2['id']}'" + "}")
|
|
|
|
def getLocalDupReportPath():
|
|
htmlReportExist = "true" if os.path.isfile(htmlReportName) else "false"
|
|
localPath = htmlReportName.replace("\\", "\\\\")
|
|
jsonReturn = "{'LocalDupReportExist' : " + f"{htmlReportExist}" + ", 'Path': '" + f"{localPath}" + "'}"
|
|
stash.Log(f"Sending json value {jsonReturn}")
|
|
sys.stdout.write(jsonReturn)
|
|
|
|
def deleteLocalDupReportHtmlFiles(doJsonOutput = True):
|
|
htmlReportExist = "true" if os.path.isfile(htmlReportName) else "false"
|
|
if os.path.isfile(htmlReportName):
|
|
stash.Log(f"Deleting file {htmlReportName}")
|
|
os.remove(htmlReportName)
|
|
for x in range(2, 9999):
|
|
fileName = htmlReportName.replace(".html", f"_{x-1}.html")
|
|
stash.Debug(f"Checking if file '{fileName}' exist.")
|
|
if not os.path.isfile(fileName):
|
|
break
|
|
stash.Log(f"Deleting file {fileName}")
|
|
os.remove(fileName)
|
|
else:
|
|
stash.Log(f"Report file does not exist: {htmlReportName}")
|
|
if doJsonOutput:
|
|
jsonReturn = "{'LocalDupReportExist' : " + f"{htmlReportExist}" + ", 'Path': '" + f"{htmlReportName}" + "', 'qty': '" + f"{x}" + "'}"
|
|
stash.Log(f"Sending json value {jsonReturn}")
|
|
sys.stdout.write(jsonReturn)
|
|
|
|
def removeTagFromAllScenes(tagName, deleteTags):
|
|
# ToDo: Replace code with SQL code if DB version 68
|
|
tagId = stash.find_tags(q=tagName)
|
|
if len(tagId) > 0 and 'id' in tagId[0]:
|
|
if deleteTags:
|
|
stash.Debug(f"Deleting tag name {tagName} with Tag ID {tagId[0]['id']} from stash.")
|
|
stash.destroy_tag(int(tagId[0]['id']))
|
|
else:
|
|
stash.Debug(f"Removing tag name {tagName} with Tag ID {tagId[0]['id']} from all scenes.")
|
|
manageTagggedDuplicates(clearTag=True, tagId=int(tagId[0]['id']))
|
|
return True
|
|
return False
|
|
|
|
def removeAllDupTagsFromAllScenes(deleteTags=False):
|
|
tagsToClear = [duplicateMarkForDeletion, base1_duplicateMarkForDeletion, base2_duplicateMarkForDeletion, graylistMarkForDeletion, longerDurationLowerResolution, duplicateWhitelistTag]
|
|
for x in range(0, 3):
|
|
tagsToClear += [base1_duplicateMarkForDeletion + f"_{x}"]
|
|
for x in range(0, 3):
|
|
tagsToClear += [base2_duplicateMarkForDeletion + f"_{x}"]
|
|
tagsToClear = list(set(tagsToClear)) # Remove duplicates
|
|
validTags = []
|
|
for tagToClear in tagsToClear:
|
|
if removeTagFromAllScenes(tagToClear, deleteTags):
|
|
validTags +=[tagToClear]
|
|
if doJsonReturn:
|
|
jsonReturn = "{'removeAllDupTagFromAllScenes' : " + f"{duplicateMarkForDeletion}" + ", 'OtherTags': '" + f"{validTags}" + "'}"
|
|
stash.Log(f"Sending json value {jsonReturn}")
|
|
sys.stdout.write(jsonReturn)
|
|
else:
|
|
stash.Log(f"Clear tags {tagsToClear}")
|
|
|
|
def insertDisplayNone(htmlReportName, scene):
|
|
stash.Log(f"Inserting display none for scene {scene} in file {htmlReportName}")
|
|
import in_place
|
|
doStyleEndTagCheck = True
|
|
with in_place.InPlace(htmlReportName) as file:
|
|
for line in file:
|
|
if doStyleEndTagCheck and line.startsWith("</style>"):
|
|
file.write(f".ID_{scene}" + "{display:none;}")
|
|
doStyleEndTagCheck = False
|
|
file.write(line)
|
|
file.close()
|
|
|
|
def hideScene(scene):
|
|
if os.path.isfile(htmlReportName):
|
|
insertDisplayNone(htmlReportName, scene)
|
|
for x in range(2, 9999):
|
|
fileName = htmlReportName.replace(".html", f"_{x-1}.html")
|
|
stash.Debug(f"Checking if file '{fileName}' exist.")
|
|
if not os.path.isfile(fileName):
|
|
break
|
|
insertDisplayNone(fileName, scene)
|
|
else:
|
|
stash.Log(f"Report file does not exist: {htmlReportName}")
|
|
|
|
def deleteScene(hideInReport=True, deleteFile=True):
|
|
if 'Target' not in stash.JSON_INPUT['args']:
|
|
stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})")
|
|
return
|
|
scene = stash.JSON_INPUT['args']['Target']
|
|
stash.Log(f"Processing scene ID# {scene}")
|
|
result = stash.destroyScene(scene, delete_file=deleteFile)
|
|
if hideInReport:
|
|
hideScene(scene)
|
|
stash.Log(f"{stash.PLUGIN_TASK_NAME} complete for scene {scene} with results = {result}")
|
|
sys.stdout.write("{" + f"{stash.PLUGIN_TASK_NAME} : 'complete', id: '{scene}', result: '{result}'" + "}")
|
|
|
|
def copyScene(moveScene=False):
|
|
scene1, scene2 = getParseData()
|
|
if scene1 == None or scene2 == None:
|
|
sys.stdout.write("{" + f"{stash.PLUGIN_TASK_NAME} : 'failed', id1: '{scene1}', id2: '{scene2}'" + "}")
|
|
return
|
|
if moveScene:
|
|
stash.mergeMetadata(scene1, scene2)
|
|
result = shutil.copy(scene1['file']['path'], scene2['file']['path'])
|
|
if moveScene:
|
|
result = stash.destroyScene(scene1['id'], delete_file=True)
|
|
stash.Log(f"destroyScene for scene {scene1['id']} results = {result}")
|
|
stash.Log(f"{stash.PLUGIN_TASK_NAME} complete for scene {scene1['id']} and {scene2['id]}")
|
|
sys.stdout.write("{" + f"{stash.PLUGIN_TASK_NAME} : 'complete', id1: '{scene1['id']}', id2: '{scene2['id']}', result: '{result}'" + "}")
|
|
|
|
def renameFile():
|
|
scene, newName = getParseData(getSceneDetails2=False)
|
|
if scene == None or newName == None:
|
|
sys.stdout.write("{" + f"{stash.PLUGIN_TASK_NAME} : 'failed', scene: '{scene}', newName: '{newName}'" + "}")
|
|
return
|
|
newName = scene['file']['path'].replace(pathlib.Path(scene['file']['path']).stem, newName)
|
|
result = shutil.rename(scene['file']['path'], newName)
|
|
stash.Log(f"{stash.PLUGIN_TASK_NAME} complete for scene {scene['id']} ;renamed to {newName}; result={resul}")
|
|
sys.stdout.write("{" + f"{stash.PLUGIN_TASK_NAME} : 'complete', scene: '{scene['id']}', newName: '{newName}', result: '{result}'" + "}")
|
|
|
|
|
|
# ToDo: Add to UI menu
|
|
# Remove all Dup tagged files (Just remove from stash, and leave file)
|
|
# Clear GraylistMarkForDel tag
|
|
# Delete GraylistMarkForDel tag
|
|
# Remove from stash all files no longer part of stash library
|
|
# Remove from stash all files in the Exclusion list (Not supporting regexps)
|
|
# ToDo: Add to advance menu
|
|
# Remove only graylist dup
|
|
# Exclude graylist from delete
|
|
# Include graylist in delete
|
|
|
|
try:
|
|
if stash.PLUGIN_TASK_NAME == "tag_duplicates_task":
|
|
mangeDupFiles(tagDuplicates=True, merge=mergeDupFilename)
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "delete_tagged_duplicates_task":
|
|
manageTagggedDuplicates(deleteScenes=True)
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "delete_duplicates_task":
|
|
mangeDupFiles(deleteDup=True, merge=mergeDupFilename)
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "clear_duplicate_tags_task":
|
|
removeAllDupTagsFromAllScenes()
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "graylist_tag_task":
|
|
manageTagggedDuplicates(setGrayListTag=True)
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "generate_phash_task":
|
|
stash.metadata_generate({"phashes": True})
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "deleteScene":
|
|
deleteScene()
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "removeScene":
|
|
deleteScene(deleteFile=False)
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "renameFile":
|
|
renameFile()
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "copyScene":
|
|
copyScene()
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "moveScene":
|
|
copyScene(moveScene=True)
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "removeDupTag":
|
|
removeDupTag()
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "addExcludeTag":
|
|
addExcludeTag()
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "removeExcludeTag":
|
|
removeExcludeTag()
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "mergeTags":
|
|
mergeTags()
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "getLocalDupReportPath":
|
|
getLocalDupReportPath()
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "deleteLocalDupReportHtmlFiles":
|
|
deleteLocalDupReportHtmlFiles()
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "createDuplicateReportWithoutTagging":
|
|
mangeDupFiles(tagDuplicates=False, merge=mergeDupFilename)
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "deleteAllDupFileManagerTags":
|
|
removeAllDupTagsFromAllScenes(deleteTags=True)
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "deleteBlackListTaggedDuplicatesTask":
|
|
mangeDupFiles(deleteDup=True, merge=mergeDupFilename, deleteBlacklistOnly=True)
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "deleteTaggedDuplicatesLwrResOrLwrDuration":
|
|
mangeDupFiles(deleteDup=True, merge=mergeDupFilename, deleteLowerResAndDuration=True)
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif stash.PLUGIN_TASK_NAME == "deleteBlackListTaggedDuplicatesLwrResOrLwrDuration":
|
|
mangeDupFiles(deleteDup=True, merge=mergeDupFilename, deleteBlacklistOnly=True, deleteLowerResAndDuration=True)
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
elif parse_args.dup_tag:
|
|
stash.PLUGIN_TASK_NAME = "dup_tag"
|
|
mangeDupFiles(tagDuplicates=True, merge=mergeDupFilename)
|
|
stash.Debug(f"Tag duplicate EXIT")
|
|
elif parse_args.del_tag:
|
|
stash.PLUGIN_TASK_NAME = "del_tag"
|
|
manageTagggedDuplicates(deleteScenes=True)
|
|
stash.Debug(f"Delete Tagged duplicates EXIT")
|
|
elif parse_args.clear_tag:
|
|
stash.PLUGIN_TASK_NAME = "clear_tag"
|
|
removeAllDupTagsFromAllScenes()
|
|
stash.Debug(f"Clear duplicate tags EXIT")
|
|
elif parse_args.remove:
|
|
stash.PLUGIN_TASK_NAME = "remove"
|
|
mangeDupFiles(deleteDup=True, merge=mergeDupFilename)
|
|
stash.Debug(f"Delete duplicate EXIT")
|
|
elif len(sys.argv) < 2 and stash.PLUGIN_TASK_NAME in advanceMenuOptions:
|
|
manageTagggedDuplicates(deleteScenes=True, advanceMenuOptionSelected=True)
|
|
stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT")
|
|
else:
|
|
stash.Log(f"Nothing to do!!! (PLUGIN_ARGS_MODE={stash.PLUGIN_TASK_NAME})")
|
|
except Exception as e:
|
|
tb = traceback.format_exc()
|
|
|
|
stash.Error(f"Exception while running DupFileManager Task({stash.PLUGIN_TASK_NAME}); Error: {e}\nTraceBack={tb}")
|
|
killScanningJobs()
|
|
stash.convertToAscii = False
|
|
stash.Error(f"Error: {e}\nTraceBack={tb}")
|
|
if doJsonReturn:
|
|
sys.stdout.write("{" + f"Exception : '{e}; See log file for TraceBack' " + "}")
|
|
|
|
stash.Log("\n*********************************\nEXITING ***********************\n*********************************")
|