chris/Axter-Stash
1
0
Fork 0
forked from Github/Axter-Stash
Code Pull Requests Activity

Too many changes to list

Browse Source
This commit is contained in:
David Maisonave
2024-09-13 10:10:37 -04:00
parent 5b34502963
commit 452c08df03
18 changed files with 1645 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

365
plugins/DupFileManager/DupFileManager.py
View File

@@ -5,17 +5,20 @@
# Example: python DupFileManager.py --url http://localhost:9999 -a
import os, sys, time, pathlib, argparse, platform, shutil, logging
from StashPluginHelper import StashPluginHelper
from stashapi.stash_types import PhashDistance
from DupFileManager_config import config # Import config from DupFileManager_config.py
parser = argparse.ArgumentParser()
parser.add_argument('--url', '-u', dest='stash_url', type=str, help='Add Stash URL')
parser.add_argument('--trace', '-t', dest='trace', action='store_true', help='Enables debug trace mode.')
parser.add_argument('--add_dup_tag', '-a', dest='dup_tag', action='store_true', help='Set a tag to duplicate files.')
parser.add_argument('--clear_dup_tag', '-c', dest='clear_tag', action='store_true', help='Clear duplicates of duplicate tags.')
parser.add_argument('--del_tag_dup', '-d', dest='del_tag', action='store_true', help='Only delete scenes having DuplicateMarkForDeletion tag.')
parser.add_argument('--remove_dup', '-r', dest='remove', action='store_true', help='Remove (delete) duplicate files.')
parse_args = parser.parse_args()
settings = {
"doNotGeneratePhash": False,
"mergeDupFilename": False,
"permanentlyDelete": False,
"whitelistDelDupInSameFolder": False,
@@ -23,9 +26,13 @@ settings = {
"zCleanAfterDel": False,
"zSwapHighRes": False,
"zSwapLongLength": False,
"zSwapBetterBitRate": False,
"zSwapCodec": False,
"zSwapBetterFrameRate": False,
"zWhitelist": "",
"zxGraylist": "",
"zyBlacklist": "",
"zyMatchDupDistance": 0,
"zyMaxDupToProcess": 0,
"zzdebugTracing": False,
}
@@ -40,7 +47,9 @@ if len(sys.argv) > 1:
stash.Log(f"argv = {sys.argv}")
else:
stash.Trace(f"No command line arguments. JSON_INPUT['args'] = {stash.JSON_INPUT['args']}")
stash.Status(logLevel=logging.DEBUG)
stash.status(logLevel=logging.DEBUG)
stash.modulesInstalled(["send2trash", "requests"])
# stash.Trace(f"\nStarting (__file__={__file__}) (stash.CALLED_AS_STASH_PLUGIN={stash.CALLED_AS_STASH_PLUGIN}) (stash.DEBUG_TRACING={stash.DEBUG_TRACING}) (stash.PLUGIN_TASK_NAME={stash.PLUGIN_TASK_NAME})************************************************")
# stash.encodeToUtf8 = True
@@ -49,22 +58,48 @@ stash.Status(logLevel=logging.DEBUG)
LOG_STASH_N_PLUGIN = stash.LOG_TO_STASH if stash.CALLED_AS_STASH_PLUGIN else stash.LOG_TO_CONSOLE + stash.LOG_TO_FILE
listSeparator = stash.Setting('listSeparator', ',', notEmpty=True)
addPrimaryDupPathToDetails = stash.Setting('addPrimaryDupPathToDetails')
doNotGeneratePhash = stash.Setting('doNotGeneratePhash')
mergeDupFilename = stash.Setting('mergeDupFilename')
moveToTrashCan = False if stash.Setting('permanentlyDelete') else True
alternateTrashCanPath = stash.Setting('dup_path')
whitelistDelDupInSameFolder = stash.Setting('whitelistDelDupInSameFolder')
whitelistDoTagLowResDup = stash.Setting('whitelistDoTagLowResDup')
maxDupToProcess = int(stash.Setting('zyMaxDupToProcess'))
swapHighRes = stash.Setting('zSwapHighRes')
swapLongLength = stash.Setting('zSwapLongLength')
significantTimeDiff = stash.Setting('significantTimeDiff')
toRecycleBeforeSwap = stash.Setting('toRecycleBeforeSwap')
cleanAfterDel = stash.Setting('zCleanAfterDel')
duration_diff = float(stash.Setting('duration_diff'))
if duration_diff > 10:
duration_diff = 10
elif duration_diff < 1:
duration_diff = 1
swapHighRes = stash.Setting('zSwapHighRes')
swapLongLength = stash.Setting('zSwapLongLength')
swapBetterBitRate = stash.Setting('zSwapBetterBitRate')
swapCodec = stash.Setting('zSwapCodec')
swapBetterFrameRate = stash.Setting('zSwapBetterFrameRate')
favorLongerFileName = stash.Setting('favorLongerFileName')
favorLargerFileSize = stash.Setting('favorLargerFileSize')
favorBitRateChange = stash.Setting('favorBitRateChange')
favorHighBitRate = stash.Setting('favorHighBitRate')
favorFrameRateChange = stash.Setting('favorFrameRateChange')
favorHigherFrameRate = stash.Setting('favorHigherFrameRate')
favorCodecRanking = stash.Setting('favorCodecRanking')
codecRankingSetToUse = stash.Setting('codecRankingSetToUse')
if codecRankingSetToUse == 4:
codecRanking = stash.Setting('codecRankingSet4')
elif codecRankingSetToUse == 3:
codecRanking = stash.Setting('codecRankingSet3')
elif codecRankingSetToUse == 2:
codecRanking = stash.Setting('codecRankingSet2')
else:
codecRanking = stash.Setting('codecRankingSet1')
matchDupDistance = int(stash.Setting('zyMatchDupDistance'))
matchPhaseDistance = PhashDistance.EXACT
matchPhaseDistanceText = "Exact Match"
if matchDupDistance == 1:
matchPhaseDistance = PhashDistance.HIGH
matchPhaseDistanceText = "High Match"
elif matchDupDistance == 2:
matchPhaseDistance = PhashDistance.MEDIUM
matchPhaseDistanceText = "Medium Match"
# significantTimeDiff can not be higher than 1 and shouldn't be lower than .5
if significantTimeDiff > 1:
@@ -79,10 +114,14 @@ if duplicateMarkForDeletion == "":
duplicateWhitelistTag = stash.Setting('DupWhiteListTag')
if duplicateWhitelistTag == "":
duplicateWhitelistTag = 'DuplicateWhitelistFile'
duplicateWhitelistTag = '_DuplicateWhitelistFile'
excludeMergeTags = [duplicateMarkForDeletion, duplicateWhitelistTag]
stash.init_mergeMetadata(excludeMergeTags)
excludeDupFileDeleteTag = stash.Setting('excludeDupFileDeleteTag')
if excludeDupFileDeleteTag == "":
excludeDupFileDeleteTag = '_ExcludeDuplicateMarkForDeletion'
excludeMergeTags = [duplicateMarkForDeletion, duplicateWhitelistTag, excludeDupFileDeleteTag]
stash.initMergeMetadata(excludeMergeTags)
graylist = stash.Setting('zxGraylist').split(listSeparator)
graylist = [item.lower() for item in graylist]
@@ -169,36 +208,26 @@ def testReparsePointAndSymLink(merge=False, deleteDup=False):
stash.Log(f"Not isSymLink '{myTestPath6}'")
return
detailPrefix = "BaseDup="
detailPostfix = "<BaseDup>\n"
def createTagId(tagName, tagName_descp, deleteIfExist = False):
tagId = stash.find_tags(q=tagName)
if len(tagId):
tagId = tagId[0]
if deleteIfExist:
stash.destroy_tag(int(tagId['id']))
else:
return tagId['id']
tagId = stash.create_tag({"name":tagName, "description":tagName_descp, "ignore_auto_tag": True})
stash.Log(f"Dup-tagId={tagId['id']}")
return tagId['id']
def setTagId(tagId, tagName, sceneDetails, DupFileToKeep):
def setTagId(tagName, sceneDetails, DupFileToKeep):
details = ""
ORG_DATA_DICT = {'id' : sceneDetails['id']}
dataDict = ORG_DATA_DICT.copy()
doAddTag = True
if addPrimaryDupPathToDetails:
BaseDupStr = f"BaseDup={DupFileToKeep['files'][0]['path']}\n{stash.STASH_URL}/scenes/{DupFileToKeep['id']}\n"
BaseDupStr = f"{detailPrefix}{DupFileToKeep['files'][0]['path']}\n{stash.STASH_URL}/scenes/{DupFileToKeep['id']}\n(matchDupDistance={matchPhaseDistanceText})\n{detailPostfix}"
if sceneDetails['details'] == "":
details = BaseDupStr
elif not sceneDetails['details'].startswith(BaseDupStr):
elif not sceneDetails['details'].startswith(detailPrefix):
details = f"{BaseDupStr};\n{sceneDetails['details']}"
for tag in sceneDetails['tags']:
if tag['name'] == tagName:
doAddTag = False
break
if doAddTag:
dataDict.update({'tag_ids' : tagId})
stash.addTag(sceneDetails, tagName)
if details != "":
dataDict.update({'details' : details})
if dataDict != ORG_DATA_DICT:
@@ -208,13 +237,27 @@ def setTagId(tagId, tagName, sceneDetails, DupFileToKeep):
stash.Trace(f"[setTagId] Nothing to update {sceneDetails['files'][0]['path']}.", toAscii=True)
def isInList(listToCk, pathToCk):
pathToCk = pathToCk.lower()
def isInList(listToCk, itemToCk):
itemToCk = itemToCk.lower()
for item in listToCk:
if pathToCk.startswith(item):
if itemToCk.startswith(item):
return True
return False
NOT_IN_LIST = 65535
def indexInList(listToCk, itemToCk):
itemToCk = itemToCk.lower()
index = -1
lenItemMatch = 0
returnValue = NOT_IN_LIST
for item in listToCk:
index += 1
if itemToCk.startswith(item):
if len(item) > lenItemMatch: # Make sure the best match is selected by getting match with longest string.
lenItemMatch = len(item)
returnValue = index
return returnValue
def hasSameDir(path1, path2):
if pathlib.Path(path1).resolve().parent == pathlib.Path(path2).resolve().parent:
return True
@@ -244,6 +287,26 @@ def significantLessTime(durrationToKeep, durrationOther):
return True
return False
def isBetterVideo(scene1, scene2, swapCandidateCk = False):
# Prioritize higher reslution over codec, bit rate, and frame rate
if int(scene1['files'][0]['width']) > int(scene2['files'][0]['width']) or int(scene1['files'][0]['height']) > int(scene2['files'][0]['height']):
return False
if (favorBitRateChange and swapCandidateCk == False) or (swapCandidateCk and swapBetterBitRate):
if (favorHighBitRate and int(scene2['files'][0]['bit_rate']) > int(scene1['files'][0]['bit_rate'])) or (not favorHighBitRate and int(scene2['files'][0]['bit_rate']) < int(scene1['files'][0]['bit_rate'])):
stash.Trace(f"[isBetterVideo]:[favorHighBitRate={favorHighBitRate}] Better bit rate. {scene1['files'][0]['path']}={scene1['files'][0]['bit_rate']} v.s. {scene2['files'][0]['path']}={scene2['files'][0]['bit_rate']}")
return True
if (favorCodecRanking and swapCandidateCk == False) or (swapCandidateCk and swapCodec):
scene1CodecRank = indexInList(codecRanking, scene1['files'][0]['video_codec'])
scene2CodecRank = indexInList(codecRanking, scene2['files'][0]['video_codec'])
if scene2CodecRank < scene1CodecRank:
stash.Trace(f"[isBetterVideo] Better codec. {scene1['files'][0]['path']}={scene1['files'][0]['video_codec']}:Rank={scene1CodecRank} v.s. {scene2['files'][0]['path']}={scene2['files'][0]['video_codec']}:Rank={scene2CodecRank}")
return True
if (favorFrameRateChange and swapCandidateCk == False) or (swapCandidateCk and swapBetterFrameRate):
if (favorHigherFrameRate and int(scene2['files'][0]['frame_rate']) > int(scene1['files'][0]['frame_rate'])) or (not favorHigherFrameRate and int(scene2['files'][0]['frame_rate']) < int(scene1['files'][0]['frame_rate'])):
stash.Trace(f"[isBetterVideo]:[favorHigherFrameRate={favorHigherFrameRate}] Better frame rate. {scene1['files'][0]['path']}={scene1['files'][0]['frame_rate']} v.s. {scene2['files'][0]['path']}={scene2['files'][0]['frame_rate']}")
return True
return False
def isSwapCandidate(DupFileToKeep, DupFile):
# Don't move if both are in whitelist
if isInList(whitelist, DupFileToKeep['files'][0]['path']) and isInList(whitelist, DupFile['files'][0]['path']):
@@ -256,20 +319,69 @@ def isSwapCandidate(DupFileToKeep, DupFile):
if swapLongLength and int(DupFileToKeep['files'][0]['duration']) > int(DupFile['files'][0]['duration']):
if int(DupFileToKeep['files'][0]['width']) >= int(DupFile['files'][0]['width']) or int(DupFileToKeep['files'][0]['height']) >= int(DupFile['files'][0]['height']):
return True
if isBetterVideo(DupFile, DupFileToKeep, swapCandidateCk=True):
if not significantLessTime(int(DupFileToKeep['files'][0]['duration']), int(DupFile['files'][0]['duration'])):
return True
else:
stash.Warn(f"File '{DupFileToKeep['files'][0]['path']}' has better codec/bit-rate than '{DupFile['files'][0]['path']}', but the duration is significantly shorter.", toAscii=True)
return False
dupWhitelistTagId = None
def addDupWhitelistTag():
global dupWhitelistTagId
stash.Trace(f"Adding tag duplicateWhitelistTag = {duplicateWhitelistTag}")
descp = 'Tag added to duplicate scenes which are in the whitelist. This means there are two or more duplicates in the whitelist.'
dupWhitelistTagId = stash.createTagId(duplicateWhitelistTag, descp, ignoreAutoTag=True)
stash.Trace(f"dupWhitelistTagId={dupWhitelistTagId} name={duplicateWhitelistTag}")
excludeDupFileDeleteTagId = None
def addExcludeDupTag():
global excludeDupFileDeleteTagId
stash.Trace(f"Adding tag excludeDupFileDeleteTag = {excludeDupFileDeleteTag}")
descp = 'Excludes duplicate scene from DupFileManager tagging and deletion process. A scene having this tag will not get deleted by DupFileManager'
excludeDupFileDeleteTagId = stash.createTagId(excludeDupFileDeleteTag, descp, ignoreAutoTag=True)
stash.Trace(f"dupWhitelistTagId={excludeDupFileDeleteTagId} name={excludeDupFileDeleteTag}")
def isTaggedExcluded(Scene):
for tag in Scene['tags']:
if tag['name'] == excludeDupFileDeleteTag:
return True
return False
def isWorseKeepCandidate(DupFileToKeep, Scene):
if not isInList(whitelist, Scene['files'][0]['path']) and isInList(whitelist, DupFileToKeep['files'][0]['path']):
return True
if not isInList(graylist, Scene['files'][0]['path']) and isInList(graylist, DupFileToKeep['files'][0]['path']):
return True
if not isInList(blacklist, DupFileToKeep['files'][0]['path']) and isInList(blacklist, Scene['files'][0]['path']):
return True
if isInList(graylist, Scene['files'][0]['path']) and isInList(graylist, DupFileToKeep['files'][0]['path']) and indexInList(graylist, DupFileToKeep['files'][0]['path']) < indexInList(graylist, Scene['files'][0]['path']):
return True
if isInList(blacklist, DupFileToKeep['files'][0]['path']) and isInList(blacklist, Scene['files'][0]['path']) and indexInList(blacklist, DupFileToKeep['files'][0]['path']) < indexInList(blacklist, Scene['files'][0]['path']):
return True
return False
stopProcessBarSpin = True
def spinProcessBar(sleepSeconds = 1):
pos = 1
maxPos = 30
while stopProcessBarSpin == False:
stash.progressBar(pos, maxPos)
pos +=1
if pos > maxPos:
pos = 1
time.sleep(sleepSeconds)
def mangeDupFiles(merge=False, deleteDup=False, tagDuplicates=False):
global stopProcessBarSpin
duplicateMarkForDeletion_descp = 'Tag added to duplicate scenes so-as to tag them for deletion.'
stash.Trace(f"duplicateMarkForDeletion = {duplicateMarkForDeletion}")
dupTagId = createTagId(duplicateMarkForDeletion, duplicateMarkForDeletion_descp)
dupTagId = stash.createTagId(duplicateMarkForDeletion, duplicateMarkForDeletion_descp, ignoreAutoTag=True)
stash.Trace(f"dupTagId={dupTagId} name={duplicateMarkForDeletion}")
dupWhitelistTagId = None
if whitelistDoTagLowResDup:
stash.Trace(f"duplicateWhitelistTag = {duplicateWhitelistTag}")
duplicateWhitelistTag_descp = 'Tag added to duplicate scenes which are in the whitelist. This means there are two or more duplicates in the whitelist.'
dupWhitelistTagId = createTagId(duplicateWhitelistTag, duplicateWhitelistTag_descp)
stash.Trace(f"dupWhitelistTagId={dupWhitelistTagId} name={duplicateWhitelistTag}")
addDupWhitelistTag()
addExcludeDupTag()
QtyDupSet = 0
QtyDup = 0
@@ -278,26 +390,30 @@ def mangeDupFiles(merge=False, deleteDup=False, tagDuplicates=False):
QtyRealTimeDiff = 0
QtyTagForDel = 0
QtySkipForDel = 0
QtyExcludeForDel = 0
QtySwap = 0
QtyMerge = 0
QtyDeleted = 0
stash.Log("#########################################################################")
stash.Trace("#########################################################################")
stash.Log(f"Waiting for find_duplicate_scenes_diff to return results; duration_diff={duration_diff}; significantTimeDiff={significantTimeDiff}", printTo=LOG_STASH_N_PLUGIN)
DupFileSets = stash.find_duplicate_scenes_diff(duration_diff=duration_diff)
stash.Log(f"Waiting for find_duplicate_scenes_diff to return results; matchDupDistance={matchPhaseDistanceText}; significantTimeDiff={significantTimeDiff}", printTo=LOG_STASH_N_PLUGIN)
stopProcessBarSpin = False
stash.submit(spinProcessBar)
DupFileSets = stash.find_duplicate_scenes(matchPhaseDistance)
stopProcessBarSpin = True
time.sleep(1) # Make sure we give time for spinProcessBar to exit
qtyResults = len(DupFileSets)
stash.Trace("#########################################################################")
for DupFileSet in DupFileSets:
stash.Trace(f"DupFileSet={DupFileSet}")
QtyDupSet+=1
stash.Progress(QtyDupSet, qtyResults)
stash.progressBar(QtyDupSet, qtyResults)
SepLine = "---------------------------"
DupFileToKeep = ""
DupToCopyFrom = ""
DupFileDetailList = []
for DupFile in DupFileSet:
QtyDup+=1
stash.log.sl.progress(f"Scene ID = {DupFile['id']}")
time.sleep(2)
Scene = stash.find_scene(DupFile['id'])
sceneData = f"Scene = {Scene}"
@@ -311,19 +427,45 @@ def mangeDupFiles(merge=False, deleteDup=False, tagDuplicates=False):
SepLine = "***************************"
if significantLessTime(int(DupFileToKeep['files'][0]['duration']), int(Scene['files'][0]['duration'])):
QtyRealTimeDiff += 1
if int(DupFileToKeep['files'][0]['width']) < int(Scene['files'][0]['width']) or int(DupFileToKeep['files'][0]['height']) < int(Scene['files'][0]['height']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=resolution: {DupFileToKeep['files'][0]['width']}x{DupFileToKeep['files'][0]['height']} < {Scene['files'][0]['width']}x{Scene['files'][0]['height']}")
DupFileToKeep = Scene
elif int(DupFileToKeep['files'][0]['duration']) < int(Scene['files'][0]['duration']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=duration: {DupFileToKeep['files'][0]['duration']} < {Scene['files'][0]['duration']}")
DupFileToKeep = Scene
elif isBetterVideo(DupFileToKeep, Scene):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=codec,bit_rate, or frame_rate: {DupFileToKeep['files'][0]['video_codec']}, {DupFileToKeep['files'][0]['bit_rate']}, {DupFileToKeep['files'][0]['frame_rate']} : {Scene['files'][0]['video_codec']}, {Scene['files'][0]['bit_rate']}, {Scene['files'][0]['frame_rate']}")
DupFileToKeep = Scene
elif isInList(whitelist, Scene['files'][0]['path']) and not isInList(whitelist, DupFileToKeep['files'][0]['path']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=not whitelist vs whitelist")
DupFileToKeep = Scene
elif isTaggedExcluded(Scene) and not isTaggedExcluded(DupFileToKeep):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=not ExcludeTag vs ExcludeTag")
DupFileToKeep = Scene
elif isInList(blacklist, DupFileToKeep['files'][0]['path']) and not isInList(blacklist, Scene['files'][0]['path']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=blacklist vs not blacklist")
DupFileToKeep = Scene
elif isInList(blacklist, DupFileToKeep['files'][0]['path']) and isInList(blacklist, Scene['files'][0]['path']) and indexInList(blacklist, DupFileToKeep['files'][0]['path']) > indexInList(blacklist, Scene['files'][0]['path']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=blacklist-index {indexInList(blacklist, DupFileToKeep['files'][0]['path'])} > {indexInList(blacklist, Scene['files'][0]['path'])}")
DupFileToKeep = Scene
elif isInList(graylist, Scene['files'][0]['path']) and not isInList(graylist, DupFileToKeep['files'][0]['path']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=not graylist vs graylist")
DupFileToKeep = Scene
elif len(DupFileToKeep['files'][0]['path']) < len(Scene['files'][0]['path']):
elif isInList(graylist, Scene['files'][0]['path']) and isInList(graylist, DupFileToKeep['files'][0]['path']) and indexInList(graylist, DupFileToKeep['files'][0]['path']) > indexInList(graylist, Scene['files'][0]['path']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=graylist-index {indexInList(graylist, DupFileToKeep['files'][0]['path'])} > {indexInList(graylist, Scene['files'][0]['path'])}")
DupFileToKeep = Scene
elif int(DupFileToKeep['files'][0]['size']) < int(Scene['files'][0]['size']):
elif favorLongerFileName and len(DupFileToKeep['files'][0]['path']) < len(Scene['files'][0]['path']) and not isWorseKeepCandidate(DupFileToKeep, Scene):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=path-len {len(DupFileToKeep['files'][0]['path'])} < {len(Scene['files'][0]['path'])}")
DupFileToKeep = Scene
elif favorLargerFileSize and int(DupFileToKeep['files'][0]['size']) < int(Scene['files'][0]['size']) and not isWorseKeepCandidate(DupFileToKeep, Scene):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=size {DupFileToKeep['files'][0]['size']} < {Scene['files'][0]['size']}")
DupFileToKeep = Scene
elif not favorLongerFileName and len(DupFileToKeep['files'][0]['path']) > len(Scene['files'][0]['path']) and not isWorseKeepCandidate(DupFileToKeep, Scene):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=path-len {len(DupFileToKeep['files'][0]['path'])} > {len(Scene['files'][0]['path'])}")
DupFileToKeep = Scene
elif not favorLargerFileSize and int(DupFileToKeep['files'][0]['size']) > int(Scene['files'][0]['size']) and not isWorseKeepCandidate(DupFileToKeep, Scene):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=size {DupFileToKeep['files'][0]['size']} > {Scene['files'][0]['size']}")
DupFileToKeep = Scene
else:
DupFileToKeep = Scene
@@ -333,14 +475,14 @@ def mangeDupFiles(merge=False, deleteDup=False, tagDuplicates=False):
for DupFile in DupFileDetailList:
if DupFile['id'] != DupFileToKeep['id']:
if merge:
result = stash.merge_metadata(DupFile, DupFileToKeep)
result = stash.mergeMetadata(DupFile, DupFileToKeep)
if result != "Nothing To Merge":
QtyMerge += 1
if isInList(whitelist, DupFile['files'][0]['path']) and (not whitelistDelDupInSameFolder or not hasSameDir(DupFile['files'][0]['path'], DupFileToKeep['files'][0]['path'])):
if isSwapCandidate(DupFileToKeep, DupFile):
if merge:
stash.merge_metadata(DupFileToKeep, DupFile)
stash.mergeMetadata(DupFileToKeep, DupFile)
if toRecycleBeforeSwap:
sendToTrash(DupFile['files'][0]['path'])
shutil.move(DupFileToKeep['files'][0]['path'], DupFile['files'][0]['path'])
@@ -350,41 +492,48 @@ def mangeDupFiles(merge=False, deleteDup=False, tagDuplicates=False):
else:
stash.Log(f"NOT processing duplicate, because it's in whitelist. '{DupFile['files'][0]['path']}'", toAscii=True)
if dupWhitelistTagId and tagDuplicates:
setTagId(dupWhitelistTagId, duplicateWhitelistTag, DupFile, DupFileToKeep)
setTagId(duplicateWhitelistTag, DupFile, DupFileToKeep)
QtySkipForDel+=1
else:
if deleteDup:
DupFileName = DupFile['files'][0]['path']
DupFileNameOnly = pathlib.Path(DupFileName).stem
stash.Warn(f"Deleting duplicate '{DupFileName}'", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
if alternateTrashCanPath != "":
destPath = f"{alternateTrashCanPath }{os.sep}{DupFileNameOnly}"
if os.path.isfile(destPath):
destPath = f"{alternateTrashCanPath }{os.sep}_{time.time()}_{DupFileNameOnly}"
shutil.move(DupFileName, destPath)
elif moveToTrashCan:
sendToTrash(DupFileName)
stash.destroy_scene(DupFile['id'], delete_file=True)
QtyDeleted += 1
elif tagDuplicates:
if QtyTagForDel == 0:
stash.Log(f"Tagging duplicate {DupFile['files'][0]['path']} for deletion with tag {duplicateMarkForDeletion}.", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
else:
stash.Log(f"Tagging duplicate {DupFile['files'][0]['path']} for deletion.", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
setTagId(dupTagId, duplicateMarkForDeletion, DupFile, DupFileToKeep)
QtyTagForDel+=1
if isTaggedExcluded(DupFile):
stash.Log(f"Excluding file {DupFile['files'][0]['path']} because tagged for exclusion via tag {excludeDupFileDeleteTag}")
QtyExcludeForDel+=1
else:
if deleteDup:
DupFileName = DupFile['files'][0]['path']
DupFileNameOnly = pathlib.Path(DupFileName).stem
stash.Warn(f"Deleting duplicate '{DupFileName}'", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
if alternateTrashCanPath != "":
destPath = f"{alternateTrashCanPath }{os.sep}{DupFileNameOnly}"
if os.path.isfile(destPath):
destPath = f"{alternateTrashCanPath }{os.sep}_{time.time()}_{DupFileNameOnly}"
shutil.move(DupFileName, destPath)
elif moveToTrashCan:
sendToTrash(DupFileName)
stash.destroy_scene(DupFile['id'], delete_file=True)
QtyDeleted += 1
elif tagDuplicates:
if QtyTagForDel == 0:
stash.Log(f"Tagging duplicate {DupFile['files'][0]['path']} for deletion with tag {duplicateMarkForDeletion}.", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
else:
stash.Log(f"Tagging duplicate {DupFile['files'][0]['path']} for deletion.", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
setTagId(duplicateMarkForDeletion, DupFile, DupFileToKeep)
QtyTagForDel+=1
stash.Trace(SepLine)
if maxDupToProcess > 0 and QtyDup > maxDupToProcess:
break
stash.Log(f"QtyDupSet={QtyDupSet}, QtyDup={QtyDup}, QtyDeleted={QtyDeleted}, QtySwap={QtySwap}, QtyTagForDel={QtyTagForDel}, QtySkipForDel={QtySkipForDel}, QtyExactDup={QtyExactDup}, QtyAlmostDup={QtyAlmostDup}, QtyMerge={QtyMerge}, QtyRealTimeDiff={QtyRealTimeDiff}", printTo=LOG_STASH_N_PLUGIN)
stash.Log(f"QtyDupSet={QtyDupSet}, QtyDup={QtyDup}, QtyDeleted={QtyDeleted}, QtySwap={QtySwap}, QtyTagForDel={QtyTagForDel}, QtySkipForDel={QtySkipForDel}, QtyExcludeForDel={QtyExcludeForDel}, QtyExactDup={QtyExactDup}, QtyAlmostDup={QtyAlmostDup}, QtyMerge={QtyMerge}, QtyRealTimeDiff={QtyRealTimeDiff}", printTo=LOG_STASH_N_PLUGIN)
if doNotGeneratePhash == False:
stash.metadata_generate({"phashes": True})
if cleanAfterDel:
stash.Log("Adding clean jobs to the Task Queue", printTo=LOG_STASH_N_PLUGIN)
stash.metadata_clean(paths=stash.STASH_PATHS)
stash.metadata_clean_generated()
stash.optimise_database()
def deleteTagggedDuplicates():
def manageTagggedDuplicates(clearTag=False):
global stopProcessBarSpin
tagId = stash.find_tags(q=duplicateMarkForDeletion)
if len(tagId) > 0 and 'id' in tagId[0]:
tagId = tagId[0]['id']
@@ -393,63 +542,85 @@ def deleteTagggedDuplicates():
return
QtyDup = 0
QtyDeleted = 0
QtyClearedTags = 0
QtyFailedQuery = 0
stash.Trace("#########################################################################")
stopProcessBarSpin = False
stash.submit(spinProcessBar)
sceneIDs = stash.find_scenes(f={"tags": {"value":tagId, "modifier":"INCLUDES"}}, fragment='id')
stopProcessBarSpin = True
time.sleep(1) # Make sure we give time for spinProcessBar to exit
qtyResults = len(sceneIDs)
stash.Trace(f"Found {qtyResults} scenes with tag ({duplicateMarkForDeletion}): sceneIDs = {sceneIDs}")
for sceneID in sceneIDs:
# stash.Trace(f"Getting scene data for scene ID {sceneID['id']}.")
QtyDup += 1
prgs = QtyDup / qtyResults
stash.Progress(QtyDup, qtyResults)
stash.progressBar(QtyDup, qtyResults)
scene = stash.find_scene(sceneID['id'])
if scene == None or len(scene) == 0:
stash.Warn(f"Could not get scene data for scene ID {sceneID['id']}.")
QtyFailedQuery += 1
continue
# stash.Log(f"scene={scene}")
DupFileName = scene['files'][0]['path']
DupFileNameOnly = pathlib.Path(DupFileName).stem
stash.Warn(f"Deleting duplicate '{DupFileName}'", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
if alternateTrashCanPath != "":
destPath = f"{alternateTrashCanPath }{os.sep}{DupFileNameOnly}"
if os.path.isfile(destPath):
destPath = f"{alternateTrashCanPath }{os.sep}_{time.time()}_{DupFileNameOnly}"
shutil.move(DupFileName, destPath)
elif moveToTrashCan:
sendToTrash(DupFileName)
result = stash.destroy_scene(scene['id'], delete_file=True)
stash.Trace(f"destroy_scene result={result} for file {DupFileName}", toAscii=True)
QtyDeleted += 1
stash.Log(f"QtyDup={QtyDup}, QtyDeleted={QtyDeleted}, QtyFailedQuery={QtyFailedQuery}", printTo=LOG_STASH_N_PLUGIN)
return
def testSetDupTagOnScene(sceneId):
scene = stash.find_scene(sceneId)
stash.Log(f"scene={scene}")
stash.Log(f"scene tags={scene['tags']}")
tag_ids = [dupTagId]
for tag in scene['tags']:
tag_ids = tag_ids + [tag['id']]
stash.Log(f"tag_ids={tag_ids}")
stash.update_scene({'id' : scene['id'], 'tag_ids' : tag_ids})
# stash.Trace(f"scene={scene}")
if clearTag:
tags = [int(item['id']) for item in scene["tags"] if item['id'] != tagId]
stash.TraceOnce(f"tagId={tagId}, len={len(tags)}, tags = {tags}")
dataDict = {'id' : scene['id']}
if addPrimaryDupPathToDetails:
sceneDetails = scene['details']
if sceneDetails.find(detailPrefix) == 0 and sceneDetails.find(detailPostfix) > 1:
Pos1 = sceneDetails.find(detailPrefix)
Pos2 = sceneDetails.find(detailPostfix)
sceneDetails = sceneDetails[0:Pos1] + sceneDetails[Pos2 + len(detailPostfix):]
dataDict.update({'details' : sceneDetails})
dataDict.update({'tag_ids' : tags})
stash.Log(f"Updating scene with {dataDict}")
stash.update_scene(dataDict)
# stash.removeTag(scene, duplicateMarkForDeletion)
QtyClearedTags += 1
else:
DupFileName = scene['files'][0]['path']
DupFileNameOnly = pathlib.Path(DupFileName).stem
stash.Warn(f"Deleting duplicate '{DupFileName}'", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
if alternateTrashCanPath != "":
destPath = f"{alternateTrashCanPath }{os.sep}{DupFileNameOnly}"
if os.path.isfile(destPath):
destPath = f"{alternateTrashCanPath }{os.sep}_{time.time()}_{DupFileNameOnly}"
shutil.move(DupFileName, destPath)
elif moveToTrashCan:
sendToTrash(DupFileName)
result = stash.destroy_scene(scene['id'], delete_file=True)
stash.Trace(f"destroy_scene result={result} for file {DupFileName}", toAscii=True)
QtyDeleted += 1
stash.Log(f"QtyDup={QtyDup}, QtyClearedTags={QtyClearedTags}, QtyDeleted={QtyDeleted}, QtyFailedQuery={QtyFailedQuery}", printTo=LOG_STASH_N_PLUGIN)
if doNotGeneratePhash == False and clearTag == False:
stash.metadata_generate({"phashes": True})
if stash.PLUGIN_TASK_NAME == "tag_duplicates_task":
mangeDupFiles(tagDuplicates=True, merge=mergeDupFilename)
stash.Trace(f"{stash.PLUGIN_TASK_NAME} EXIT")
elif stash.PLUGIN_TASK_NAME == "delete_tagged_duplicates_task":
deleteTagggedDuplicates()
manageTagggedDuplicates()
stash.Trace(f"{stash.PLUGIN_TASK_NAME} EXIT")
elif stash.PLUGIN_TASK_NAME == "delete_duplicates_task":
mangeDupFiles(deleteDup=True, merge=mergeDupFilename)
stash.Trace(f"{stash.PLUGIN_TASK_NAME} EXIT")
elif stash.PLUGIN_TASK_NAME == "clear_duplicate_tags_task":
manageTagggedDuplicates(clearTag=True)
stash.Trace(f"{stash.PLUGIN_TASK_NAME} EXIT")
elif stash.PLUGIN_TASK_NAME == "generate_phash_task":
stash.metadata_generate({"phashes": True})
stash.Trace(f"{stash.PLUGIN_TASK_NAME} EXIT")
elif parse_args.dup_tag:
mangeDupFiles(tagDuplicates=True, merge=mergeDupFilename)
stash.Trace(f"Tag duplicate EXIT")
elif parse_args.del_tag:
deleteTagggedDuplicates()
manageTagggedDuplicates()
stash.Trace(f"Delete Tagged duplicates EXIT")
elif parse_args.clear_tag:
manageTagggedDuplicates(clearTag=True)
stash.Trace(f"Clear duplicate tags EXIT")
elif parse_args.remove:
mangeDupFiles(deleteDup=True, merge=mergeDupFilename)
stash.Trace(f"Delete duplicate EXIT")

40
plugins/DupFileManager/DupFileManager.yml
View File

@@ -1,8 +1,12 @@
name: DupFileManager
description: Manages duplicate files.
version: 0.1.2
version: 0.1.4
url: https://github.com/David-Maisonave/Axter-Stash/tree/main/plugins/DupFileManager
settings:
doNotGeneratePhash:
displayName: Do Not Generate PHASH
description: Do not generate PHASH after tag or delete task.
type: BOOLEAN
mergeDupFilename:
displayName: Merge Duplicate Tags
description: Before deletion, merge metadata from duplicate. E.g. Tag names, performers, studios, title, galleries, rating, details, etc...
@@ -23,8 +27,20 @@ settings:
displayName: Run Clean After Delete
description: After running a 'Delete Duplicates' task, run Clean, Clean-Generated, and Optimize-Database.
type: BOOLEAN
zSwapBetterBitRate:
displayName: Swap Better Bit Rate
description: Swap better bit rate for duplicate files. Use with DupFileManager_config.py file option favorHighBitRate
type: BOOLEAN
zSwapBetterFrameRate:
displayName: Swap Better Frame Rate
description: Swap better frame rate for duplicates. Use with DupFileManager_config.py file option favorHigherFrameRate
type: BOOLEAN
zSwapCodec:
displayName: Swap Better Codec
description: If enabled, swap better codec duplicate files to preferred path.
type: BOOLEAN
zSwapHighRes:
displayName: Swap High Resolution
displayName: Swap Higher Resolution
description: If enabled, swap higher resolution duplicate files to preferred path.
type: BOOLEAN
zSwapLongLength:
@@ -37,19 +53,23 @@ settings:
type: STRING
zxGraylist:
displayName: Gray List
description: List of preferential paths to determine which duplicate should be the primary. E.g. C:\2nd_Favorite\,H:\ShouldKeep\
description: Preferential paths to determine which duplicate should be kept. E.g. C:\2nd_Fav,C:\3rd_Fav,C:\4th_Fav,H:\ShouldKeep
type: STRING
zyBlacklist:
displayName: Black List
description: List of LEAST preferential paths to determine primary candidates for deletion. E.g. C:\Downloads\,F:\DeleteMeFirst\
description: Least preferential paths; Determine primary deletion candidates. E.g. C:\Downloads,C:\DelMe-3rd,C:\DelMe-2nd,C:\DeleteMeFirst
type: STRING
zyMatchDupDistance:
displayName: Match Duplicate Distance
description: (Default=0) Where 0 = Exact Match, 1 = High Match, and 2 = Medium Match.
type: NUMBER
zyMaxDupToProcess:
displayName: Max Dup Process
description: Maximum number of duplicates to process. If 0, infinity
type: NUMBER
zzdebugTracing:
displayName: Debug Tracing
description: (Default=false) [***For Advanced Users***] Enable debug tracing. When enabled, additional tracing logging is added to Stash\plugins\DupFileManager\DupFileManager.log
description: Enable debug tracing so-as to add additional debug logging in Stash\plugins\DupFileManager\DupFileManager.log
type: BOOLEAN
exec:
- python
@@ -60,7 +80,11 @@ tasks:
description: Set tag DuplicateMarkForDeletion to the duplicates with lower resolution, duration, file name length, or black list path.
defaultArgs:
mode: tag_duplicates_task
- name: Delete Tagged Duplicates
- name: Clear Tags
description: Clear tag DuplicateMarkForDeletion. Remove the tag from all files.
defaultArgs:
mode: clear_duplicate_tags_task
- name: Delete Tagged Scenes
description: Only delete scenes having DuplicateMarkForDeletion tag.
defaultArgs:
mode: delete_tagged_duplicates_task
@@ -68,3 +92,7 @@ tasks:
description: Delete duplicate scenes. Performs deletion without first tagging.
defaultArgs:
mode: delete_duplicates_task
- name: Generate PHASH Matching
description: Generate PHASH file matching. Used for file comparisons.
defaultArgs:
mode: generate_phash_task

40
plugins/DupFileManager/DupFileManager_config.py
View File

@@ -8,19 +8,51 @@ config = {
"dup_path": "", #Example: "C:\\TempDeleteFolder"
# The threshold as to what percentage is consider a significant shorter time.
"significantTimeDiff" : .90, # 90% threshold
# Valued passed to stash API function FindDuplicateScenes.
"duration_diff" : 10, # (default=10) A value from 1 to 10.
# If enabled, moves destination file to recycle bin before swapping Hi-Res file.
"toRecycleBeforeSwap" : True,
# Character used to seperate items on the whitelist, blacklist, and graylist
"listSeparator" : ",",
# Tag used to tag duplicates with lower resolution, duration, and file name length.
"DupFileTag" : "DuplicateMarkForDeletion",
# Tag name used to tag duplicates in the whitelist. E.g. DuplicateWhitelistFile
"DupWhiteListTag" : "DuplicateWhitelistFile",
# Tag name used to tag duplicates in the whitelist. E.g. _DuplicateWhitelistFile
"DupWhiteListTag" : "_DuplicateWhitelistFile",
# Tags used to exclude duplicate from deletion
"excludeDupFileDeleteTag" : "_ExcludeDuplicateMarkForDeletion",
# If enabled, favor longer file name over shorter. If disabled, favor shorter file name.
"favorLongerFileName" : True,
# If enabled, favor larger file size over smaller. If disabled, favor smaller file size.
"favorLargerFileSize" : True,
# If enabled, favor videos with better codec according to codecRanking
"favorCodecRanking" : True,
# Codec Ranking in order of preference (default (codecRankingSet1) is order of ranking based on maximum potential efficiency)
"codecRankingSet1" : ["h266", "vvc", "av1", "vvdec", "shvc", "h265", "hevc", "xvc", "vp9", "h264", "avc", "mvc", "msmpeg4v10", "vp8", "vcb", "msmpeg4v3", "h263", "h263i", "msmpeg4v2", "msmpeg4v1", "mpeg4", "mpeg-4", "mpeg4video", "theora", "vc3", "vc-3", "vp7", "vp6f", "vp6", "vc1", "vc-1", "mpeg2", "mpeg-2", "mpeg2video", "h262", "h222", "h261", "vp5", "vp4", "vp3", "wmv3", "mpeg1", "mpeg-1", "mpeg1video", "vp3", "wmv2", "wmv1", "wmv", "flv1", "png", "gif", "jpeg", "m-jpeg", "mjpeg"],
# codecRankingSet2 is in order of least potential efficiency
"codecRankingSet2" : ["gif", "png", "flv1", "mpeg1video", "mpeg1", "wmv1", "wmv2", "wmv3", "mpeg2video", "mpeg2", "AVC", "vc1", "vc-1", "msmpeg4v1", "msmpeg4v2", "msmpeg4v3", "mpeg4", "vp6f", "vp8", "h263i", "h263", "h264", "h265", "av1", "vp9", "h266"],
# codecRankingSet3 is in order of quality
"codecRankingSet3" : ["h266", "vp9", "av1", "h265", "h264", "h263", "h263i", "vp8", "vp6f", "mpeg4", "msmpeg4v3", "msmpeg4v2", "msmpeg4v1", "vc-1", "vc1", "AVC", "mpeg2", "mpeg2video", "wmv3", "wmv2", "wmv1", "mpeg1", "mpeg1video", "flv1", "png", "gif"],
# codecRankingSet4 is in order of compatibility
"codecRankingSet4" : ["h264", "vp8", "mpeg4", "msmpeg4v3", "msmpeg4v2", "msmpeg4v1", "h266", "vp9", "av1", "h265", "h263", "h263i", "vp6f", "vc-1", "vc1", "AVC", "mpeg2", "mpeg2video", "wmv3", "wmv2", "wmv1", "mpeg1", "mpeg1video", "flv1", "png", "gif"],
# Determines which codecRankingSet to use when ranking codec. Default is 1 for codecRankingSet1
"codecRankingSetToUse" : 1,
# If enabled, favor videos with a different bit rate value. If favorHighBitRate is true, favor higher rate. If favorHighBitRate is false, favor lower rate
"favorBitRateChange" : True,
# If enabled, favor videos with higher bit rate. Used with either favorBitRateChange option or UI [Swap Bit Rate Change] option.
"favorHighBitRate" : True,
# If enabled, favor videos with a different frame rate value. If favorHigherFrameRate is true, favor higher rate. If favorHigherFrameRate is false, favor lower rate
"favorFrameRateChange" : True,
# If enabled, favor videos with higher frame rate. Used with either favorFrameRateChange option or UI [Swap Better Frame Rate] option.
"favorHigherFrameRate" : True,
# The following fields are ONLY used when running DupFileManager in script mode
"endpoint_Scheme" : "http", # Define endpoint to use when contacting the Stash server
"endpoint_Host" : "0.0.0.0", # Define endpoint to use when contacting the Stash server
"endpoint_Port" : 9999, # Define endpoint to use when contacting the Stash server
}
# Codec ranking research source:
# https://imagekit.io/blog/video-encoding/
# https://support.spinetix.com/wiki/Video_decoding
# https://en.wikipedia.org/wiki/Comparison_of_video_codecs
# https://en.wikipedia.org/wiki/List_of_open-source_codecs
# https://en.wikipedia.org/wiki/List_of_codecs
# https://en.wikipedia.org/wiki/Comparison_of_video_container_formats

8
plugins/DupFileManager/README.md
View File

@@ -1,4 +1,4 @@
# DupFileManager: Ver 0.1.2 (By David Maisonave)
# DupFileManager: Ver 0.1.4 (By David Maisonave)
DupFileManager is a [Stash](https://github.com/stashapp/stash) plugin which manages duplicate file in the Stash system.
@@ -31,9 +31,9 @@ DupFileManager is a [Stash](https://github.com/stashapp/stash) plugin which mana
### Requirements
`pip install --upgrade stashapp-tools`
`pip install pyYAML`
`pip install Send2Trash`
- `pip install --upgrade stashapp-tools`
- `pip install requests`
- `pip install Send2Trash`
### Installation

371
plugins/DupFileManager/StashPluginHelper.py
View File

@@ -1,6 +1,6 @@
from stashapi.stashapp import StashInterface
from logging.handlers import RotatingFileHandler
import re, inspect, sys, os, pathlib, logging, json
import re, inspect, sys, os, pathlib, logging, json, platform, subprocess, traceback, time
import concurrent.futures
from stashapi.stash_types import PhashDistance
import __main__
@@ -61,6 +61,14 @@ class StashPluginHelper(StashInterface):
LOG_FILE_DIR = None
LOG_FILE_NAME = None
STDIN_READ = None
stopProcessBarSpin = True
IS_DOCKER = False
IS_WINDOWS = False
IS_LINUX = False
IS_FREEBSD = False
IS_MAC_OS = False
pluginLog = None
logLinePreviousHits = []
thredPool = None
@@ -107,6 +115,16 @@ class StashPluginHelper(StashInterface):
DryRunFieldName = "zzdryRun",
setStashLoggerAsPluginLogger = False):
self.thredPool = concurrent.futures.ThreadPoolExecutor(max_workers=2)
if any(platform.win32_ver()):
self.IS_WINDOWS = True
elif platform.system().lower().startswith("linux"):
self.IS_LINUX = True
if self.isDocker():
self.IS_DOCKER = True
elif platform.system().lower().startswith("freebsd"):
self.IS_FREEBSD = True
elif sys.platform == "darwin":
self.IS_MAC_OS = True
if logToWrnSet: self.log_to_wrn_set = logToWrnSet
if logToErrSet: self.log_to_err_set = logToErrSet
if logToNormSet: self.log_to_norm = logToNormSet
@@ -300,37 +318,43 @@ class StashPluginHelper(StashInterface):
lineNo = inspect.currentframe().f_back.f_lineno
self.Log(logMsg, printTo, logging.ERROR, lineNo, toAscii=toAscii)
def Status(self, printTo = 0, logLevel = logging.INFO, lineNo = -1):
# Above logging functions all use UpperCamelCase naming convention to avoid conflict with parent class logging function names.
# The below non-loggging functions use (lower) camelCase naming convention.
def status(self, printTo = 0, logLevel = logging.INFO, lineNo = -1):
if printTo == 0: printTo = self.log_to_norm
if lineNo == -1:
lineNo = inspect.currentframe().f_back.f_lineno
self.Log(f"StashPluginHelper Status: (CALLED_AS_STASH_PLUGIN={self.CALLED_AS_STASH_PLUGIN}), (RUNNING_IN_COMMAND_LINE_MODE={self.RUNNING_IN_COMMAND_LINE_MODE}), (DEBUG_TRACING={self.DEBUG_TRACING}), (DRY_RUN={self.DRY_RUN}), (PLUGIN_ID={self.PLUGIN_ID}), (PLUGIN_TASK_NAME={self.PLUGIN_TASK_NAME}), (STASH_URL={self.STASH_URL}), (MAIN_SCRIPT_NAME={self.MAIN_SCRIPT_NAME})",
printTo, logLevel, lineNo)
def ExecuteProcess(self, args, ExecDetach=False):
import platform, subprocess
is_windows = any(platform.win32_ver())
def executeProcess(self, args, ExecDetach=False):
pid = None
self.Trace(f"is_windows={is_windows} args={args}")
if is_windows:
self.Trace(f"self.IS_WINDOWS={self.IS_WINDOWS} args={args}")
if self.IS_WINDOWS:
if ExecDetach:
self.Trace("Executing process using Windows DETACHED_PROCESS")
self.Trace(f"Executing process using Windows DETACHED_PROCESS; args=({args})")
DETACHED_PROCESS = 0x00000008
pid = subprocess.Popen(args,creationflags=DETACHED_PROCESS, shell=True).pid
else:
pid = subprocess.Popen(args, shell=True).pid
else:
self.Trace("Executing process using normal Popen")
pid = subprocess.Popen(args).pid
if ExecDetach:
# For linux detached, use nohup. I.E. subprocess.Popen(["nohup", "python", "test.py"])
if self.IS_LINUX:
args = ["nohup"] + args
self.Trace(f"Executing detached process using Popen({args})")
else:
self.Trace(f"Executing process using normal Popen({args})")
pid = subprocess.Popen(args).pid # On detach, may need the following for MAC OS subprocess.Popen(args, shell=True, start_new_session=True)
self.Trace(f"pid={pid}")
return pid
def ExecutePythonScript(self, args, ExecDetach=True):
def executePythonScript(self, args, ExecDetach=True):
PythonExe = f"{sys.executable}"
argsWithPython = [f"{PythonExe}"] + args
return self.ExecuteProcess(argsWithPython,ExecDetach=ExecDetach)
return self.executeProcess(argsWithPython,ExecDetach=ExecDetach)
def Submit(self, *args, **kwargs):
def submit(self, *args, **kwargs):
return self.thredPool.submit(*args, **kwargs)
def asc2(self, data, convertToAscii=None):
@@ -340,24 +364,214 @@ class StashPluginHelper(StashInterface):
# data = str(data).encode('ascii','ignore') # This works better for logging than ascii function
# return str(data)[2:-1] # strip out b'str'
def init_mergeMetadata(self, excludeMergeTags=None):
def initMergeMetadata(self, excludeMergeTags=None):
self.excludeMergeTags = excludeMergeTags
self._mergeMetadata = mergeMetadata(self, self.excludeMergeTags)
# Must call init_mergeMetadata, before calling merge_metadata
def merge_metadata(self, SrcData, DestData): # Input arguments can be scene ID or scene metadata
# Must call initMergeMetadata, before calling mergeMetadata
def mergeMetadata(self, SrcData, DestData): # Input arguments can be scene ID or scene metadata
if type(SrcData) is int:
SrcData = self.find_scene(SrcData)
DestData = self.find_scene(DestData)
return self._mergeMetadata.merge(SrcData, DestData)
def Progress(self, currentIndex, maxCount):
def progressBar(self, currentIndex, maxCount):
progress = (currentIndex / maxCount) if currentIndex < maxCount else (maxCount / currentIndex)
self.log.progress(progress)
def run_plugin(self, plugin_id, task_mode=None, args:dict={}, asyn=False):
# Test via command line: pip uninstall -y pyYAML watchdog schedule requests
def modulesInstalled(self, moduleNames, install=True, silent=False): # moduleNames=["stashapp-tools", "requests", "pyYAML"]
retrnValue = True
for moduleName in moduleNames:
try: # Try Python 3.3 > way
import importlib
import importlib.util
if moduleName in sys.modules:
if not silent: self.Trace(f"{moduleName!r} already in sys.modules")
elif self.isModuleInstalled(moduleName):
if not silent: self.Trace(f"Module {moduleName!r} is available.")
else:
if install and (results:=self.installModule(moduleName)) > 0:
if results == 1:
self.Log(f"Module {moduleName!r} has been installed")
else:
if not silent: self.Trace(f"Module {moduleName!r} is already installed")
continue
else:
if install:
self.Error(f"Can't find the {moduleName!r} module")
retrnValue = False
except Exception as e:
try:
i = importlib.import_module(moduleName)
except ImportError as e:
if install and (results:=self.installModule(moduleName)) > 0:
if results == 1:
self.Log(f"Module {moduleName!r} has been installed")
else:
if not silent: self.Trace(f"Module {moduleName!r} is already installed")
continue
else:
if install:
tb = traceback.format_exc()
self.Error(f"Can't find the {moduleName!r} module! Error: {e}\nTraceBack={tb}")
retrnValue = False
return retrnValue
def isModuleInstalled(self, moduleName):
try:
__import__(moduleName)
# self.Trace(f"Module {moduleName!r} is installed")
return True
except Exception as e:
tb = traceback.format_exc()
self.Warn(f"Module {moduleName!r} is NOT installed!")
self.Trace(f"Error: {e}\nTraceBack={tb}")
pass
return False
def installModule(self,moduleName):
# if not self.IS_DOCKER:
# try:
# self.Log(f"Attempting to install package {moduleName!r} using pip import method.")
# First try pip import method. (This may fail in a future version of pip.)
# self.installPackage(moduleName)
# self.Trace(f"installPackage called for module {moduleName!r}")
# if self.modulesInstalled(moduleNames=[moduleName], install=False):
# self.Trace(f"Module {moduleName!r} installed")
# return 1
# self.Trace(f"Module {moduleName!r} still not installed.")
# except Exception as e:
# tb = traceback.format_exc()
# self.Warn(f"pip import method failed for module {moduleName!r}. Will try command line method; Error: {e}\nTraceBack={tb}")
# pass
# else:
# self.Trace("Running in Docker, so skipping pip import method.")
try:
if self.IS_LINUX:
# Note: Linux may first need : sudo apt install python3-pip
# if error starts with "Command 'pip' not found"
# or includes "No module named pip"
self.Log("Checking if pip installed.")
results = os.popen(f"pip --version").read()
if results.find("Command 'pip' not found") != -1 or results.find("No module named pip") != -1:
results = os.popen(f"sudo apt install python3-pip").read()
results = os.popen(f"pip --version").read()
if results.find("Command 'pip' not found") != -1 or results.find("No module named pip") != -1:
self.Error(f"Error while calling 'pip'. Make sure pip is installed, and make sure module {moduleName!r} is installed. Results = '{results}'")
return -1
self.Trace("pip good.")
if self.IS_FREEBSD:
self.Warn("installModule may NOT work on freebsd")
pipArg = ""
if self.IS_DOCKER:
pipArg = " --break-system-packages"
self.Log(f"Attempting to install package {moduleName!r} via popen.")
results = os.popen(f"{sys.executable} -m pip install {moduleName}{pipArg}").read() # May need to be f"{sys.executable} -m pip install {moduleName}"
results = results.strip("\n")
self.Trace(f"pip results = {results}")
if results.find("Requirement already satisfied:") > -1:
self.Trace(f"Requirement already satisfied for module {moduleName!r}")
return 2
elif results.find("Successfully installed") > -1:
self.Trace(f"Successfully installed module {moduleName!r}")
return 1
elif self.modulesInstalled(moduleNames=[moduleName], install=False):
self.Trace(f"modulesInstalled returned True for module {moduleName!r}")
return 1
self.Error(f"Failed to install module {moduleName!r}")
except Exception as e:
tb = traceback.format_exc()
self.Error(f"Failed to install module {moduleName!r}. Error: {e}\nTraceBack={tb}")
return 0
def installPackage(self,package): # Should delete this. It doesn't work consistently
try:
import pip
if hasattr(pip, 'main'):
pip.main(['install', package])
self.Trace()
else:
pip._internal.main(['install', package])
self.Trace()
except Exception as e:
tb = traceback.format_exc()
self.Error(f"Failed to install module {moduleName!r}. Error: {e}\nTraceBack={tb}")
return False
return True
def isDocker(self):
cgroup = pathlib.Path('/proc/self/cgroup')
return pathlib.Path('/.dockerenv').is_file() or cgroup.is_file() and 'docker' in cgroup.read_text()
def spinProcessBar(self, sleepSeconds = 1, maxPos = 30, trace = False):
if trace:
self.Trace(f"Starting spinProcessBar loop; sleepSeconds={sleepSeconds}, maxPos={maxPos}")
pos = 1
while self.stopProcessBarSpin == False:
if trace:
self.Trace(f"progressBar({pos}, {maxPos})")
self.progressBar(pos, maxPos)
pos +=1
if pos > maxPos:
pos = 1
time.sleep(sleepSeconds)
def startSpinningProcessBar(self, sleepSeconds = 1, maxPos = 30, trace = False):
self.stopProcessBarSpin = False
if trace:
self.Trace(f"submitting spinProcessBar; sleepSeconds={sleepSeconds}, maxPos={maxPos}, trace={trace}")
self.submit(self.spinProcessBar, sleepSeconds, maxPos, trace)
def stopSpinningProcessBar(self, sleepSeconds = 1):
self.stopProcessBarSpin = True
time.sleep(sleepSeconds)
def createTagId(self, tagName, tagName_descp = "", deleteIfExist = False, ignoreAutoTag = False):
tagId = self.find_tags(q=tagName)
if len(tagId):
tagId = tagId[0]
if deleteIfExist:
self.destroy_tag(int(tagId['id']))
else:
return tagId['id']
tagId = self.create_tag({"name":tagName, "description":tagName_descp, "ignore_auto_tag": ignoreAutoTag})
self.Log(f"Dup-tagId={tagId['id']}")
return tagId['id']
def removeTag(self, scene, tagName): # scene can be scene ID or scene metadata
scene_details = scene
if 'id' not in scene:
scene_details = self.find_scene(scene)
tagIds = []
doesHaveTagName = False
for tag in scene_details['tags']:
if tag['name'] != tagName:
tagIds += [tag['id']]
else:
doesHaveTagName = True
if doesHaveTagName:
dataDict = {'id' : scene_details['id']}
dataDict.update({'tag_ids' : tagIds})
self.update_scene(dataDict)
return doesHaveTagName
def addTag(self, scene, tagName): # scene can be scene ID or scene metadata
scene_details = scene
if 'id' not in scene:
scene_details = self.find_scene(scene)
tagIds = [self.createTagId(tagName)]
for tag in scene_details['tags']:
if tag['name'] != tagName:
tagIds += [tag['id']]
dataDict = {'id' : scene_details['id']}
dataDict.update({'tag_ids' : tagIds})
self.update_scene(dataDict)
def runPlugin(self, plugin_id, task_mode=None, args:dict={}, asyn=False):
"""Runs a plugin operation.
The operation is run immediately and does not use the job queue.
This is a blocking call, and does not return until plugin completes.
Args:
plugin_id (ID): plugin_id
task_name (str, optional): Plugin task to perform
@@ -375,43 +589,26 @@ class StashPluginHelper(StashInterface):
"args": args,
}
if asyn:
self.Submit(self.call_GQL, query, variables)
self.submit(self.call_GQL, query, variables)
return f"Made asynchronous call for plugin {plugin_id}"
else:
return self.call_GQL(query, variables)
def find_duplicate_scenes_diff(self, distance: PhashDistance=PhashDistance.EXACT, fragment='id', duration_diff: float=10.00 ):
query = """
query FindDuplicateScenes($distance: Int, $duration_diff: Float) {
findDuplicateScenes(distance: $distance, duration_diff: $duration_diff) {
...SceneSlim
}
}
"""
if fragment:
query = re.sub(r'\.\.\.SceneSlim', fragment, query)
else:
query += "fragment SceneSlim on Scene { id }"
variables = { "distance": distance, "duration_diff": duration_diff }
result = self.call_GQL(query, variables)
return result['findDuplicateScenes']
# #################################################################################################
# ############################################################################################################
# Functions which are candidates to be added to parent class use snake_case naming convention.
# ############################################################################################################
# The below functions extends class StashInterface with functions which are not yet in the class or
# fixes for functions which have not yet made it into official class.
def metadata_scan(self, paths:list=[], flags={}):
def metadata_scan(self, paths:list=[], flags={}): # ToDo: Add option to add path to library if path not included when calling metadata_scan
query = "mutation MetadataScan($input:ScanMetadataInput!) { metadataScan(input: $input) }"
scan_metadata_input = {"paths": paths}
if flags:
scan_metadata_input.update(flags)
else:
scanData = self.get_configuration_defaults("scan { ...ScanMetadataOptions }")
if scanData['scan'] != None:
scan_metadata_input.update(scanData.get("scan",{}))
elif scan_config := self.get_configuration_defaults("scan { ...ScanMetadataOptions }").get("scan"):
scan_metadata_input.update(scan_config)
result = self.call_GQL(query, {"input": scan_metadata_input})
return result["metadataScan"]
def get_all_scenes(self):
query_all_scenes = """
query AllScenes {
@@ -464,6 +661,43 @@ class StashPluginHelper(StashInterface):
def rename_generated_files(self):
return self.call_GQL("mutation MigrateHashNaming {migrateHashNaming}")
def find_duplicate_scenes_diff(self, distance: PhashDistance=PhashDistance.EXACT, fragment='id', duration_diff: float=10.00 ):
query = """
query FindDuplicateScenes($distance: Int, $duration_diff: Float) {
findDuplicateScenes(distance: $distance, duration_diff: $duration_diff) {
...SceneSlim
}
}
"""
if fragment:
query = re.sub(r'\.\.\.SceneSlim', fragment, query)
else:
query += "fragment SceneSlim on Scene { id }"
variables = { "distance": distance, "duration_diff": duration_diff }
result = self.call_GQL(query, variables)
return result['findDuplicateScenes']
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# Direct SQL associated functions
def get_file_metadata(self, data, raw_data = False): # data is either file ID or scene metadata
results = None
if data == None:
return results
if 'files' in data and len(data['files']) > 0 and 'id' in data['files'][0]:
results = self.sql_query(f"select * from files where id = {data['files'][0]['id']}")
else:
results = self.sql_query(f"select * from files where id = {data}")
if raw_data:
return results
if 'rows' in results:
return results['rows'][0]
self.Error(f"Unknown error while SQL query with data='{data}'; Results='{results}'.")
return None
def set_file_basename(self, id, basename):
return self.sql_commit(f"update files set basename = '{basename}' where id = {id}")
class mergeMetadata: # A class to merge scene metadata from source scene to destination scene
srcData = None
@@ -537,3 +771,54 @@ class mergeMetadata: # A class to merge scene metadata from source scene to dest
listToAdd += [item['id']]
self.dataDict.update({ updateFieldName : listToAdd})
# self.stash.Trace(f"Added {fieldName} ({dataAdded}) to scene ID({self.destData['id']})", toAscii=True)
class taskQueue:
taskqueue = None
def __init__(self, taskqueue):
self.taskqueue = taskqueue
def tooManyScanOnTaskQueue(self, tooManyQty = 5):
count = 0
if self.taskqueue == None:
return False
for jobDetails in self.taskqueue:
if jobDetails['description'] == "Scanning...":
count += 1
if count < tooManyQty:
return False
return True
def cleanJobOnTaskQueue(self):
for jobDetails in self.taskqueue:
if jobDetails['description'] == "Cleaning...":
return True
return False
def cleanGeneratedJobOnTaskQueue(self):
for jobDetails in self.taskqueue:
if jobDetails['description'] == "Cleaning generated files...":
return True
return False
def isRunningPluginTaskJobOnTaskQueue(self, taskName):
for jobDetails in self.taskqueue:
if jobDetails['description'] == "Running plugin task: {taskName}":
return True
return False
def tagDuplicatesJobOnTaskQueue(self):
return self.isRunningPluginTaskJobOnTaskQueue("Tag Duplicates")
def clearDupTagsJobOnTaskQueue(self):
return self.isRunningPluginTaskJobOnTaskQueue("Clear Tags")
def generatePhashMatchingJobOnTaskQueue(self):
return self.isRunningPluginTaskJobOnTaskQueue("Generate PHASH Matching")
def deleteDuplicatesJobOnTaskQueue(self):
return self.isRunningPluginTaskJobOnTaskQueue("Delete Duplicates")
def deleteTaggedScenesJobOnTaskQueue(self):
return self.isRunningPluginTaskJobOnTaskQueue("Delete Tagged Scenes")

3
plugins/DupFileManager/requirements.txt
View File

@@ -1,4 +1,3 @@
stashapp-tools >= 0.2.50
pyYAML
watchdog
requests
Send2Trash