Fixed bug in RenameFile when running on Linux

This commit is contained in:
David Maisonave
2024-09-02 13:42:49 -04:00
parent 1e74a8b09c
commit 54c99a1564
10 changed files with 133 additions and 184 deletions

View File

@@ -7,13 +7,10 @@ import os, sys, time, pathlib, argparse, platform, shutil, logging
from StashPluginHelper import StashPluginHelper
from DupFileManager_config import config # Import config from DupFileManager_config.py
# ToDo: Add schedule for deletion date argument at command line. Variable can also be fetched from JSON_INPUT["args"]
# This variable will be used in function setTagId when initializing variable BaseDupStr
parser = argparse.ArgumentParser()
parser.add_argument('--url', '-u', dest='stash_url', type=str, help='Add Stash URL')
parser.add_argument('--trace', '-t', dest='trace', action='store_true', help='Enables debug trace mode.')
parser.add_argument('--add_dup_tag', '-a', dest='dup_tag', action='store_true', help='Set a tag to duplicate files.')
parser.add_argument('--clear_dup_tag', '-c', dest='clear_tag', action='store_true', help='Clear duplicates of duplicate tags.')
parser.add_argument('--del_tag_dup', '-d', dest='del_tag', action='store_true', help='Only delete scenes having DuplicateMarkForDeletion tag.')
parser.add_argument('--remove_dup', '-r', dest='remove', action='store_true', help='Remove (delete) duplicate files.')
parse_args = parser.parse_args()
@@ -63,8 +60,6 @@ swapLongLength = stash.Setting('zSwapLongLength')
significantTimeDiff = stash.Setting('significantTimeDiff')
toRecycleBeforeSwap = stash.Setting('toRecycleBeforeSwap')
cleanAfterDel = stash.Setting('zCleanAfterDel')
favorLongerFileName = float(stash.Setting('favorLongerFileName'))
favorLargerFileSize = float(stash.Setting('favorLargerFileSize'))
duration_diff = float(stash.Setting('duration_diff'))
if duration_diff > 10:
duration_diff = 10
@@ -187,16 +182,13 @@ def createTagId(tagName, tagName_descp, deleteIfExist = False):
stash.Log(f"Dup-tagId={tagId['id']}")
return tagId['id']
detailPrefix = "BaseDup="
detailPostfix = "<BaseDup>\n"
def setTagId(tagId, tagName, sceneDetails, DupFileToKeep):
details = ""
ORG_DATA_DICT = {'id' : sceneDetails['id']}
dataDict = ORG_DATA_DICT.copy()
doAddTag = True
if addPrimaryDupPathToDetails:
BaseDupStr = f"{detailPrefix}{DupFileToKeep['files'][0]['path']}\n{stash.STASH_URL}/scenes/{DupFileToKeep['id']}\n{detailPostfix}"
BaseDupStr = f"BaseDup={DupFileToKeep['files'][0]['path']}\n{stash.STASH_URL}/scenes/{DupFileToKeep['id']}\n"
if sceneDetails['details'] == "":
details = BaseDupStr
elif not sceneDetails['details'].startswith(BaseDupStr):
@@ -223,20 +215,6 @@ def isInList(listToCk, pathToCk):
return True
return False
NOT_IN_LIST = 65535
def indexInList(listToCk, pathToCk):
pathToCk = pathToCk.lower()
index = -1
lenItemMatch = 0
returnValue = NOT_IN_LIST
for item in listToCk:
index += 1
if pathToCk.startswith(item):
if len(item) > lenItemMatch: # Make sure the best match is selected by getting match with longest string.
lenItemMatch = len(item)
returnValue = index
return returnValue
def hasSameDir(path1, path2):
if pathlib.Path(path1).resolve().parent == pathlib.Path(path2).resolve().parent:
return True
@@ -280,21 +258,6 @@ def isSwapCandidate(DupFileToKeep, DupFile):
return True
return False
def isWorseKeepCandidate(DupFileToKeep, Scene):
if not isInList(whitelist, Scene['files'][0]['path']) and isInList(whitelist, DupFileToKeep['files'][0]['path']):
return True
if not isInList(graylist, Scene['files'][0]['path']) and isInList(graylist, DupFileToKeep['files'][0]['path']):
return True
if not isInList(blacklist, DupFileToKeep['files'][0]['path']) and isInList(blacklist, Scene['files'][0]['path']):
return True
if isInList(graylist, Scene['files'][0]['path']) and isInList(graylist, DupFileToKeep['files'][0]['path']) and indexInList(graylist, DupFileToKeep['files'][0]['path']) < indexInList(graylist, Scene['files'][0]['path']):
return True
if isInList(blacklist, DupFileToKeep['files'][0]['path']) and isInList(blacklist, Scene['files'][0]['path']) and indexInList(blacklist, DupFileToKeep['files'][0]['path']) < indexInList(blacklist, Scene['files'][0]['path']):
return True
return False
def mangeDupFiles(merge=False, deleteDup=False, tagDuplicates=False):
duplicateMarkForDeletion_descp = 'Tag added to duplicate scenes so-as to tag them for deletion.'
stash.Trace(f"duplicateMarkForDeletion = {duplicateMarkForDeletion}")
@@ -349,37 +312,18 @@ def mangeDupFiles(merge=False, deleteDup=False, tagDuplicates=False):
if significantLessTime(int(DupFileToKeep['files'][0]['duration']), int(Scene['files'][0]['duration'])):
QtyRealTimeDiff += 1
if int(DupFileToKeep['files'][0]['width']) < int(Scene['files'][0]['width']) or int(DupFileToKeep['files'][0]['height']) < int(Scene['files'][0]['height']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=resolution: {DupFileToKeep['files'][0]['width']}x{DupFileToKeep['files'][0]['height']} < {Scene['files'][0]['width']}x{Scene['files'][0]['height']}")
DupFileToKeep = Scene
elif int(DupFileToKeep['files'][0]['duration']) < int(Scene['files'][0]['duration']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=duration: {DupFileToKeep['files'][0]['duration']} < {Scene['files'][0]['duration']}")
DupFileToKeep = Scene
elif isInList(whitelist, Scene['files'][0]['path']) and not isInList(whitelist, DupFileToKeep['files'][0]['path']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=not whitelist vs whitelist")
DupFileToKeep = Scene
elif isInList(blacklist, DupFileToKeep['files'][0]['path']) and not isInList(blacklist, Scene['files'][0]['path']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=blacklist vs not blacklist")
DupFileToKeep = Scene
elif isInList(blacklist, DupFileToKeep['files'][0]['path']) and isInList(blacklist, Scene['files'][0]['path']) and indexInList(blacklist, DupFileToKeep['files'][0]['path']) > indexInList(blacklist, Scene['files'][0]['path']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=blacklist-index {indexInList(blacklist, DupFileToKeep['files'][0]['path'])} > {indexInList(blacklist, Scene['files'][0]['path'])}")
DupFileToKeep = Scene
elif isInList(graylist, Scene['files'][0]['path']) and not isInList(graylist, DupFileToKeep['files'][0]['path']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=not graylist vs graylist")
DupFileToKeep = Scene
elif isInList(graylist, Scene['files'][0]['path']) and isInList(graylist, DupFileToKeep['files'][0]['path']) and indexInList(graylist, DupFileToKeep['files'][0]['path']) > indexInList(graylist, Scene['files'][0]['path']):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=graylist-index {indexInList(graylist, DupFileToKeep['files'][0]['path'])} > {indexInList(graylist, Scene['files'][0]['path'])}")
elif len(DupFileToKeep['files'][0]['path']) < len(Scene['files'][0]['path']):
DupFileToKeep = Scene
elif favorLongerFileName and len(DupFileToKeep['files'][0]['path']) < len(Scene['files'][0]['path']) and not isWorseKeepCandidate(DupFileToKeep, Scene):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=path-len {len(DupFileToKeep['files'][0]['path'])} < {len(Scene['files'][0]['path'])}")
DupFileToKeep = Scene
elif favorLargerFileSize and int(DupFileToKeep['files'][0]['size']) < int(Scene['files'][0]['size']) and not isWorseKeepCandidate(DupFileToKeep, Scene):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=size {DupFileToKeep['files'][0]['size']} < {Scene['files'][0]['size']}")
DupFileToKeep = Scene
elif not favorLongerFileName and len(DupFileToKeep['files'][0]['path']) > len(Scene['files'][0]['path']) and not isWorseKeepCandidate(DupFileToKeep, Scene):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=path-len {len(DupFileToKeep['files'][0]['path'])} > {len(Scene['files'][0]['path'])}")
DupFileToKeep = Scene
elif not favorLargerFileSize and int(DupFileToKeep['files'][0]['size']) > int(Scene['files'][0]['size']) and not isWorseKeepCandidate(DupFileToKeep, Scene):
stash.Trace(f"Replacing {DupFileToKeep['files'][0]['path']} with {Scene['files'][0]['path']} for candidate to keep. Reason=size {DupFileToKeep['files'][0]['size']} > {Scene['files'][0]['size']}")
elif int(DupFileToKeep['files'][0]['size']) < int(Scene['files'][0]['size']):
DupFileToKeep = Scene
else:
DupFileToKeep = Scene
@@ -440,7 +384,7 @@ def mangeDupFiles(merge=False, deleteDup=False, tagDuplicates=False):
stash.metadata_clean_generated()
stash.optimise_database()
def manageTaggedDuplicates(deleteFiles=False):
def deleteTagggedDuplicates():
tagId = stash.find_tags(q=duplicateMarkForDeletion)
if len(tagId) > 0 and 'id' in tagId[0]:
tagId = tagId[0]['id']
@@ -465,34 +409,19 @@ def manageTaggedDuplicates(deleteFiles=False):
QtyFailedQuery += 1
continue
# stash.Log(f"scene={scene}")
if deleteFiles:
DupFileName = scene['files'][0]['path']
DupFileNameOnly = pathlib.Path(DupFileName).stem
stash.Warn(f"Deleting duplicate '{DupFileName}'", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
if alternateTrashCanPath != "":
destPath = f"{alternateTrashCanPath }{os.sep}{DupFileNameOnly}"
if os.path.isfile(destPath):
destPath = f"{alternateTrashCanPath }{os.sep}_{time.time()}_{DupFileNameOnly}"
shutil.move(DupFileName, destPath)
elif moveToTrashCan:
sendToTrash(DupFileName)
result = stash.destroy_scene(scene['id'], delete_file=True)
stash.Trace(f"destroy_scene result={result} for file {DupFileName}", toAscii=True)
QtyDeleted += 1
else:
tags = [int(item['id']) for item in scene["tags"] if item['id'] != tagId]
stash.TraceOnce(f"tagId={tagId}, len={len(tags)}, tags = {tags}")
dataDict = {'id' : scene['id']}
if addPrimaryDupPathToDetails:
sceneDetails = scene['details']
if sceneDetails.find(detailPrefix) == 0 and sceneDetails.find(detailPostfix) > 1:
Pos1 = sceneDetails.find(detailPrefix)
Pos2 = sceneDetails.find(detailPostfix)
sceneDetails = sceneDetails[0:Pos1] + sceneDetails[Pos2 + len(detailPostfix):]
dataDict.update({'details' : sceneDetails})
dataDict.update({'tag_ids' : tags})
stash.Log(f"Updating scene with {dataDict}")
stash.update_scene(dataDict)
DupFileName = scene['files'][0]['path']
DupFileNameOnly = pathlib.Path(DupFileName).stem
stash.Warn(f"Deleting duplicate '{DupFileName}'", toAscii=True, printTo=LOG_STASH_N_PLUGIN)
if alternateTrashCanPath != "":
destPath = f"{alternateTrashCanPath }{os.sep}{DupFileNameOnly}"
if os.path.isfile(destPath):
destPath = f"{alternateTrashCanPath }{os.sep}_{time.time()}_{DupFileNameOnly}"
shutil.move(DupFileName, destPath)
elif moveToTrashCan:
sendToTrash(DupFileName)
result = stash.destroy_scene(scene['id'], delete_file=True)
stash.Trace(f"destroy_scene result={result} for file {DupFileName}", toAscii=True)
QtyDeleted += 1
stash.Log(f"QtyDup={QtyDup}, QtyDeleted={QtyDeleted}, QtyFailedQuery={QtyFailedQuery}", printTo=LOG_STASH_N_PLUGIN)
return
@@ -510,10 +439,7 @@ if stash.PLUGIN_TASK_NAME == "tag_duplicates_task":
mangeDupFiles(tagDuplicates=True, merge=mergeDupFilename)
stash.Trace(f"{stash.PLUGIN_TASK_NAME} EXIT")
elif stash.PLUGIN_TASK_NAME == "delete_tagged_duplicates_task":
manageTaggedDuplicates(deleteFiles=True)
stash.Trace(f"{stash.PLUGIN_TASK_NAME} EXIT")
elif stash.PLUGIN_TASK_NAME == "clear_duplicate_tags_task":
manageTaggedDuplicates(deleteFiles=False)
deleteTagggedDuplicates()
stash.Trace(f"{stash.PLUGIN_TASK_NAME} EXIT")
elif stash.PLUGIN_TASK_NAME == "delete_duplicates_task":
mangeDupFiles(deleteDup=True, merge=mergeDupFilename)
@@ -522,11 +448,8 @@ elif parse_args.dup_tag:
mangeDupFiles(tagDuplicates=True, merge=mergeDupFilename)
stash.Trace(f"Tag duplicate EXIT")
elif parse_args.del_tag:
manageTaggedDuplicates(deleteFiles=True)
stash.Trace(f"Delete tagged duplicates EXIT")
elif parse_args.clear_tag:
manageTaggedDuplicates(deleteFiles=False)
stash.Trace(f"Clear duplicate tags EXIT")
deleteTagggedDuplicates()
stash.Trace(f"Delete Tagged duplicates EXIT")
elif parse_args.remove:
mangeDupFiles(deleteDup=True, merge=mergeDupFilename)
stash.Trace(f"Delete duplicate EXIT")

View File

@@ -1,6 +1,6 @@
name: DupFileManager
description: Manages duplicate files.
version: 0.1.3
version: 0.1.2
url: https://github.com/David-Maisonave/Axter-Stash/tree/main/plugins/DupFileManager
settings:
mergeDupFilename:
@@ -37,11 +37,11 @@ settings:
type: STRING
zxGraylist:
displayName: Gray List
description: Preferential paths to determine which duplicate should be kept. E.g. C:\2nd_Fav,C:\3rd_Fav,C:\4th_Fav,H:\ShouldKeep
description: List of preferential paths to determine which duplicate should be the primary. E.g. C:\2nd_Favorite\,H:\ShouldKeep\
type: STRING
zyBlacklist:
displayName: Black List
description: Least preferential paths; Determine primary deletion candidates. E.g. C:\Downloads,C:\DelMe-3rd,C:\DelMe-2nd,C:\DeleteMeFirst
description: List of LEAST preferential paths to determine primary candidates for deletion. E.g. C:\Downloads\,F:\DeleteMeFirst\
type: STRING
zyMaxDupToProcess:
displayName: Max Dup Process
@@ -49,7 +49,7 @@ settings:
type: NUMBER
zzdebugTracing:
displayName: Debug Tracing
description: Enable debug tracing so-as to add additional debug logging in Stash\plugins\DupFileManager\DupFileManager.log
description: (Default=false) [***For Advanced Users***] Enable debug tracing. When enabled, additional tracing logging is added to Stash\plugins\DupFileManager\DupFileManager.log
type: BOOLEAN
exec:
- python
@@ -68,7 +68,3 @@ tasks:
description: Delete duplicate scenes. Performs deletion without first tagging.
defaultArgs:
mode: delete_duplicates_task
- name: Clear Duplicate Tags
description: Removes DuplicateMarkForDeletion tag from all files.
defaultArgs:
mode: clear_duplicate_tags_task

View File

@@ -18,10 +18,6 @@ config = {
"DupFileTag" : "DuplicateMarkForDeletion",
# Tag name used to tag duplicates in the whitelist. E.g. DuplicateWhitelistFile
"DupWhiteListTag" : "DuplicateWhitelistFile",
# If enabled, favor longer file name over shorter. If disabled, favor shorter file name.
"favorLongerFileName" : True,
# If enabled, favor larger file size over smaller. If disabled, favor smaller file size.
"favorLargerFileSize" : True,
# The following fields are ONLY used when running DupFileManager in script mode
"endpoint_Scheme" : "http", # Define endpoint to use when contacting the Stash server

View File

@@ -1,10 +1,11 @@
# DupFileManager: Ver 0.1.3 (By David Maisonave)
# DupFileManager: Ver 0.1.2 (By David Maisonave)
DupFileManager is a [Stash](https://github.com/stashapp/stash) plugin which manages duplicate file in the Stash system.
<img width="500" alt="DupFileManager_Task_UI" src="https://github.com/user-attachments/assets/d25fd76b-4624-401b-bb12-091a4dadbe0c">
### Features
- Can merge potential source in the duplicate file names for tag names, performers, and studios.
- Normally when Stash searches the file name for tag names, performers, and studios, it only does so using the primary file.
- Delete duplicate file task with the following options:
- Tasks (Settings->Task->[Plugin Tasks]->DupFileManager)
- **Tag Duplicates** - Set tag DuplicateMarkForDeletion to the duplicates with lower resolution, duration, file name length, and/or black list path.
@@ -14,21 +15,19 @@ DupFileManager is a [Stash](https://github.com/stashapp/stash) plugin which mana
- Has a 3 tier path selection to determine which duplicates to keep, and which should be candidates for deletions.
- **Whitelist** - List of paths NOT to be deleted.
- E.g. C:\Favorite\,E:\MustKeep\
- **Gray-List** - List of preferential paths to determine which duplicate should be the primary. The list should be in order of most preferable in the beginning of the list.
- E.g. C:\2nd_Fav,C:\3rd_Fav,C:\4th_Fav,H:\ShouldKeep
- **Blacklist** - List of LEAST preferential paths to determine primary candidates for deletion. The list should be in order of least preferable at the end.
- E.g. C:\Downloads,C:\DeleteMe-3rd,C:\DeleteMe-2nd,C:\DeleteMeFirst
- **Gray-List** - List of preferential paths to determine which duplicate should be the primary.
- E.g. C:\2nd_Favorite\,H:\ShouldKeep\
- **Blacklist** - List of LEAST preferential paths to determine primary candidates for deletion.
- E.g. C:\Downloads\,F:\DeleteMeFirst\
- **Permanent Delete** - Enable to permanently delete files, instead of moving files to trash can.
- **Max Dup Process** - Use to limit the maximum files to process. Can be used to do a limited test run.
galleries, rating, details, etc...
- **Merge Duplicate Tags** - Before deletion, merge metadata from duplicate. E.g. Tag names, performers, studios, title, galleries, rating, details, etc...
- **Swap High Resolution** - When enabled, swaps higher resolution files between whitelist and blacklist/graylist files.
- **Swap Longer Duration** - When enabled, swaps scene with longer duration.
- Options available via DupFileManager_config.py
- **dup_path** - Alternate path to move deleted files to. Example: "C:\TempDeleteFolder"
- **toRecycleBeforeSwap** - When enabled, moves destination file to recycle bin before swapping files.
- **addPrimaryDupPathToDetails** - If enabled, adds the primary duplicate path to the scene detail.
- Optionally merge metadata between duplicates before file deletion. (tag names, performers, studios, etc...)
- <img width="500" alt="MergeMetadataOption" src="https://github.com/user-attachments/assets/73ca7775-37e4-4409-8dac-1ddc7f31415d">
### Requirements

View File

@@ -1,31 +1,29 @@
"""
StashPluginHelper (By David Maisonave aka Axter)
See end of this file for example usage
Log Features:
Can optionally log out to multiple outputs for each Log or Trace call.
Logging includes source code line number
Sets a maximum plugin log file size
Stash Interface Features:
Gets STASH_URL value from command line argument and/or from STDIN_READ
Sets FRAGMENT_SERVER based on command line arguments or STDIN_READ
Sets PLUGIN_ID based on the main script file name (in lower case)
Gets PLUGIN_TASK_NAME value
Sets pluginSettings (The plugin UI settings)
Misc Features:
Gets DRY_RUN value from command line argument and/or from UI and/or from config file
Gets DEBUG_TRACING value from command line argument and/or from UI and/or from config file
Sets RUNNING_IN_COMMAND_LINE_MODE to True if detects multiple arguments
Sets CALLED_AS_STASH_PLUGIN to True if it's able to read from STDIN_READ
"""
from stashapi.stashapp import StashInterface
from logging.handlers import RotatingFileHandler
import re, inspect, sys, os, pathlib, logging, json, ctypes
import re, inspect, sys, os, pathlib, logging, json
import concurrent.futures
from stashapi.stash_types import PhashDistance
import __main__
_ARGUMENT_UNSPECIFIED_ = "_ARGUMENT_UNSPECIFIED_"
# StashPluginHelper (By David Maisonave aka Axter)
# See end of this file for example usage
# Log Features:
# Can optionally log out to multiple outputs for each Log or Trace call.
# Logging includes source code line number
# Sets a maximum plugin log file size
# Stash Interface Features:
# Gets STASH_URL value from command line argument and/or from STDIN_READ
# Sets FRAGMENT_SERVER based on command line arguments or STDIN_READ
# Sets PLUGIN_ID based on the main script file name (in lower case)
# Gets PLUGIN_TASK_NAME value
# Sets pluginSettings (The plugin UI settings)
# Misc Features:
# Gets DRY_RUN value from command line argument and/or from UI and/or from config file
# Gets DEBUG_TRACING value from command line argument and/or from UI and/or from config file
# Sets RUNNING_IN_COMMAND_LINE_MODE to True if detects multiple arguments
# Sets CALLED_AS_STASH_PLUGIN to True if it's able to read from STDIN_READ
class StashPluginHelper(StashInterface):
# Primary Members for external reference
PLUGIN_TASK_NAME = None
@@ -400,7 +398,20 @@ class StashPluginHelper(StashInterface):
return result['findDuplicateScenes']
# #################################################################################################
# The below functions extends class StashInterface with functions which are not yet in the class
# The below functions extends class StashInterface with functions which are not yet in the class or
# fixes for functions which have not yet made it into official class.
def metadata_scan(self, paths:list=[], flags={}):
query = "mutation MetadataScan($input:ScanMetadataInput!) { metadataScan(input: $input) }"
scan_metadata_input = {"paths": paths}
if flags:
scan_metadata_input.update(flags)
else:
scanData = self.get_configuration_defaults("scan { ...ScanMetadataOptions }")
if scanData['scan'] != None:
scan_metadata_input.update(scanData.get("scan",{}))
result = self.call_GQL(query, {"input": scan_metadata_input})
return result["metadataScan"]
def get_all_scenes(self):
query_all_scenes = """
query AllScenes {