can do this to turn a loose file mod into a pak file mod
Source Code
► resource_overwriter_gui.py
Code: Select all
import os
import sys
import re
from pathlib import Path
from lxml import etree
import glob
import shutil
import requests
import subprocess
from bs4 import BeautifulSoup
from PyQt6.QtWidgets import (QApplication, QMainWindow, QWidget, QVBoxLayout,
QHBoxLayout, QPushButton, QLabel, QFileDialog,
QProgressBar, QTextEdit, QMessageBox, QCheckBox)
from PyQt6.QtCore import Qt, QThread, pyqtSignal
from PyQt6.QtGui import QColor, QTextCharFormat
class ResourceOverwriterWorker(QThread):
"""Worker thread to handle the resource overwriting process"""
progress_update = pyqtSignal(str, str) # message, color
progress_value = pyqtSignal(int)
progress_max = pyqtSignal(int)
finished = pyqtSignal()
def __init__(self, unpacked_data_folder, vanilla_overrides_folder, skip_sort=False):
super().__init__()
self.unpacked_data_folder = unpacked_data_folder
self.vanilla_overrides_folder = vanilla_overrides_folder
self.skip_sort = skip_sort
self.processed_files = 0
self.total_files = 0
self.successful_extractions = 0
self.failed_extractions = 0
self.extracted_files = []
self.mod_folder_name = None
def find_meta_lsx(self):
"""Find meta.lsx file and extract the Folder value"""
try:
# First, look specifically in the Mods directory (common BG3 mod structure)
mods_dir = os.path.join(self.unpacked_data_folder, "Mods")
if os.path.exists(mods_dir):
self.progress_update.emit(f"Looking for meta.lsx in Mods directory: {mods_dir}", "black")
# Look in each subfolder of Mods for a meta.lsx file
for mod_folder in os.listdir(mods_dir):
mod_path = os.path.join(mods_dir, mod_folder)
if os.path.isdir(mod_path):
meta_path = os.path.join(mod_path, "meta.lsx")
if os.path.exists(meta_path):
self.progress_update.emit(f"Found meta.lsx at: {meta_path}", "green")
# Parse the meta.lsx file to extract the Folder value
try:
tree = etree.parse(meta_path)
folder_attrs = tree.xpath("//attribute[@id='Folder']")
if folder_attrs and len(folder_attrs) > 0:
folder_value = folder_attrs[0].get("value")
if folder_value:
self.progress_update.emit(f"Extracted Folder value: {folder_value}", "green")
return folder_value
else:
self.progress_update.emit("Error: Folder attribute has empty value", "red")
else:
self.progress_update.emit("Error: No Folder attribute found in meta.lsx", "red")
except Exception as e:
self.progress_update.emit(f"Error parsing meta.lsx: {str(e)}", "red")
# If not found in Mods directory, search the entire directory structure
self.progress_update.emit("Looking for meta.lsx in the entire directory structure...", "black")
for root, _, files in os.walk(self.unpacked_data_folder):
for file in files:
if file.lower() == "meta.lsx":
meta_path = os.path.join(root, file)
self.progress_update.emit(f"Found meta.lsx at: {meta_path}", "green")
# Parse the meta.lsx file to extract the Folder value
try:
tree = etree.parse(meta_path)
folder_attrs = tree.xpath("//attribute[@id='Folder']")
if folder_attrs and len(folder_attrs) > 0:
folder_value = folder_attrs[0].get("value")
if folder_value:
self.progress_update.emit(f"Extracted Folder value: {folder_value}", "green")
return folder_value
else:
self.progress_update.emit("Error: Folder attribute has empty value", "red")
else:
self.progress_update.emit("Error: No Folder attribute found in meta.lsx", "red")
except Exception as e:
self.progress_update.emit(f"Error parsing meta.lsx: {str(e)}", "red")
self.progress_update.emit("Error: Could not find meta.lsx file with a valid Folder attribute", "red")
return None
except Exception as e:
self.progress_update.emit(f"Error finding meta.lsx: {str(e)}", "red")
return None
def run(self):
try:
# Find the mod folder name from meta.lsx
self.mod_folder_name = self.find_meta_lsx()
if not self.mod_folder_name:
self.progress_update.emit("Error: Could not determine mod folder name from meta.lsx", "red")
self.finished.emit()
return
# Verify the Generated folder exists
if not os.path.exists(self.vanilla_overrides_folder):
self.progress_update.emit(f"Error: Generated folder does not exist: {self.vanilla_overrides_folder}", "red")
self.finished.emit()
return
self.progress_update.emit(f"Scanning files in: {self.vanilla_overrides_folder}", "black")
# Get all files in Generated directory and its subdirectories
all_files = []
for root, _, files in os.walk(self.vanilla_overrides_folder):
for file in files:
# Only include relevant file types
if file.lower().endswith(('.gr2', '.dds', '.lsf', '.gtp', '.tga')):
file_path = os.path.join(root, file)
all_files.append(file_path)
if not all_files:
self.progress_update.emit(f"Error: No supported files found in {self.vanilla_overrides_folder}", "red")
self.finished.emit()
return
self.total_files = len(all_files)
self.progress_max.emit(self.total_files)
self.progress_update.emit(f"Found {self.total_files} files to process", "black")
# Process each file
for file_path in all_files:
file_name = os.path.basename(file_path)
rel_path = os.path.relpath(file_path, self.vanilla_overrides_folder)
self.progress_update.emit(f"\nProcessing file: {file_name}", "black")
self.progress_update.emit(f" Path: {rel_path}", "black")
# Copy Generated file to Generated/Public/FolderName/
generated_dest_path = os.path.join(self.unpacked_data_folder, "Generated", "Public", self.mod_folder_name, rel_path)
# Create destination directory if it doesn't exist
os.makedirs(os.path.dirname(generated_dest_path), exist_ok=True)
try:
# Copy the vanilla file to the Generated folder
shutil.copy2(file_path, generated_dest_path)
self.progress_update.emit(f" Copied vanilla file to Generated/Public/{self.mod_folder_name}/{rel_path}", "green")
except Exception as e:
self.progress_update.emit(f" Error copying file to Generated folder: {str(e)}", "red")
self.extract_lsx_for_file(file_name)
self.processed_files += 1
self.progress_value.emit(self.processed_files)
# Generate summary
self.progress_update.emit("\n--- Processing Summary ---", "black")
self.progress_update.emit(f"Files processed: {self.processed_files}", "black")
self.progress_update.emit(f"Successful extractions: {self.successful_extractions}", "green")
self.progress_update.emit(f"Failed extractions: {self.failed_extractions}", "red")
if self.extracted_files:
self.progress_update.emit("\nExtracted files:", "black")
for file_path in self.extracted_files:
self.progress_update.emit(f" - {os.path.basename(file_path)}", "green")
# Final step: Run sort_lsx_files.py on the Public\[ModFolder] directory
if not self.skip_sort:
self.progress_update.emit("\nRunning LSX file sorter as final step...", "blue")
self.run_sort_lsx_files()
else:
self.progress_update.emit("\nSkipping LSX file sorting step (disabled by user)", "blue")
self.progress_update.emit("\nProcessing complete.", "black")
self.finished.emit()
except Exception as e:
self.progress_update.emit(f"Error: {str(e)}", "red")
self.finished.emit()
def extract_lsx_for_file(self, file_name):
"""Extract LSX data for a specific file"""
try:
# Search for the specific file to get its LSX
search_url = f"https://bg3.norbyte.dev/search?q={file_name}"
self.progress_update.emit(f" Searching for LSX data at: {search_url}", "black")
# Set up a proper user agent to avoid being blocked
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Referer': 'https://bg3.norbyte.dev/'
}
try:
# Add timeout to avoid hanging forever
response = requests.get(search_url, headers=headers, timeout=30)
response.raise_for_status() # Raise an exception for HTTP errors
except requests.exceptions.RequestException as e:
self.progress_update.emit(f" Error: Failed to connect to bg3.norbyte.dev: {str(e)}", "red")
self.failed_extractions += 1
return
if response.status_code != 200:
self.progress_update.emit(f" Error: Search failed with status code {response.status_code}", "red")
self.failed_extractions += 1
return
# Debug the response content
html_content = response.text
if not html_content or len(html_content) < 100:
self.progress_update.emit(f" Error: Received empty or truncated response from server", "red")
self.progress_update.emit(f" Response content: {html_content[:100]}", "red")
self.failed_extractions += 1
return
# Log response content for debugging
self.progress_update.emit(f" Retrieved HTML content of length: {len(html_content)}", "black")
# Parse the HTML response
soup = BeautifulSoup(html_content, 'html.parser')
# APPROACH 1: Look for the LSX content directly
# Check for pre elements that might contain the LSX XML
pre_elements = soup.find_all('pre')
lsx_content = None
for pre in pre_elements:
content = pre.get_text()
# Check if this looks like LSX content (contains XML-like or LSX-specific tags)
if content and ('<?xml' in content or '<save>' in content or '<region' in content):
lsx_content = content
self.progress_update.emit(" Found LSX content directly in pre element", "green")
break
# APPROACH 2: If not found, look for the copy button and associated data
if not lsx_content:
# Find all buttons that might be the "Copy LSX" button
buttons = soup.find_all('a', {'class': 'btn'})
copy_button = None
for button in buttons:
button_text = button.get_text().strip()
if 'Copy LSX' in button_text:
copy_button = button
self.progress_update.emit(f" Found 'Copy LSX' button: {button_text}", "green")
break
if copy_button:
# Get the data-copy-source attribute which points to the content ID
copy_source_id = copy_button.get('data-copy-source')
if copy_source_id:
# Find the element that contains the actual content
content_elem = soup.find(id=copy_source_id)
if content_elem:
lsx_content = content_elem.get_text()
self.progress_update.emit(f" Found LSX content from button reference", "green")
# APPROACH 3: Look for direct data attributes
if not lsx_content:
elements_with_data = soup.find_all(attrs={"data-copy-content": True})
for elem in elements_with_data:
content = elem.get('data-copy-content')
if content and len(content) > 50: # Minimum size to be an LSX
lsx_content = content
self.progress_update.emit(" Found LSX content in data-copy-content attribute", "green")
break
# APPROACH 4: Extract from JavaScript data
if not lsx_content:
script_tags = soup.find_all('script')
for script in script_tags:
if script.string:
# Look for JS variable assignments that might contain the LSX
js_content = script.string
# Common patterns for embedded data
lsx_matches = re.findall(r'(raw-body-[^"\']+)["\']?\s*:\s*["\']([^"\']+)', js_content)
if lsx_matches:
for match_id, match_content in lsx_matches:
if match_content and len(match_content) > 50:
lsx_content = match_content
self.progress_update.emit(f" Found LSX content in JavaScript data", "green")
break
# APPROACH 5: If still not found, try looking at the page more broadly
if not lsx_content:
# Look for any element with 'raw-body' in its ID
raw_elements = soup.find_all(id=re.compile(r'raw-body-.*'))
for elem in raw_elements:
content = elem.get_text()
if content and len(content) > 50:
lsx_content = content
self.progress_update.emit(" Found LSX content in raw-body element", "green")
break
# Handle case where no LSX content could be found
if not lsx_content:
# Save the HTML for debugging
debug_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "debug_response.html")
with open(debug_path, 'w', encoding='utf-8') as f:
f.write(html_content)
self.progress_update.emit(f" Error: Could not extract LSX content (debug saved to {debug_path})", "red")
self.failed_extractions += 1
return
# Validate the LSX content
if len(lsx_content) < 50 or ('<' not in lsx_content or '>' not in lsx_content):
self.progress_update.emit(" Error: Extracted content doesn't appear to be valid LSX", "red")
self.progress_update.emit(f" Content (first 100 chars): {lsx_content[:100]}", "red")
self.failed_extractions += 1
return
self.progress_update.emit(" Successfully extracted LSX content", "green")
# Extract the filename from the file name
filename_base = os.path.splitext(os.path.basename(file_name))[0]
file_extension = os.path.splitext(os.path.basename(file_name))[1].lower()
# Step 1: First create the files in Public/FolderName/
temp_output_path = os.path.join(self.unpacked_data_folder, "Public", self.mod_folder_name)
self.progress_update.emit(f" Using mod folder: {self.mod_folder_name}", "blue")
# Create the temporary output directory structure
if not os.path.exists(temp_output_path):
try:
os.makedirs(temp_output_path, exist_ok=True)
self.progress_update.emit(f" Created directory structure: {temp_output_path}", "green")
except Exception as e:
self.progress_update.emit(f" Error creating directory structure: {str(e)}", "red")
self.failed_extractions += 1
return
# Step 2: Prepare the final output directory in Generated/Public/FolderName/
final_output_path = os.path.join(self.unpacked_data_folder, "Generated", "Public", self.mod_folder_name)
# Create the final output directory structure
if not os.path.exists(final_output_path):
try:
os.makedirs(final_output_path, exist_ok=True)
self.progress_update.emit(f" Created final directory structure: {final_output_path}", "green")
except Exception as e:
self.progress_update.emit(f" Error creating final directory structure: {str(e)}", "red")
self.failed_extractions += 1
return
# Determine if this will be a VisualBank (GR2) file or a regular overwrite
is_visual_bank = False
# Find the Resource node to possibly wrap
resource_node_match = None
if '<node id="Resource">' in lsx_content:
# Try to extract just the Resource node
resource_node_match = re.search(r'<node id="Resource">(.*?)</node>', lsx_content, re.DOTALL)
# Check if we need to wrap the content in a template based on file type
if file_extension == '.gr2' or file_extension == '.dds':
# Instead of extracting attributes, just insert the literal clipboard (LSX content) into <children>
if file_extension == '.gr2':
self.progress_update.emit(" Using VisualBank template for GR2 file", "blue")
lsx_content = f'''<?xml version="1.0" encoding="utf-8"?>
<save>
<version major="4" minor="0" revision="9" build="0" lslib_meta="v1,bswap_guids" />
<region id="VisualBank">
<node id="VisualBank">
<children>
{lsx_content}
</children>
</node>
</region>
</save>'''
is_visual_bank = True
elif file_extension == '.dds':
self.progress_update.emit(" Using TextureBank template for DDS file", "blue")
lsx_content = f'''<?xml version="1.0" encoding="utf-8"?>
<save>
<version major="4" minor="0" revision="9" build="0" lslib_meta="v1,bswap_guids" />
<region id="TextureBank">
<node id="TextureBank">
<children>
{lsx_content}
</children>
</node>
</region>
</save>'''
# Extract UUID or ID for filename
uuid_match = re.search(r'<attribute id="UUID" type="FixedString" value="([^"]+)"', lsx_content)
id_match = re.search(r'<attribute id="ID" type="FixedString" value="([^"]+)"', lsx_content)
if uuid_match:
filename = f"{uuid_match.group(1)}.lsf.lsx"
self.progress_update.emit(f" Found UUID: {uuid_match.group(1)}", "green")
identifier = uuid_match.group(1)
elif id_match:
filename = f"{id_match.group(1)}.lsf.lsx"
self.progress_update.emit(f" Found ID: {id_match.group(1)}", "green")
identifier = id_match.group(1)
else:
# Use the file name as a base for the filename
filename = f"{filename_base}.lsf.lsx"
self.progress_update.emit(f" No UUID/ID found, using filename: {filename_base}", "orange")
identifier = filename_base
# Delete existing LSX files based on two conditions:
# 1. Files containing the same UUID/ID in filename
# 2. Any LSX files containing "Generated" string
deleted_count = 0
for root, _, files in os.walk(final_output_path):
for file in files:
existing_file_path = os.path.join(root, file)
delete_file = False
reason = ""
# Condition 1: Filename contains the identifier/UUID
if identifier in file:
delete_file = True
reason = "matching UUID/ID in filename"
# Condition 2: File contains "Generated" string (for .lsx files only)
if file.endswith('.lsx') and not delete_file:
try:
with open(existing_file_path, 'r', encoding='utf-8') as f:
content = f.read()
if "Generated" in content:
delete_file = True
reason = "containing Generated reference"
except Exception as e:
self.progress_update.emit(f" Warning: Could not read file {existing_file_path}: {str(e)}", "orange")
# Delete the file if either condition is met
if delete_file:
try:
os.remove(existing_file_path)
deleted_count += 1
self.progress_update.emit(f" Deleted file {reason}: {existing_file_path}", "blue")
except Exception as e:
self.progress_update.emit(f" Warning: Could not delete file {existing_file_path}: {str(e)}", "orange")
if deleted_count > 0:
self.progress_update.emit(f" Deleted {deleted_count} existing files based on UUID or Generated references", "blue")
# Step 1: Save the LSX content to the temporary file location first
temp_file_path = os.path.join(temp_output_path, filename)
self.progress_update.emit(f" Saving LSX file to: {temp_file_path}", "blue")
try:
# Ensure directory exists
os.makedirs(os.path.dirname(temp_file_path), exist_ok=True)
# Save the file to temporary location
with open(temp_file_path, 'w', encoding='utf-8') as f:
f.write(lsx_content)
self.progress_update.emit(f" Successfully saved LSX content to: {temp_file_path}", "green")
# Add to the list of successfully created files
self.successful_extractions += 1
self.extracted_files.append(temp_file_path)
self.progress_update.emit(f" Completed processing: file saved to {temp_file_path}", "green")
except Exception as e:
self.progress_update.emit(f" Error saving LSX file: {str(e)}", "red")
self.failed_extractions += 1
return
# Update all file path attributes to use the mod folder instead of Shared or Generated
try:
tree = etree.parse(temp_file_path)
# Select every attribute node that has a 'value' attribute
attribute_nodes = tree.xpath("//attribute[@value]")
for attr in attribute_nodes:
old_value = attr.get("value")
new_value = old_value
# Check for paths containing "Shared" or "Generated" and replace with the mod folder
if old_value:
# Replace "Generated/Public/Shared/" paths
if "Generated/Public/Shared/" in old_value:
new_value = old_value.replace("Generated/Public/Shared/", f"Generated/Public/{self.mod_folder_name}/")
attr.set("value", new_value)
self.progress_update.emit(
f" Updated attribute '{attr.get('id')}' path from Shared: {old_value} -> {new_value}", "black"
)
# Replace "Generated/Public/Generated/" paths
elif "Generated/Public/Generated/" in old_value:
new_value = old_value.replace("Generated/Public/Generated/", f"Generated/Public/{self.mod_folder_name}/")
attr.set("value", new_value)
self.progress_update.emit(
f" Updated attribute '{attr.get('id')}' path from Generated: {old_value} -> {new_value}", "black"
)
# Replace direct "Public/Shared/" references to maintain Generated/Public pattern
elif "Public/Shared/" in old_value:
new_value = old_value.replace("Public/Shared/", f"Generated/Public/{self.mod_folder_name}/")
attr.set("value", new_value)
self.progress_update.emit(
f" Updated attribute '{attr.get('id')}' direct path: {old_value} -> {new_value}", "black"
)
# Replace direct "Public/Generated/" references to maintain Generated/Public pattern
elif "Public/Generated/" in old_value:
new_value = old_value.replace("Public/Generated/", f"Generated/Public/{self.mod_folder_name}/")
attr.set("value", new_value)
self.progress_update.emit(
f" Updated attribute '{attr.get('id')}' direct path: {old_value} -> {new_value}", "black"
)
# Write the updated XML back to the temporary file
tree.write(temp_file_path, pretty_print=True, encoding="utf-8", xml_declaration=True)
except Exception as e:
self.progress_update.emit(f" Warning: Could not update attribute file paths: {str(e)}", "orange")
# We're no longer moving the files to the Generated folder as per requirements
# Instead we're keeping them in Public/FolderName/ while ensuring paths in LSX files point to Generated/Public/FolderName/
self.progress_update.emit(f" NOTE: LSX references will point to Generated/Public/{self.mod_folder_name}/", "blue")
except Exception as e:
self.progress_update.emit(f" Error processing {file_name}: {str(e)}", "red")
self.failed_extractions += 1
def run_sort_lsx_files(self):
"""Run sort_lsx_files.py on the Public/[ModFolder] directory"""
try:
# Get the path to sort_lsx_files.py (in the same directory as this script)
script_dir = os.path.dirname(os.path.abspath(__file__))
sort_lsx_script = os.path.join(script_dir, "sort_lsx_files.py")
# Get the target directory (Public/[ModFolder])
target_dir = os.path.join(self.unpacked_data_folder, "Public", self.mod_folder_name)
if not os.path.exists(sort_lsx_script):
self.progress_update.emit(f"Error: Could not find sort_lsx_files.py in {script_dir}", "red")
return
if not os.path.exists(target_dir):
self.progress_update.emit(f"Error: Public/{self.mod_folder_name} directory not found", "red")
return
self.progress_update.emit(f"Running: sort_lsx_files.py --input_dir \"{target_dir}\" --action all --keep_all", "blue")
# Run the script as a subprocess with --keep_all option to disable __SCRAPPED folder creation
process = subprocess.Popen(
[sys.executable, sort_lsx_script, "--input_dir", target_dir, "--action", "all", "--keep_all"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
universal_newlines=True,
bufsize=1 # Line buffered output
)
# Read and log output line by line in real-time
while True:
# Read stdout line
stdout_line = process.stdout.readline()
if stdout_line:
line = stdout_line.strip()
# Skip empty lines and add log colors based on content
if line:
if "Error:" in line:
self.progress_update.emit(f" {line}", "red")
elif "Warning:" in line:
self.progress_update.emit(f" {line}", "orange")
elif "Success" in line or "Complete" in line:
self.progress_update.emit(f" {line}", "green")
else:
self.progress_update.emit(f" {line}", "black")
# Check if stderr has data
stderr_line = process.stderr.readline()
if stderr_line:
self.progress_update.emit(f" ERROR: {stderr_line.strip()}", "red")
# Check if process is still running
if process.poll() is not None:
# Process has ended, read any remaining output
for line in process.stdout:
if line.strip():
self.progress_update.emit(f" {line.strip()}", "black")
for line in process.stderr:
if line.strip():
self.progress_update.emit(f" ERROR: {line.strip()}", "red")
break
# Get exit code
exit_code = process.poll()
if exit_code == 0:
self.progress_update.emit("LSX file sorting completed successfully", "green")
else:
self.progress_update.emit(f"LSX file sorting failed with exit code {exit_code}", "red")
except Exception as e:
self.progress_update.emit(f"Error running sort_lsx_files.py: {str(e)}", "red")
# Continue with the main workflow even if sorting fails
self.progress_update.emit("Continuing with main workflow despite sorting errors", "orange")
class ResourceOverwriterGUI(QMainWindow):
"""Main GUI window for the Resource Overwriter application"""
def __init__(self):
super().__init__()
self.init_ui()
def init_ui(self):
"""Initialize the user interface"""
self.setWindowTitle("BG3 Resource Overwriter")
self.setGeometry(100, 100, 800, 600)
# Create central widget and layout
central_widget = QWidget()
self.setCentralWidget(central_widget)
main_layout = QVBoxLayout(central_widget)
# Create folder selection sections
folder_layout1 = QHBoxLayout()
self.unpacked_folder_label = QLabel("Mod Folder (output):")
self.unpacked_folder_path = QLabel("Not selected")
self.unpacked_folder_path.setStyleSheet("font-weight: bold;")
self.unpacked_browse_button = QPushButton("Browse...")
self.unpacked_browse_button.clicked.connect(self.browse_unpacked_folder)
folder_layout1.addWidget(self.unpacked_folder_label)
folder_layout1.addWidget(self.unpacked_folder_path, 1)
folder_layout1.addWidget(self.unpacked_browse_button)
folder_layout2 = QHBoxLayout()
self.vanilla_folder_label = QLabel("Generated Folder:")
self.vanilla_folder_path = QLabel("Not selected")
self.vanilla_folder_path.setStyleSheet("font-weight: bold;")
self.vanilla_browse_button = QPushButton("Browse...")
self.vanilla_browse_button.clicked.connect(self.browse_vanilla_folder)
folder_layout2.addWidget(self.vanilla_folder_label)
folder_layout2.addWidget(self.vanilla_folder_path, 1)
folder_layout2.addWidget(self.vanilla_browse_button)
# Add option to skip sorting
options_layout = QHBoxLayout()
self.skip_sort_checkbox = QCheckBox("Skip LSX file sorting step")
self.skip_sort_checkbox.setToolTip("Check this to skip the automatic sorting of LSX files at the end of processing")
options_layout.addWidget(self.skip_sort_checkbox)
options_layout.addStretch()
# Create progress section
progress_layout = QVBoxLayout()
self.progress_bar = QProgressBar()
self.progress_bar.setRange(0, 100)
self.progress_bar.setValue(0)
# Create log section
self.log_output = QTextEdit()
self.log_output.setReadOnly(True)
# Create buttons
button_layout = QHBoxLayout()
self.start_button = QPushButton("Start Processing")
self.start_button.clicked.connect(self.start_processing)
self.start_button.setEnabled(False)
self.exit_button = QPushButton("Exit")
self.exit_button.clicked.connect(self.close)
button_layout.addWidget(self.start_button)
button_layout.addWidget(self.exit_button)
# Add all layouts to main layout
main_layout.addLayout(folder_layout1)
main_layout.addLayout(folder_layout2)
main_layout.addLayout(options_layout)
main_layout.addLayout(progress_layout)
main_layout.addWidget(self.progress_bar)
main_layout.addWidget(self.log_output, 1)
main_layout.addLayout(button_layout)
# Initialize worker
self.worker = None
# Show the window
self.show()
# Add helper text to the log
self.log("This tool will process all files in the Generated folder,", "black")
self.log("find their corresponding LSX files on bg3.norbyte.dev, and save them", "black")
self.log("following this workflow:", "black")
self.log("\nThe tool will automatically:", "black")
self.log("1. Find the meta.lsx file in your mod project", "black")
self.log("2. Extract the 'Folder' value from meta.lsx", "black")
self.log("3. Create LSX files in Public/[Folder Value]/", "black")
self.log("4. Copy Generated files to Generated/Public/[Folder Value]/", "black")
self.log("5. Update all file path references in LSX to point to Generated/Public/[Folder Value]/", "black")
self.log("6. Run the LSX file sorter on Public/[Folder Value]/ to organize files by type", "black")
self.log(" (VisualBanks, TextureBanks, etc.) and check for unreferenced files", "black")
self.log("\nPlease select both folders to begin.", "black")
def browse_unpacked_folder(self):
"""Open a file dialog to select the Mod folder (output)"""
folder = QFileDialog.getExistingDirectory(self, "Select Mod Project Folder")
if folder:
# Verify if this is likely a mod project folder by checking for meta.lsx
has_meta = False
mods_dir = os.path.join(folder, "Mods")
if os.path.exists(mods_dir):
for mod_folder in os.listdir(mods_dir):
mod_path = os.path.join(mods_dir, mod_folder)
if os.path.isdir(mod_path):
meta_path = os.path.join(mod_path, "meta.lsx")
if os.path.exists(meta_path):
has_meta = True
break
if not has_meta:
self.log("Warning: No meta.lsx file found in the selected folder. Make sure this is a BG3 mod project.", "orange")
self.log("The meta.lsx file should be located in the Mods/[ModName]/meta.lsx path.", "orange")
self.unpacked_folder_path.setText(folder)
self.check_inputs()
self.log("Selected Mod folder: " + folder, "black")
def browse_vanilla_folder(self):
"""Open a file dialog to select the Generated folder"""
folder = QFileDialog.getExistingDirectory(self, "Select Generated Folder")
if folder:
# Verify if the selected folder contains asset files
has_files = False
for ext in ['.gr2', '.dds', '.lsf', '.gtp', '.tga']:
if glob.glob(os.path.join(folder, f"**/*{ext}"), recursive=True):
has_files = True
break
if not has_files:
self.log(f"Warning: No supported files found in the selected folder.", "orange")
self.vanilla_folder_path.setText(folder)
self.check_inputs()
self.log("Selected Generated folder: " + folder, "black")
def check_inputs(self):
"""Check if both folders are selected"""
has_unpacked = self.unpacked_folder_path.text() != "Not selected"
has_vanilla = self.vanilla_folder_path.text() != "Not selected"
self.start_button.setEnabled(has_unpacked and has_vanilla)
def log(self, message, color="black"):
"""Add a message to the log output with color"""
# Create text format with the specified color
text_format = QTextCharFormat()
text_format.setForeground(QColor(color))
# Save current cursor position and format
cursor = self.log_output.textCursor()
old_position = cursor.position()
old_format = cursor.charFormat()
# Move cursor to end and set new format
cursor.movePosition(cursor.MoveOperation.End)
cursor.setCharFormat(text_format)
# Insert the text with the new format
cursor.insertText(message + "\n")
# Restore cursor position and format
cursor.setPosition(old_position)
cursor.setCharFormat(old_format)
# Scroll to the bottom
self.log_output.verticalScrollBar().setValue(
self.log_output.verticalScrollBar().maximum()
)
def start_processing(self):
"""Start the resource overwriting process"""
unpacked_data_folder = self.unpacked_folder_path.text()
vanilla_overrides_folder = self.vanilla_folder_path.text()
skip_sort = self.skip_sort_checkbox.isChecked()
if not os.path.exists(unpacked_data_folder):
QMessageBox.warning(self, "Error", "Mod folder does not exist.")
return
if not os.path.exists(vanilla_overrides_folder):
QMessageBox.warning(self, "Error", "Folder does not exist.")
return
# Check if meta.lsx file likely exists in the mod project
has_meta = False
mods_dir = os.path.join(unpacked_data_folder, "Mods")
if os.path.exists(mods_dir):
for mod_folder in os.listdir(mods_dir):
mod_path = os.path.join(mods_dir, mod_folder)
if os.path.isdir(mod_path):
meta_path = os.path.join(mod_path, "meta.lsx")
if os.path.exists(meta_path):
has_meta = True
break
if not has_meta:
result = QMessageBox.warning(self, "Warning",
"No meta.lsx file found in the Mods directory.\n\n"
"This tool requires a valid meta.lsx file to identify the mod folder name.\n\n"
"Do you want to continue anyway?",
QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No)
if result == QMessageBox.StandardButton.No:
return
# Disable UI elements during processing
self.start_button.setEnabled(False)
self.unpacked_browse_button.setEnabled(False)
self.vanilla_browse_button.setEnabled(False)
# Clear the log
self.log_output.clear()
# Create and start the worker thread for processing
self.worker = ResourceOverwriterWorker(unpacked_data_folder, vanilla_overrides_folder, skip_sort)
self.worker.progress_update.connect(self.log)
self.worker.progress_value.connect(self.progress_bar.setValue)
self.worker.progress_max.connect(self.progress_bar.setMaximum)
self.worker.finished.connect(self.processing_finished)
self.log("Starting automatic processing of all files...", "black")
if skip_sort:
self.log("NOTE: LSX file sorting step will be skipped (disabled by user)", "blue")
self.worker.start()
def processing_finished(self):
"""Called when the processing is finished"""
self.start_button.setEnabled(True)
self.unpacked_browse_button.setEnabled(True)
self.vanilla_browse_button.setEnabled(True)
self.log("Processing finished.", "black")
# Show a message box with appropriate text based on whether sorting was skipped
if self.worker and self.worker.skip_sort:
QMessageBox.information(self, "Processing Complete",
"Resource extraction process has completed.\n\n"
"LSX file sorting was skipped as requested.")
else:
QMessageBox.information(self, "Processing Complete",
"Resource extraction and LSX file sorting process has completed.")
def main():
app = QApplication(sys.argv)
window = ResourceOverwriterGUI()
sys.exit(app.exec())
if __name__ == "__main__":
main()
► sort_lsx_files.py
Code: Select all
#!/usr/bin/env python
"""
BG3 Heads LSX File Sorter
This script organizes LSX files for Baldur's Gate 3 head mods:
- Renames plain LSX files to include .lsf before .lsx extension
- Sorts LSX files by resource type into [PAK]_ folders
- Organizes VisualBank files by race and gender
- Identifies and scraps unreferenced LSX files
- Checks for LSX files with mismatched UUIDs
Usage: python sort_lsx_files.py --input_dir "/path/to/mod/folder" --action all
"""
import os
import csv
import shutil
import xml.etree.ElementTree as ET
import re
import glob
import sys
import argparse
from collections import Counter, defaultdict
from pathlib import Path
def rename_plain_lsx_files(input_dir, verbose=False):
"""
Renames all files ending in '.lsx' that do not include '.lsf' in their name
to include '.lsf' before the '.lsx' extension.
For example, 'example.lsx' will be renamed to 'example.lsf.lsx'.
"""
input_dir = Path(input_dir)
if not input_dir.exists():
print(f"Error: Input directory {input_dir} does not exist!")
sys.exit(1)
# Recursively search for all .lsx files in the directory
for file_path in input_dir.rglob("*.lsx"):
# Skip files that already include '.lsf' in their filename
if ".lsf" not in file_path.name:
new_name = file_path.with_name(file_path.stem + ".lsf.lsx")
try:
file_path.rename(new_name)
if verbose:
print(f"Renamed {file_path} to {new_name}")
except Exception as e:
print(f"Error renaming {file_path}: {e}")
def get_resource_type(file_path):
"""
Extract the resource type from an LSX file.
Returns the first non-empty region id found in the file.
"""
try:
tree = ET.parse(file_path)
root = tree.getroot()
# Check each region
for region in root.findall('.//region'):
region_id = region.get('id')
# Check if this region has any resources (children nodes)
node = region.find(f'./node[@id="{region_id}"]')
if node is not None and node.find('.//children/node[@id="Resource"]') is not None:
return region_id
# If no resource-containing region found, return the first region id
first_region = root.find('.//region')
if first_region is not None:
return first_region.get('id')
return "Unknown"
except Exception as e:
print(f"Error processing {file_path}: {e}")
return "Error"
def get_visual_id_from_file(file_path):
"""
Extract the ID from a Visual Bank LSX file.
"""
try:
tree = ET.parse(file_path)
root = tree.getroot()
# Find the Resource node in the VisualBank region
visual_bank = root.find('.//region[@id="VisualBank"]/node[@id="VisualBank"]/children/node[@id="Resource"]')
if visual_bank is not None:
# Get the ID attribute
id_attr = visual_bank.find('./attribute[@id="ID"]')
if id_attr is not None:
return id_attr.get('value', '').strip().lower()
return None
except Exception as e:
print(f"Error processing {file_path}: {e}")
return None
def read_head_csv(csv_file):
"""
Read the CSV file containing head information and create a mapping of
VisualResourceID to race/species and gender.
"""
id_to_info = {}
with open(csv_file, 'r', encoding='utf-8') as f:
reader = csv.DictReader(f)
for row in reader:
visual_id = row.get('VisualResourceID', '').strip().lower()
if visual_id:
race = row.get('race/species', '').strip()
gender = row.get('gender', '').strip()
ingame_text = row.get('ingameText', '').strip()
bodytype = row.get('bodytype', '').strip().lower()
# Extract head number if available in ingameText (e.g., "Head 3" -> "3")
head_number = None
if ingame_text and ingame_text.startswith('Head '):
try:
head_number = ingame_text.split(' ')[1]
except IndexError:
pass
# Store the information
id_to_info[visual_id] = {
'race': race,
'gender': gender,
'head_number': head_number,
'bodytype': bodytype
}
return id_to_info
def sort_by_resource_type(input_dir, output_dir=None, keep_originals=False, verbose=False):
"""
Sort LSX files by resource type (TextureBank, MaterialBank, etc.)
"""
# Set input and output directories using Path for better cross-platform compatibility
input_dir = Path(input_dir)
output_dir = Path(output_dir) if output_dir else input_dir
# Make sure the input directory exists
if not input_dir.exists():
print(f"Error: Input directory {input_dir} does not exist!")
sys.exit(1)
# Make sure the output directory exists
if not output_dir.exists():
try:
output_dir.mkdir(parents=True, exist_ok=True)
print(f"Created output directory: {output_dir}")
except Exception as e:
print(f"Error creating output directory {output_dir}: {e}")
sys.exit(1)
# Get all LSX files (including those with .lsf.lsx)
lsx_files = list(input_dir.glob("*.lsx"))
# Remove duplicates if any
lsx_files = list(set(lsx_files))
if not lsx_files:
print(f"No LSX files found in {input_dir}")
return
print(f"Found {len(lsx_files)} LSX files to process")
# Dictionary to track files by resource type
files_by_type = {}
error_files = []
# Process each file
for file_path in lsx_files:
if verbose:
print(f"Processing {file_path}...")
resource_type = get_resource_type(file_path)
if resource_type == "Error":
error_files.append(file_path)
continue
if resource_type not in files_by_type:
files_by_type[resource_type] = []
files_by_type[resource_type].append(file_path)
# Track statistics
stats = {
'total_files': len(lsx_files),
'processed_files': sum(len(files) for files in files_by_type.values()),
'error_files': len(error_files),
'resource_types': Counter({k: len(v) for k, v in files_by_type.items()})
}
# Create directories and move/copy files
for resource_type, files in files_by_type.items():
# Create target directory
target_dir = output_dir / f"[PAK]_{resource_type}s"
try:
target_dir.mkdir(exist_ok=True)
except Exception as e:
print(f"Error creating directory {target_dir}: {e}")
continue
print(f"{'Copy' if keep_originals else 'Move'} {len(files)} files to {target_dir}")
# Move/copy files to target directory
for file_path in files:
file_name = file_path.name
target_path = target_dir / file_name
try:
# Copy or move the file
if keep_originals:
shutil.copy2(file_path, target_path)
if verbose:
print(f" Copied {file_name} to {target_dir}")
else:
shutil.move(str(file_path), str(target_path))
if verbose:
print(f" Moved {file_name} to {target_dir}")
except Exception as e:
print(f"Error {'copying' if keep_originals else 'moving'} {file_path}: {e}")
stats['error_files'] += 1
# Print summary
print("\nSummary:")
print(f"Total files found: {stats['total_files']}")
print(f"Files processed: {stats['processed_files']}")
print(f"Files with errors: {stats['error_files']}")
print("\nResource types:")
for resource_type, count in stats['resource_types'].most_common():
print(f" {resource_type}: {count} files")
if error_files and verbose:
print("\nFiles with errors:")
for file_path in error_files:
print(f" {file_path}")
def sort_visual_banks(csv_file, visual_banks_dir, keep_originals=False, verbose=False, keep_all=False):
"""
Sort Visual Bank files based on race/species and gender from a CSV file
"""
# Convert paths to Path objects for better cross-platform compatibility
csv_file = Path(csv_file)
visual_banks_dir = Path(visual_banks_dir)
# Define race mapping for special cases
race_mapping = {
'half-elf': 'HELF',
# Add other mappings as needed
}
# Make sure the input files/directories exist
if not csv_file.exists():
print(f"Error: CSV file {csv_file} does not exist!")
sys.exit(1)
if not visual_banks_dir.exists():
print(f"Error: Visual Banks directory {visual_banks_dir} does not exist!")
sys.exit(1)
# Read the CSV file
print(f"Reading head information from {csv_file}...")
id_to_info = read_head_csv(csv_file)
print(f"Found {len(id_to_info)} entries in the CSV file")
# Get all LSX files in the Visual Banks directory (including plain LSX files, now renamed)
lsx_files = list(visual_banks_dir.glob("*.lsx"))
# Remove duplicates if any
lsx_files = list(set(lsx_files))
print(f"Found {len(lsx_files)} LSX files in {visual_banks_dir}")
# Track statistics
stats = {
'total_files': len(lsx_files),
'matched_files': 0,
'unmatched_files': 0,
'error_files': 0,
'race_gender_counts': defaultdict(int),
'scrapped_files': 0
}
# Create a [PAK]_SCRAPPED folder for unmatched files (unless keep_all is True)
scrapped_dir = None
if not keep_all:
scrapped_dir = visual_banks_dir.parent / "__SCRAPPED"
try:
scrapped_dir.mkdir(exist_ok=True)
except Exception as e:
print(f"Error creating SCRAPPED directory {scrapped_dir}: {e}")
# Dictionary to track folder numbers for each race/gender combination
folder_counters = defaultdict(int)
# First pass: Identify all race/gender combinations and count them
for file_path in lsx_files:
visual_id = get_visual_id_from_file(file_path)
if visual_id is None:
continue
info = id_to_info.get(visual_id)
if info is None:
continue
# Apply race mapping if available
race = info['race'].lower()
race = race_mapping.get(race, race).upper()
gender = info['gender'].upper() if info['gender'] else 'UNKNOWN'
bodytype = info['bodytype']
gender_code = f"{gender[0]}{'S' if bodytype == 'strong' else ''}" if gender else 'X'
race_gender_key = f"{race}_{gender_code}"
folder_counters[race_gender_key] += 1
# Process each file
for file_path in lsx_files:
if verbose:
print(f"Processing {file_path}...")
# Get the Visual ID from the file
visual_id = get_visual_id_from_file(file_path)
if visual_id is None:
print(f"Error: Could not extract Visual ID from {file_path}")
stats['error_files'] += 1
# Move error files to SCRAPPED folder if not keeping all files
if scrapped_dir:
target_path = scrapped_dir / file_path.name
try:
if keep_originals:
shutil.copy2(file_path, target_path)
else:
shutil.move(str(file_path), str(target_path))
stats['scrapped_files'] += 1
if verbose:
print(f"{'Copied' if keep_originals else 'Moved'} error file {file_path.name} to {scrapped_dir}")
except Exception as e:
print(f"Error {'copying' if keep_originals else 'moving'} {file_path}: {e}")
continue
# Look up the race/species and gender
info = id_to_info.get(visual_id)
if info is None:
print(f"Warning: No information found for Visual ID {visual_id} in {file_path}")
stats['unmatched_files'] += 1
# Move unmatched files to SCRAPPED folder if not keeping all files
if scrapped_dir:
target_path = scrapped_dir / file_path.name
try:
if keep_originals:
shutil.copy2(file_path, target_path)
else:
shutil.move(str(file_path), str(target_path))
stats['scrapped_files'] += 1
if verbose:
print(f"{'Copied' if keep_originals else 'Moved'} unmatched file {file_path.name} to {scrapped_dir}")
except Exception as e:
print(f"Error {'copying' if keep_originals else 'moving'} {file_path}: {e}")
continue
# Apply race mapping if available
race = info['race'].lower()
race = race_mapping.get(race, race).upper()
gender = info['gender'].upper() if info['gender'] else 'UNKNOWN'
bodytype = info['bodytype']
gender_code = f"{gender[0]}{'S' if bodytype == 'strong' else ''}" if gender else 'X'
head_number = info.get('head_number')
race_gender_key = f"{race}_{gender_code}"
# Use head number if available, otherwise use folder counter
if head_number:
folder_name = f"[PAK]_{race}_{gender_code}_{head_number}"
else:
# Get the folder number
folder_number = folder_counters[race_gender_key]
folder_name = f"[PAK]_{race}_{gender_code}_{folder_number}"
# Create the target directory
target_dir = visual_banks_dir.parent / folder_name
try:
target_dir.mkdir(exist_ok=True)
except Exception as e:
print(f"Error creating directory {target_dir}: {e}")
stats['error_files'] += 1
continue
# Track statistics
stats['matched_files'] += 1
stats['race_gender_counts'][race_gender_key] += 1
# Only decrement the counter if we're using it (not using head_number)
if not head_number:
# Decrement the counter for this race/gender combination
folder_counters[race_gender_key] -= 1
# Move/copy the file
target_path = target_dir / file_path.name
try:
if keep_originals:
shutil.copy2(file_path, target_path)
if verbose:
print(f"Copied {file_path.name} to {target_dir}")
else:
shutil.move(str(file_path), str(target_path))
if verbose:
print(f"Moved {file_path.name} to {target_dir}")
except Exception as e:
print(f"Error {'copying' if keep_originals else 'moving'} {file_path}: {e}")
stats['error_files'] += 1
# Print summary
print("\nSummary:")
print(f"Total files: {stats['total_files']}")
print(f"Matched files: {stats['matched_files']}")
print(f"Unmatched files: {stats['unmatched_files']}")
print(f"Error files: {stats['error_files']}")
print(f"Scrapped files: {stats['scrapped_files']}")
print("\nRace/Gender counts:")
for race_gender, count in sorted(stats['race_gender_counts'].items()):
print(f" {race_gender}: {count} files")
def is_visualbank_resource(file_path):
"""
Returns True if the file is in a VisualBank folder.
This includes:
- A folder named "[PAK]_VisualBanks"
- Any folder whose name matches the pattern "[PAK]_X_Y_Z"
(i.e. sorted head LSX files by race/gender)
"""
for part in file_path.parents:
if part.name == "[PAK]_VisualBanks":
return True
if re.match(r'^\[PAK\]_[^_]+_[^_]+_[^_]+$', part.name):
return True
return False
def check_unreferenced_lsx_files(input_dir, keep_originals=False, verbose=False, keep_all=False):
"""
Recursively scraps LSX files (that are not VisualBank resources) if their UUID,
derived from the filename (by stripping ".lsf.lsx"), is not referenced anywhere
in any LSX file in the entire directory (excluding files already scrapped).
The scrapping is attempted in the following resource folder order:
1. [PAK]_AnimationSetBanks
2. [PAK]_AnimationBanks
3. [PAK]_MaterialBanks
4. [PAK]_TextureBanks
5. [PAK]_SkeletonBanks
Files that are unreferenced are moved (or copied, if keep_originals is True)
into a __SCRAPPED folder, in a subfolder matching their resource type.
VisualBank resources are not scrapped (but still count as references).
If keep_all is True, no files will be moved to __SCRAPPED folders.
"""
input_dir = Path(input_dir)
if not input_dir.exists():
print(f"Error: Input directory {input_dir} does not exist!")
sys.exit(1)
# If keep_all is True, just identify unreferenced files but don't move them
if keep_all:
print("Keep all files mode enabled: Only identifying unreferenced files, but not moving them")
# Build a list of all LSX files in the directory (skip those in any __SCRAPPED folder)
all_lsx_files = []
all_lsx_files.extend(list(input_dir.rglob("*.lsf.lsx")))
all_lsx_files.extend(list(input_dir.rglob("*.lsf_*.lsx"))) # Add support for files like .lsf_1.lsx
all_files_for_check = [f for f in all_lsx_files if "__SCRAPPED" not in f.parts]
if verbose:
print(f"Total LSX files for reference check: {len(all_files_for_check)}")
# Build a mapping of each file to its content (this set includes VisualBank files too)
file_contents = {}
for file_path in all_files_for_check:
try:
with open(file_path, 'r', encoding='utf-8') as f:
file_contents[file_path] = f.read()
except Exception as e:
if verbose:
print(f"Error reading {file_path}: {e}")
# Define the resource order for scrapping
resource_order = [
"[PAK]_AnimationSetBanks",
"[PAK]_AnimationBanks",
"[PAK]_MaterialBanks",
"[PAK]_TextureBanks",
"[PAK]_SkeletonBanks"
]
# Prepare the __SCRAPPED folder if not keeping all files
scrapped_dir = None
if not keep_all:
scrapped_dir = input_dir / "__SCRAPPED"
scrapped_dir.mkdir(exist_ok=True)
# Group files by their resource type folder
files_by_resource = {res: [] for res in resource_order}
# Add a category for files not in specific ordered folders
files_by_resource["other"] = []
# Categorize files by their resource type
for file_path in file_contents.keys():
# Skip VisualBank resources
if is_visualbank_resource(file_path):
continue
parent_folder = file_path.parent.name
if parent_folder in resource_order:
files_by_resource[parent_folder].append(file_path)
else:
files_by_resource["other"].append(file_path)
# Track number of unreferenced files identified
total_unreferenced = 0
total_scrapped = 0
unreferenced_files = []
# Process each resource type in order
for res in resource_order:
if verbose:
print(f"\nChecking files in {res}...")
# Process all files of this resource type
for file_path in files_by_resource[res]:
filename = file_path.name
# Handle various LSX file extensions (.lsf.lsx, .lsf_1.lsx, etc.)
if filename.endswith(".lsx"):
# Extract the UUID by removing the extension (.lsx) and any .lsf or .lsf_N part
base_name = filename[:-4] # Remove .lsx
if ".lsf" in base_name:
uuid = base_name.split(".lsf")[0]
else:
uuid = base_name
else:
if verbose:
print(f"Skipping file with unexpected format: {filename}")
continue
# Check if this UUID is referenced in any other file
referenced = False
for other_path, content in file_contents.items():
if other_path == file_path:
continue
if uuid in content:
referenced = True
if verbose:
print(f"UUID '{uuid}' from {filename} is referenced in {other_path.name}")
break
if not referenced:
total_unreferenced += 1
if verbose:
print(f"UUID '{uuid}' is not referenced in any other file")
unreferenced_files.append((file_path, res))
# If keep_all is False, scrap the file
if scrapped_dir:
# Scrap the file: move (or copy) to __SCRAPPED/res folder
target_subdir = scrapped_dir / res
target_subdir.mkdir(exist_ok=True)
target_path = target_subdir / filename
try:
if keep_originals:
shutil.copy2(file_path, target_path)
if verbose:
print(f"Copied {filename} to {target_subdir}")
else:
shutil.move(str(file_path), str(target_path))
if verbose:
print(f"Moved {filename} to {target_subdir}")
# Remove scrapped file from our reference check set
del file_contents[file_path]
total_scrapped += 1
except Exception as e:
print(f"Error processing {file_path}: {e}")
continue
# Process any remaining files not in specific resource folders
if verbose and files_by_resource["other"]:
print(f"\nChecking {len(files_by_resource['other'])} files not in specific resource folders...")
for file_path in files_by_resource["other"]:
filename = file_path.name
# Handle various LSX file extensions (.lsf.lsx, .lsf_1.lsx, etc.)
if filename.endswith(".lsx"):
# Extract the UUID by removing the extension (.lsx) and any .lsf or .lsf_N part
base_name = filename[:-4] # Remove .lsx
if ".lsf" in base_name:
uuid = base_name.split(".lsf")[0]
else:
uuid = base_name
else:
if verbose:
print(f"Skipping file with unexpected format: {filename}")
continue
# Check if this UUID is referenced in any other file
referenced = False
for other_path, content in file_contents.items():
if other_path == file_path:
continue
if uuid in content:
referenced = True
if verbose:
print(f"UUID '{uuid}' from {filename} is referenced in {other_path.name}")
break
if not referenced:
total_unreferenced += 1
if verbose:
print(f"UUID '{uuid}' is not referenced in any other file")
unreferenced_files.append((file_path, "other"))
# If keep_all is False, scrap the file
if scrapped_dir:
# Determine appropriate subfolder based on parent folder
parent_folder = file_path.parent.name
target_subdir = scrapped_dir / "Other"
if parent_folder.startswith("[PAK]_"):
target_subdir = scrapped_dir / parent_folder
target_subdir.mkdir(exist_ok=True)
target_path = target_subdir / filename
try:
if keep_originals:
shutil.copy2(file_path, target_path)
if verbose:
print(f"Copied {filename} to {target_subdir}")
else:
shutil.move(str(file_path), str(target_path))
if verbose:
print(f"Moved {filename} to {target_subdir}")
# Remove scrapped file from our reference check set
del file_contents[file_path]
total_scrapped += 1
except Exception as e:
print(f"Error processing {file_path}: {e}")
continue
remaining = len(file_contents)
# Print summary
if keep_all:
print(f"\nUnreferenced file check complete. Files unreferenced: {total_unreferenced}. Remaining LSX files: {remaining}")
if verbose and total_unreferenced > 0:
print("Unreferenced files (not moved due to --keep_all flag):")
for f, res in unreferenced_files:
print(f" {f} (would be moved to {res})")
else:
print(f"\nScrapping complete. Files scrapped: {total_scrapped}. Remaining LSX files: {remaining}")
if verbose and remaining:
print("Remaining files:")
for f in file_contents.keys():
print(f" {f}")
def reset_directory_structure(input_dir, verbose=False):
"""
Move all LSX files out of their directories into the main Assets folder,
then delete all the subdirectories to start with a clean slate.
"""
# Set input directory using Path for better cross-platform compatibility
input_dir = Path(input_dir)
# Make sure the input directory exists
if not input_dir.exists():
print(f"Error: Input directory {input_dir} does not exist!")
sys.exit(1)
print(f"Resetting directory structure in {input_dir}...")
# Get all subdirectories
subdirs = [d for d in input_dir.glob("*") if d.is_dir()]
print(f"Found {len(subdirs)} subdirectories")
# Track statistics
stats = {
'files_moved': 0,
'dirs_deleted': 0,
'errors': 0
}
# Move all LSX files from subdirectories to the main directory
for subdir in subdirs:
if verbose:
print(f"Processing {subdir}...")
# Get all LSX files in this subdirectory
lsx_files = []
lsx_files.extend(list(subdir.glob("**/*.lsf.lsx")))
lsx_files.extend(list(subdir.glob("**/*.lsf_*.lsx"))) # Add support for files like .lsf_1.lsx
if verbose:
print(f" Found {len(lsx_files)} LSX files in {subdir.name}")
# Move each file to the main directory
for file_path in lsx_files:
target_path = input_dir / file_path.name
# If a file with the same name already exists, add a suffix
if target_path.exists():
base_name = file_path.stem
extension = file_path.suffix
counter = 1
while target_path.exists():
new_name = f"{base_name}_{counter}{extension}"
target_path = input_dir / new_name
counter += 1
try:
shutil.move(str(file_path), str(target_path))
stats['files_moved'] += 1
if verbose:
print(f" Moved {file_path.name} to {input_dir}")
except Exception as e:
print(f"Error moving {file_path}: {e}")
stats['errors'] += 1
# Delete all subdirectories
for subdir in subdirs:
try:
# Use shutil.rmtree to delete directory and all its contents
shutil.rmtree(subdir)
stats['dirs_deleted'] += 1
if verbose:
print(f"Deleted directory {subdir.name}")
except Exception as e:
print(f"Error deleting directory {subdir}: {e}")
stats['errors'] += 1
# Print summary
print("\nReset Summary:")
print(f"Files moved to main directory: {stats['files_moved']}")
print(f"Directories deleted: {stats['dirs_deleted']}")
print(f"Errors encountered: {stats['errors']}")
def check_mismatched_uuids(input_dir, keep_originals=False, verbose=False, keep_all=False):
"""
Checks all LSX files to find any where the UUID in the filename doesn't match
the resource UUID inside the file. Files with mismatches are moved to a
__MISMATCHED folder.
If keep_all is True, mismatched files will only be reported, not moved.
"""
input_dir = Path(input_dir)
if not input_dir.exists():
print(f"Error: Input directory {input_dir} does not exist!")
sys.exit(1)
# Build a list of all LSX files in the directory (skip those in __SCRAPPED or __MISMATCHED folders)
all_lsx_files = []
all_lsx_files.extend(list(input_dir.rglob("*.lsf.lsx")))
all_lsx_files.extend(list(input_dir.rglob("*.lsf_*.lsx"))) # Add support for files like .lsf_1.lsx
skip_folders = ["__SCRAPPED", "__MISMATCHED"]
all_files_for_check = [f for f in all_lsx_files if not any(skip in f.parts for skip in skip_folders)]
if verbose:
print(f"Total LSX files to check for UUID mismatches: {len(all_files_for_check)}")
# Prepare the __MISMATCHED folder if not keeping all files
mismatched_dir = None
if not keep_all:
mismatched_dir = input_dir / "__MISMATCHED"
mismatched_dir.mkdir(exist_ok=True)
# Track mismatched files
total_mismatched = 0
mismatched_files = []
# Check each file for UUID mismatches
for file_path in all_files_for_check:
filename = file_path.name
# Extract the expected UUID from the filename
if filename.endswith(".lsx"):
# Remove .lsx extension
base_name = filename[:-4]
if ".lsf" in base_name:
# Handle .lsf and .lsf_N patterns
expected_uuid = base_name.split(".lsf")[0]
else:
expected_uuid = base_name
else:
# Skip files without .lsx extension
continue
# Parse the file to extract the actual UUID
try:
actual_uuid = extract_resource_uuid(file_path)
if not actual_uuid:
if verbose:
print(f"Could not extract UUID from {filename}")
continue
if expected_uuid != actual_uuid:
total_mismatched += 1
if verbose:
print(f"UUID mismatch in {filename}:")
print(f" Expected: {expected_uuid}")
print(f" Actual: {actual_uuid}")
mismatched_files.append((file_path, expected_uuid, actual_uuid))
# If not keep_all, move the file to __MISMATCHED folder
if mismatched_dir:
# Create folder structure matching the original location
rel_path = file_path.relative_to(input_dir)
target_dir = mismatched_dir / rel_path.parent
target_dir.mkdir(parents=True, exist_ok=True)
target_path = target_dir / filename
try:
if keep_originals:
shutil.copy2(file_path, target_path)
if verbose:
print(f"Copied {filename} to {target_dir}")
else:
shutil.move(str(file_path), str(target_path))
if verbose:
print(f"Moved {filename} to {target_dir}")
except Exception as e:
print(f"Error processing {file_path}: {e}")
continue
except Exception as e:
if verbose:
print(f"Error processing {file_path}: {e}")
continue
# Print summary
if keep_all:
print(f"\nMismatch check complete. Files with UUID mismatches: {total_mismatched}")
if verbose and total_mismatched > 0:
print("Mismatched files (not moved due to --keep_all flag):")
for f, expected, actual in mismatched_files:
print(f" {f}")
print(f" Expected: {expected}")
print(f" Actual: {actual}")
else:
print(f"\nMismatch check complete. Files with UUID mismatches moved: {total_mismatched}")
return total_mismatched
def main():
# Create argument parser
parser = argparse.ArgumentParser(
description="BBE-Heads LSX File Sorter - Organizes LSX files from BG3 head mods",
epilog="Example: python sort_lsx_files.py --input_dir \"output/MyHeadMod\" --action all --verbose"
)
# Add arguments
parser.add_argument("--input_dir", type=str, required=True,
help="Input directory to process (usually the Content/Assets/Characters folder)")
parser.add_argument("--csv_file", type=str,
default="Spreadsheets/Head,-Skin,-Scars---orinEsque.csv",
help="CSV file containing head information for sorting VisualBanks")
parser.add_argument("--action", type=str,
choices=["rename", "reset", "sort", "visualbanks", "mismatched", "unreferenced", "all"],
default="all",
help="Action to perform: rename (fix extensions), reset (flatten directory), sort (by type), "
"visualbanks (sort by race/gender), mismatched (check UUIDs), unreferenced (find unused files), "
"or all (perform all actions)")
parser.add_argument("--keep_originals", action="store_true",
help="Keep original files instead of moving them (creates copies)")
parser.add_argument("--verbose", action="store_true",
help="Enable verbose output for detailed progress information")
# Add parameter to disable __SCRAPPED folder creation
parser.add_argument("--keep_all", action="store_true",
help="Keep all files (do not move anything to __SCRAPPED or __MISMATCHED folders)")
# Parse arguments
args = parser.parse_args()
# Verify input directory exists
if not os.path.exists(args.input_dir):
print(f"Error: Input directory '{args.input_dir}' not found!")
sys.exit(1)
# Display header
print("=== BBE-Heads LSX File Sorter ===")
print(f"Processing directory: {args.input_dir}")
# Perform requested actions
if args.action in ["rename", "all"]:
print("\n0. Renaming plain LSX files to include .lsf before .lsx...")
rename_plain_lsx_files(input_dir=args.input_dir, verbose=args.verbose)
if args.action in ["reset", "all"]:
print("\n1. Resetting directory structure...")
reset_directory_structure(input_dir=args.input_dir, verbose=args.verbose)
if args.action in ["sort", "all"]:
print("\n2. Sorting all LSX files by resource type...")
sort_by_resource_type(input_dir=args.input_dir, output_dir=None,
keep_originals=args.keep_originals, verbose=args.verbose)
if args.action in ["visualbanks", "all"]:
print("\n3. Sorting Visual Bank files by race/gender...")
# Check if CSV file exists
if not os.path.exists(args.csv_file):
print(f"Warning: CSV file '{args.csv_file}' not found. Skipping Visual Bank sorting.")
else:
# Find visual banks directory
visual_banks_dir = os.path.join(args.input_dir, "[PAK]_VisualBanks")
if os.path.exists(visual_banks_dir):
sort_visual_banks(csv_file=args.csv_file, visual_banks_dir=visual_banks_dir,
keep_originals=args.keep_originals, verbose=args.verbose, keep_all=args.keep_all)
else:
print(f"Visual banks directory not found at {visual_banks_dir}")
print("Skipping Visual Bank sorting")
if args.action in ["mismatched", "all"]:
print("\n4. Checking for files with mismatched UUIDs...")
check_mismatched_uuids(input_dir=args.input_dir,
keep_originals=args.keep_originals, verbose=args.verbose, keep_all=args.keep_all)
if args.action in ["unreferenced", "all"]:
print("\n5. Checking for unreferenced non-VisualBank LSX files...")
check_unreferenced_lsx_files(args.input_dir,
keep_originals=args.keep_originals, verbose=args.verbose, keep_all=args.keep_all)
print(f"\nAll done! Your files in '{args.input_dir}' have been organized.")
print("Check for any __SCRAPPED or __MISMATCHED folders that may contain files that need attention.")
if __name__ == "__main__":
main()