mirror of
https://github.com/worldforge/ember
synced 2026-08-13 16:23:06 -04:00
274 lines
No EOL
9.8 KiB
Python
Executable file
274 lines
No EOL
9.8 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# Requires Python 3 or higher
|
|
|
|
import os, shutil, subprocess, tempfile, fnmatch, traceback, re, sys, fileinput, signal
|
|
|
|
|
|
def find_matches(files, patterns):
|
|
matches = []
|
|
for pattern in patterns:
|
|
for file in fnmatch.filter(files, pattern):
|
|
matches.append(file)
|
|
return matches
|
|
|
|
def collect_licenses(src_assets_path, licenses, assets):
|
|
"""Walk upwards looking for any license or author files"""
|
|
for asset in assets:
|
|
asset_path = os.path.join(src_assets_path, asset)
|
|
if os.path.isfile(asset_path):
|
|
asset_path_tokens = os.path.dirname(asset).split(os.sep)
|
|
while len(asset_path_tokens) > 0:
|
|
dir = (os.sep).join(asset_path_tokens)
|
|
for file in find_matches(os.listdir(os.path.join(src_assets_path, dir)), ["*license*", "*LICENSE*", "*authors*", "*AUTHORS*"]):
|
|
licenses.add(os.path.join(dir, file))
|
|
del asset_path_tokens[-1]
|
|
|
|
|
|
|
|
def copy_assets(src_assets_path, dest_assets_path, assets, assets_no_conversion, image_max_size = None):
|
|
originalSize = 0
|
|
destinationSize = 0
|
|
copied = 0
|
|
skipped = 0
|
|
converted = 0
|
|
errors = 0
|
|
for asset in assets:
|
|
asset_path = os.path.join(src_assets_path, asset)
|
|
if os.path.isfile(asset_path):
|
|
|
|
png_to_dds_conversion_needed = False
|
|
|
|
dest_asset_path = os.path.join(dest_assets_path, asset)
|
|
|
|
dest_asset_dir_path = os.path.dirname(dest_asset_path)
|
|
if not os.path.exists(dest_asset_dir_path):
|
|
os.makedirs(dest_asset_dir_path)
|
|
|
|
|
|
if dest_asset_path.endswith(".png") and not asset.startswith("common/ui") and asset not in assets_no_conversion:
|
|
(dest_asset_path, _) = re.subn('\.png$', '.dds', dest_asset_path)
|
|
png_to_dds_conversion_needed = True
|
|
|
|
#Check if the destination file exists, and if so if it's older than the source
|
|
if os.path.exists(dest_asset_path) and os.path.getmtime(dest_asset_path) >= os.path.getmtime(asset_path):
|
|
#destination file is newer or of the same date as the source file
|
|
skipped = skipped + 1
|
|
else:
|
|
if asset_path.endswith((".png", ".dds", ".jpg")) and not asset.startswith("common/ui"):
|
|
compression_format = ""
|
|
|
|
convert_cmd = ["convert"]
|
|
|
|
if image_max_size:
|
|
convert_cmd.append('-resize "{0}x{0}>"'.format(image_max_size))
|
|
|
|
if png_to_dds_conversion_needed:
|
|
imagemetadata = subprocess.check_output(["file", asset_path])
|
|
#TODO: use bc3n or bc1n for normal maps, with support in shaders
|
|
if "RGBA" in imagemetadata.decode("utf-8"):
|
|
compression_format = "dxt5"
|
|
else:
|
|
compression_format = "dxt1"
|
|
|
|
convert_cmd.append('-define dds:compression={0}'.format(compression_format))
|
|
|
|
|
|
convert_cmd.append(asset_path)
|
|
convert_cmd.append(dest_asset_path)
|
|
|
|
print("converting image asset {0} to {1} ({2})".format(asset, dest_asset_path, compression_format))
|
|
if os.system(" ".join(convert_cmd)) == 0:
|
|
converted = converted + 1
|
|
else:
|
|
errors = errors + 1
|
|
else:
|
|
print("copying asset {0} to {1}".format(asset, dest_asset_path))
|
|
shutil.copy(asset_path, dest_asset_path)
|
|
copied = copied + 1
|
|
if asset.endswith(".material") and not asset.startswith("common/ui"):
|
|
print("processing material to replace 'png' with 'dds'")
|
|
for line in fileinput.input(dest_asset_path, inplace=True):
|
|
print(line.replace(".png", ".dds"), end='')
|
|
|
|
originalSize = originalSize + os.path.getsize(asset_path)
|
|
destinationSize = destinationSize + os.path.getsize(dest_asset_path)
|
|
|
|
else:
|
|
errors = errors + 1
|
|
print("referenced file {0} does not exist".format(asset_path))
|
|
|
|
return copied, converted, skipped, errors, originalSize, destinationSize
|
|
|
|
def copytree(src, dst):
|
|
if not os.path.exists(dst):
|
|
os.makedirs(dst)
|
|
for item in os.listdir(src):
|
|
s = os.path.join(src, item)
|
|
d = os.path.join(dst, item)
|
|
if os.path.isdir(s):
|
|
copytree(s, d)
|
|
else:
|
|
if not os.path.exists(d) or os.stat(s).st_mtime - os.stat(d).st_mtime > 1:
|
|
print("Copying " + s)
|
|
shutil.copy2(s, d)
|
|
|
|
def main():
|
|
"""The main entry point."""
|
|
usage = """usage: {0} <MEDIA_DIRECTORY_TRUNK> <OUTPUT_DIRECTORY> <IMAGE_MAX_SIZE>
|
|
|
|
This script will automatically scan the media source repo and create a repository suitable for distributions.
|
|
This process mainly involves copying only those assets which are used by Ember, and resizing textures.
|
|
|
|
|
|
MEDIA_DIRECTORY_TRUNK is the path to the root of the media directory, often named "trunk"
|
|
|
|
OUTPUT_DIRECTORY is where the processed assets will be copied
|
|
|
|
IMAGE_MAX_SIZE (optional) the max size in pixels of any image. Any larger image will be scaled
|
|
|
|
OPTIONS
|
|
-h Show this help text
|
|
""".format(sys.argv[0])
|
|
|
|
if len(sys.argv) == 1:
|
|
print("ERROR: MEDIA_DIRECTORY_TRUNK must be specified!")
|
|
print(usage)
|
|
sys.exit(1)
|
|
elif len(sys.argv) == 2:
|
|
print("ERROR: OUTPUT_DIRECTORY must be specified!")
|
|
print(usage)
|
|
sys.exit(1)
|
|
|
|
if sys.argv[1] == "-h":
|
|
print(usage)
|
|
sys.exit(0)
|
|
|
|
src_media_dir = sys.argv[1]
|
|
src_assets_dir = os.path.join(src_media_dir, "assets")
|
|
#Place the resulting media in a subdirectory called "media"
|
|
dest_media_dir = os.path.join(sys.argv[2], "media")
|
|
dest_assets_dir = os.path.join(dest_media_dir, "assets")
|
|
|
|
maxSize = None
|
|
if len(sys.argv) > 3:
|
|
maxSize = int(sys.argv[3])
|
|
|
|
if not os.path.exists(dest_media_dir):
|
|
os.makedirs(dest_media_dir)
|
|
|
|
#First copy all directories that we should just provide unchanged
|
|
copytree(os.path.join(src_media_dir, "assets_external/caelum"), os.path.join(dest_media_dir, "assets_external/caelum"))
|
|
copytree(os.path.join(src_media_dir, "assets_external/OGRE"), os.path.join(dest_media_dir, "assets_external/OGRE"))
|
|
#Copy license files
|
|
shutil.copy(os.path.join(src_media_dir, "LICENSING.txt"), dest_media_dir)
|
|
shutil.copy(os.path.join(src_media_dir, "COPYING.txt"), dest_media_dir)
|
|
|
|
#Now walk through all files in assets looking for references to textures etc.
|
|
reTextureAlias = re.compile(r"^[^/].*set_texture_alias [\w]* (.*)")
|
|
reTexture = re.compile(r"^[^\w]*texture (.*)")
|
|
reParticleImage = re.compile(r".*image\s*(.*)")
|
|
reIcon = re.compile(r'.*icon=\"([^\"]*)\"')
|
|
reMesh = re.compile(r'.*mesh=\"([^\"]*)\"')
|
|
reDiffuse = re.compile(r'.*diffusetexture=\"([^\"]*)\"')
|
|
reNormal = re.compile(r'.*normalmaptexture=\"([^\"]*)\"')
|
|
reFilename = re.compile(r'.*filename=\"([^\"]*)\"')
|
|
|
|
#Put all assets here, as paths relative to the "assets" directory in the media repo.
|
|
assets = set()
|
|
assets_no_conversion = set()
|
|
|
|
#Parse .material and .particle files, looking for references to textures. Skip "source" directories.
|
|
for dirpath, dirnames, files in os.walk(src_assets_dir):
|
|
if "source" in dirnames:
|
|
dirnames.remove("source")
|
|
for filename in fnmatch.filter(files, "*.material"):
|
|
filepath = os.path.join(dirpath, filename)
|
|
assets.add(os.path.relpath(filepath, src_assets_dir))
|
|
file = open(filepath)
|
|
for line in file:
|
|
aMatch = reTextureAlias.match(line)
|
|
if aMatch:
|
|
assets.add(aMatch.group(1))
|
|
continue
|
|
aMatch = reTexture.match(line)
|
|
if aMatch:
|
|
assets.add(aMatch.group(1))
|
|
continue
|
|
for filename in fnmatch.filter(files, "*.particle"):
|
|
filepath = os.path.join(dirpath, filename)
|
|
assets.add(os.path.relpath(filepath, src_assets_dir))
|
|
file = open(filepath)
|
|
for line in file:
|
|
aMatch = reParticleImage.match(line)
|
|
if aMatch:
|
|
assets.add(aMatch.group(1))
|
|
assets_no_conversion.add(aMatch.group(1)) #Particle scripts can't handle DDS textures
|
|
continue
|
|
for filename in files:
|
|
if filename.endswith(('.modeldef', '.entitymap', '.skeleton', '.mesh', '.program', '.cg', '.glsl', '.vert', '.frag', '.hlsl', '.overlay', '.compositor')):
|
|
assets.add(os.path.relpath(os.path.join(dirpath, filename), src_assets_dir))
|
|
|
|
#Copy files found in the "common" assets. Skip "source" directories.
|
|
for dirpath, dirnames, files in os.walk(os.path.join(src_assets_dir, "common")):
|
|
if "source" in dirnames:
|
|
dirnames.remove("source")
|
|
for filename in files:
|
|
if filename.endswith(('.modeldef', '.entitymap', '.jpg', '.png', '.dds', '.skeleton', '.mesh', '.ttf', '.material', '.program', '.cg', '.glsl', '.vert', '.frag', '.hlsl', '.overlay', '.compositor', '.fontdef')):
|
|
assets.add(os.path.relpath(os.path.join(dirpath, filename), src_assets_dir))
|
|
|
|
#Parse .modeldef and .terrain files from the Ember source and look for references
|
|
datadir = os.path.join("@PROJECT_SOURCE_DIR@", "data")
|
|
for directory in [datadir, src_assets_dir]:
|
|
print("Looking for data files in " + directory)
|
|
for dirpath, dirnames, files in os.walk(directory):
|
|
if "source" in dirnames:
|
|
dirnames.remove("source")
|
|
for filename in fnmatch.filter(files, "*.modeldef*"):
|
|
file = open(os.path.join(dirpath, filename))
|
|
for line in file:
|
|
aMatch = reIcon.match(line)
|
|
if aMatch:
|
|
assets.add(aMatch.group(1))
|
|
continue
|
|
aMatch = reMesh.match(line)
|
|
if aMatch:
|
|
assets.add(aMatch.group(1))
|
|
continue
|
|
for filename in fnmatch.filter(files, "*.terrain*"):
|
|
file = open(os.path.join(dirpath, filename))
|
|
for line in file:
|
|
aMatch = reDiffuse.match(line)
|
|
if aMatch:
|
|
assets.add(aMatch.group(1))
|
|
continue
|
|
aMatch = reNormal.match(line)
|
|
if aMatch:
|
|
assets.add(aMatch.group(1))
|
|
continue
|
|
for filename in fnmatch.filter(files, "*.sounddef*"):
|
|
file = open(os.path.join(dirpath, filename))
|
|
for line in file:
|
|
aMatch = reFilename.match(line)
|
|
if aMatch:
|
|
assets.add(aMatch.group(1))
|
|
continue
|
|
|
|
|
|
|
|
|
|
licenses = set()
|
|
collect_licenses(src_assets_dir, licenses, assets)
|
|
|
|
assets.update(licenses)
|
|
|
|
copied, converted, skipped, errors, originalSize, destinationSize = copy_assets(src_assets_dir, dest_assets_dir, assets, assets_no_conversion, maxSize)
|
|
print("Media conversion completed.\n{0} files copied, {1} images converted, {2} files skipped, {3} with errors".format(copied, converted, skipped, errors))
|
|
print("Original size: {0} MB".format(originalSize / 1000000))
|
|
print("Destination size: {0} MB".format(destinationSize / 1000000))
|
|
|
|
pass
|
|
|
|
if __name__ == '__main__':
|
|
main() |