From e422e80377c221cf5391bc45f26af21c551f8852 Mon Sep 17 00:00:00 2001 From: Beq Date: Mon, 19 Dec 2022 10:49:11 +0000 Subject: [PATCH] Refactor and improve to accept github zip archives and autounpack --- fsutils/download_list.py | 170 ++++++++++++++++++++------------------- 1 file changed, 89 insertions(+), 81 deletions(-) diff --git a/fsutils/download_list.py b/fsutils/download_list.py index 13defa2290..5981e7c511 100644 --- a/fsutils/download_list.py +++ b/fsutils/download_list.py @@ -3,6 +3,9 @@ import argparse import os import sys import time +import zipfile +import glob +import shutil # iterate over the files in a directory and pass them to a command line subshell def get_files(path): @@ -78,6 +81,23 @@ def get_md5(mdfile): md5sum = md5sum[1:] return md5sum +def unzip_file(zip_file, unzip_dir): + with zipfile.ZipFile(zip_file, 'r') as zip_ref: + zip_ref.extractall(unzip_dir) + +def flatten_tree(tree_root): + for root, flatten_dirs, files in os.walk(tree_root, topdown=False): + for file in files: + # Construct the full path to the file + file_path = os.path.join(root, file) + # Move the file to the root directory + shutil.move(file_path, tree_root) + for dir in flatten_dirs: + # Construct the full path to the subdirectory + subdir_path = os.path.join(root, dir) + # Delete the subdirectory and its contents + shutil.rmtree(subdir_path) + # parse args first arg optional -r (release) second arg mandatory string path_to_directory @@ -85,59 +105,68 @@ parser = argparse.ArgumentParser( prog="print_download_list", description="Prints the list of files for download and their md5 checksums" ) -parser.add_argument("-r", "--release", required=False, default=False, action="store_true") +parser.add_argument("-r", "--release", required=False, default=False, action="store_true", help="use the release folder in the target URL") +parser.add_argument("-u", "--unzip", required=False, default=False, action="store_true", help="unzip the github artifact first") # add path_to_directory required parameter to parser -parser.add_argument("path_to_directory") +parser.add_argument("path_to_directory", help="path to the directory in which we'll look for the files") args = parser.parse_args() path_to_directory = args.path_to_directory release = args.release -platforms_printable = {"windows":"MS Windows", "mac":"MacOS", "linux":"Linux"} dirs = ["windows", "mac", "linux"] -print(''' -DOWNLOADS''') +if args.unzip: + # unzip the github artifact for this OS (`dir`) into the folder `dir` + # get the .zip files in args.path_to_directory using glob + zips = glob.glob(f"{args.path_to_directory}/*.zip") + for file in zips: + # print(f"unzipping {file}") + if "ubuntu" in file.lower(): + unzip_file(file, os.path.join(args.path_to_directory, "linux")) + if "windows" in file.lower(): + unzip_file(file, os.path.join(args.path_to_directory, "windows")) + if "macos" in file.lower(): + unzip_file(file, os.path.join(args.path_to_directory, "mac")) + for dir in dirs: + flatten_tree(os.path.join(args.path_to_directory, dir)) + # Now move the symbols files to the symbols folder + symbols_folder = os.path.join(args.path_to_directory, "symbols") + os.mkdir(symbols_folder) + # Traverse the directory tree and move all of the files to the root directory + symbol_archives = glob.glob(f"{args.path_to_directory}/**/*_hvk*", recursive=True) + for sym_file in symbol_archives: + print(f"Moving {sym_file} to {symbols_folder}") + shutil.move(sym_file, symbols_folder) + symbol_archives = glob.glob(f"{args.path_to_directory}/**/*_oss*", recursive=True) + for sym_file in symbol_archives: + print(f"Moving {sym_file} to {symbols_folder}") + shutil.move(sym_file, symbols_folder) + + file_dict = {} md5_dict = {} + for dir in dirs: - # print(f"looking in {os.path.join(sys.argv[1], dir)}") dir = dir.lower() files = get_files(os.path.join(args.path_to_directory, dir)) for file in files: - full_file = os.path.join(sys.argv[1], dir, file) + full_file = os.path.join(args.path_to_directory, dir, file) md5 = get_md5(full_file) - if(dir=="windows"): - # print(f"testing {file} as {full_file}") - if "Firestorm-Release-" in os.path.basename(file): - file_dict["SLWin32"] = full_file - md5_dict["SLWin32"] = md5 - elif "FirestormOS-Release-" in os.path.basename(file): - file_dict["OSWin32"] = full_file - md5_dict["OSWin32"] = md5 - elif "Firestorm-Releasex64-" in os.path.basename(file): - file_dict["SLWin64"] = full_file - md5_dict["SLWin64"] = md5 - elif "FirestormOS-Releasex64-" in os.path.basename(file): - file_dict["OSWin64"] = full_file - md5_dict["OSWin64"] = md5 - if(dir=="mac"): - # print(f"testing {file} as {full_file}") - if "Firestorm-Releasex64-" in os.path.basename(file): - # print(f"storing {file} as SLMac64") - file_dict["SLMac64"] = full_file - md5_dict["SLMac64"] = md5 - elif "FirestormOS-Releasex64-" in os.path.basename(file): - file_dict["OSMac64"] = full_file - md5_dict["OSMac64"] = md5 - if(dir=="linux"): - # print(f"testing {file} as {full_file}") - if "Firestorm-Releasex64-" in os.path.basename(file): - file_dict["SLLinux64"] = full_file - md5_dict["SLLinux64"] = md5 - elif "FirestormOS-Releasex64-" in os.path.basename(file): - file_dict["OSLinux64"] = full_file - md5_dict["OSLinux64"] = md5 + base_name = os.path.basename(file) + if "-Release-" in base_name: + wordsize = "32" + else: + wordsize = "64" + + if "FirestormOS-" in base_name: + grid = "OS" + else: + grid = "SL" + + if dir in dirs: + file_dict[f"{grid}{dir}{wordsize}"] = full_file + md5_dict[f"{grid}{dir}{wordsize}"] = md5 download_root_preview = "https://downloads.firestormviewer.org/preview" download_root_release = "https://downloads.firestormviewer.org/release" @@ -147,57 +176,36 @@ if args.release: else: download_root = download_root_preview +print(''' +DOWNLOADS''') + +platforms_printable = {"windows":"MS Windows", "mac":"MacOS", "linux":"Linux"} +grids_printable = {"SL":"Second Life", "OS":"OpenSim"} + for dir in dirs: print(f'''------------------------------------------------------------------------------------------------------- {platforms_printable[dir]} ''') dir=dir.lower() - if(dir=="linux"): - print ("Linux for SL") - print ( "{}/{}/{}".format(download_root,dir,os.path.basename(file_dict["SLLinux64"])) ) + wordsize = "64" + platform = f"{platforms_printable[dir]}" + for grid in ["SL", "OS"]: + grid_printable = f"{grids_printable[grid]}" + print (f"{platform} for {grid_printable} ({wordsize}-bit)") + print ( "{}/{}/{}".format(download_root,dir,os.path.basename(file_dict[f"{grid}{dir}{wordsize}"])) ) print () - print ( "MD5: {}".format(md5_dict["SLLinux64"]) ) - print () - print ("Linux for OpenSim") - print ( "{}/{}/{}".format(download_root,dir,os.path.basename(file_dict["OSLinux64"])) ) - print () - print ( "MD5: {}".format(md5_dict["OSLinux64"]) ) - print () - if(dir=="mac"): - print ("MacOS for SL") - print ( "{}/{}/{}".format(download_root,dir,os.path.basename(file_dict["SLMac64"])) ) - print () - print ( "MD5: {}".format(md5_dict["SLMac64"]) ) - print () - print ("MacOS for OpenSim") - print ( "{}/{}/{}".format(download_root,dir,os.path.basename(file_dict["OSMac64"])) ) - print () - print ( "MD5: {}".format(md5_dict["OSMac64"]) ) - print () - if(dir=="windows"): - print ("Windows for SL 64-bit") - print ( "{}/{}/{}".format(download_root,dir,os.path.basename(file_dict["SLWin64"])) ) - print () - print ( "MD5: {}".format(md5_dict["SLWin64"]) ) - print () - print ("Windows for SL 32-Bit") - print ( "{}/{}/{}".format(download_root,dir,os.path.basename(file_dict["SLWin32"])) ) - print () - print ( "MD5: {}".format(md5_dict["SLWin32"]) ) - print () - print ("Windows for OS 64-bit") - print ( "{}/{}/{}".format(download_root,dir,os.path.basename(file_dict["OSWin64"])) ) - print () - print ( "MD5: {}".format(md5_dict["OSWin64"]) ) - print () - print ("Windows for OS 32-Bit") - print ( "{}/{}/{}".format(download_root,dir,os.path.basename(file_dict["OSWin32"])) ) - print () - print ( "MD5: {}".format(md5_dict["OSWin32"]) ) + print ( "MD5: {}".format(md5_dict[f"{grid}{dir}{wordsize}"]) ) print () + if(dir == "windows"): + # Need to do 32 bit as well + wordsize = "32" + print (f"{platform} for {grid_printable} ({wordsize}-bit)") + print ( "{}/{}/{}".format(download_root,dir,os.path.basename(file_dict[f"{grid}{dir}{wordsize}"])) ) + print () + print ( "MD5: {}".format(md5_dict[f"{grid}{dir}{wordsize}"]) ) + print () + wordsize = "64" print(''' -------------------------------------------------------------------------------------------------------''') - -