From 3a79977bcf01f83f6be3e31501f5d17b458d46f2 Mon Sep 17 00:00:00 2001 From: Beq Date: Sun, 30 Apr 2023 11:11:22 +0100 Subject: [PATCH] Rework deployment script to unpack and support discord webhook need to fetch the script Make script more tolerant of errors. Fix extraction of build_type for GitHub layout Need to ignore pre-existing folders We have multiple artifacts that we are merging into a common tree. despite the clash of folders the files are not going to clash. revert the artifact collection to per platform. Simplify 64/32 test to remove ReleaseType dependency --- .github/workflows/build_viewer.yml | 62 ++++++-- fsutils/download_list.py | 248 ++++++++++++++++++----------- 2 files changed, 210 insertions(+), 100 deletions(-) diff --git a/.github/workflows/build_viewer.yml b/.github/workflows/build_viewer.yml index 3b02062d89..8adec3ef10 100644 --- a/.github/workflows/build_viewer.yml +++ b/.github/workflows/build_viewer.yml @@ -8,7 +8,7 @@ on: - cron: '00 03 * * *' # Run every day at 3am UTC env: AUTOBUILD_VARIABLES_FILE: ${{github.workspace}}/build-variables/variables - EXTRA_ARGS: -DFMODSTUDIO=ON -DUSE_KDU=ON --crashreporting + EXTRA_ARGS: -DUSE_FMODSTUDIO=ON -DUSE_KDU=ON --crashreporting build_secrets_checkout: ${{github.workspace}}/signing @@ -84,6 +84,8 @@ jobs: fi if [[ "${{ matrix.addrsize }}" == "64" ]]; then FS_RELEASE_CHAN="${FS_RELEASE_TYPE}x64" + else + FS_RELEASE_CHAN=${FS_RELEASE_TYPE} fi echo "FS_RELEASE_TYPE=${FS_RELEASE_TYPE}" >> $GITHUB_ENV echo "FS_RELEASE_CHAN=${FS_RELEASE_CHAN}" >> $GITHUB_ENV @@ -198,27 +200,65 @@ jobs: run: rm *${{ env.fallback_platform }}*bz2 shell: bash - name: Configure - run: autobuild configure --debug -c ReleaseFS -A${{matrix.addrsize}} -- --package --chan ${{env.FS_RELEASE_CHAN}} ${{env.EXTRA_ARGS}} ${{env.FS_GRID}} + run: autobuild configure -c ReleaseFS -A${{matrix.addrsize}} -- --package --chan ${{env.FS_RELEASE_CHAN}} ${{env.EXTRA_ARGS}} ${{env.FS_GRID}} shell: bash - name: build - run: autobuild build --debug -c ReleaseFS -A${{matrix.addrsize}} --no-configure + run: autobuild build -c ReleaseFS -A${{matrix.addrsize}} --no-configure shell: bash - name: Publish artifacts + if: runner.os == 'Windows' uses: actions/upload-artifact@v3 with: name: ${{ env.FS_RELEASE_TYPE }}-${{ matrix.os }}-${{ matrix.addrsize }}-${{ matrix.grid }}-artifacts.zip path: | - build-*/newview/Release/*Setup.exe - build-*/newview/Release/*.xz - build-linux-*/newview/*.bz2 - build-darwin-*/newview/*.dmg + build-*/newview/Release/*Setup.exe + build-*/newview/Release/*.xz + + - name: publish Linux artifacts + if: runner.os == 'Linux' + uses: actions/upload-artifact@v3 + with: + name: ${{ env.FS_RELEASE_TYPE }}-${{ matrix.os }}-${{matrix.addrsize}}-${{matrix.grid}}-artifacts.zip + path: | + build-linux-*/newview/*.xz + build-linux-*/newview/*.bz2 + + - name: publish MacOS artifacts + if: runner.os == 'macOS' + uses: actions/upload-artifact@v3 + with: + name: ${{ env.FS_RELEASE_TYPE }}-${{ matrix.os }}-${{matrix.addrsize}}-${{matrix.grid}}-artifacts.zip + path: | + build-darwin-*/newview/*.dmg + build-darwin-*/newview/*.bz2 deploy: runs-on: ubuntu-latest needs: build_matrix if: always() steps: + - name: Checkout files + uses: Bhacaz/checkout-files@v2 + with: + files: fsutils/download_list.py + branch: ${{ github.head_ref || github.ref_name || 'master' }} + - name: Install discord-webhook library + run: pip install discord-webhook + + - name: find channel from Branch name + run: | + if [[ "${{ github.ref_name }}" == *Release* ]]; then + FS_RELEASE_FOLDER=release + else + if [[ "${{github.event_name}}" == 'schedule' ]]; then + FS_RELEASE_FOLDER=nightly + else + FS_RELEASE_FOLDER=preview + fi + fi + echo "FS_RELEASE_FOLDER=${FS_RELEASE_FOLDER}" >> $GITHUB_ENV + - name: Download artifacts uses: actions/download-artifact@v3 id: download @@ -227,11 +267,15 @@ jobs: - name: List artifacts download run: ls -R working-directory: ${{steps.download.outputs.download-path}} - + + - name: Reorganise artifacts ready for server upload. + run: python ./fsutils/download_list.py -u ${{steps.download.outputs.download-path}} -w ${{ secrets.RELEASE_WEBHOOK_URL }} + - name: Setup rclone and download the folder uses: beqjanus/setup-rclone@main with: rclone_config: ${{ secrets.RCLONE_CONFIG }} - name: Copy files to remote host - run: rclone copy ${{steps.download.outputs.download-path}} fs_deploy:test/ + run: rclone copy ${{steps.download.outputs.download-path}}/${{ env.FS_RELEASE_FOLDER }} fs_deploy:${{ env.FS_RELEASE_FOLDER }} + diff --git a/fsutils/download_list.py b/fsutils/download_list.py index 0747ae7e4c..c68bb7b083 100644 --- a/fsutils/download_list.py +++ b/fsutils/download_list.py @@ -6,14 +6,9 @@ import time import zipfile import glob import shutil +from discord_webhook import DiscordWebhook + -# iterate over the files in a directory and pass them to a command line subshell -def get_files(path): - files = [] - for root, dirs, files in os.walk(path): - # print(f"Found : {files}") - return files - return None # run a command line subshell and return the output @@ -65,6 +60,14 @@ def get_files(path): # MD5: 9D5D8021F376194B42F6E7D8E537E45E # ------------------------------------------------------------------------------------------------------- +# iterate over the files in a directory and pass them to a command line subshell +def get_files(path): + files = [] + for root, dirs, filenames in os.walk(path): + for filename in filenames: + files.append(filename) + print(f"Found : {files} on {path}") + return files def run_cmd(cmd): # print(cmd) @@ -73,12 +76,12 @@ def run_cmd(cmd): #using the md5sum command get the md5 for the file def get_md5(mdfile): - # print(f"mdfile is {mdfile}") md5sum = run_cmd(f"md5sum {mdfile}") #split md5sum on space md5sum = md5sum.split()[0] #remove leading '\' md5sum = md5sum[1:] + print(f"generating md5sum for {mdfile} as {md5sum}") return md5sum def unzip_file(zip_file, unzip_dir): @@ -86,6 +89,7 @@ def unzip_file(zip_file, unzip_dir): zip_ref.extractall(unzip_dir) def flatten_tree(tree_root): + print(f"Flattening tree {tree_root}") for root, flatten_dirs, files in os.walk(tree_root, topdown=False): for file in files: # Construct the full path to the file @@ -107,6 +111,8 @@ parser = argparse.ArgumentParser( ) parser.add_argument("-r", "--release", required=False, default=False, action="store_true", help="use the release folder in the target URL") parser.add_argument("-u", "--unzip", required=False, default=False, action="store_true", help="unzip the github artifact first") +parser.add_argument("-w", "--webhook", help="post details to the webhook") + # add path_to_directory required parameter to parser parser.add_argument("path_to_directory", help="path to the directory in which we'll look for the files") @@ -114,102 +120,162 @@ args = parser.parse_args() path_to_directory = args.path_to_directory release = args.release +# Create a webhook object with the webhook URL +if args.webhook: + webhook = DiscordWebhook(url=args.webhook) + dirs = ["windows", "mac", "linux"] -if args.unzip: - # unzip the github artifact for this OS (`dir`) into the folder `dir` - # get the .zip files in args.path_to_directory using glob - zips = glob.glob(f"{args.path_to_directory}/*.zip") - for file in zips: - # print(f"unzipping {file}") - if "ubuntu" in file.lower(): - unzip_file(file, os.path.join(args.path_to_directory, "linux")) - if "windows" in file.lower(): - unzip_file(file, os.path.join(args.path_to_directory, "windows")) - if "macos" in file.lower(): - unzip_file(file, os.path.join(args.path_to_directory, "mac")) +# build_types is a map from Beta, Release and Nightly to folder names preview release and nightly +build_types = { + "Beta": "preview", + "Release": "release", + "Nightly": "nightly" +} + +target_folder = { + "ubuntu":"linux", + "windows":"windows", + "macos":"mac" +} + +# unzip the github artifact for this OS (`dir`) into the folder `dir` +# get the .zip files in args.path_to_directory using glob +print(f"Processing artifacts in {args.path_to_directory}") +build_types_created = set() +zips = glob.glob(f"{args.path_to_directory}/*.zip") +for file in zips: + # print(f"unzipping {file}") + #extract first word (delimited by '-' from the file name) + # build_type is a fullpath but we only want the last folder, remove the leading part of the path leaving just the foldername using basename + filename = os.path.basename(file) + build_type = filename.split("-")[0] + platform = filename.split("-")[1].lower() + + # print(f"build_type is {build_type}") + if build_type not in build_types: + print(f"Invalid build_type {build_type} using file {file}") + continue + else: + build_folder = build_types[build_type] + + build_types_created.add(build_type) + + build_type_dir = os.path.join(args.path_to_directory, build_folder) + + if platform not in target_folder: + print(f"Invalid platform {platform} using file {file}") + continue + + unpack_folder = os.path.join(build_type_dir, target_folder[platform]) + print(f"unpacking {filename} to {unpack_folder}") + + if os.path.isfile(file): + # this is an actual zip file + unzip_file(file, unpack_folder) + else: + # Create the destination folder if it doesn't exist + # if not os.path.exists(unpack_folder): + # os.makedirs(unpack_folder) + # Copy the contents of the source folder to the destination folder recursively + shutil.copytree(file, unpack_folder, dirs_exist_ok=True) + +output = "" +for build_type in build_types_created: + build_type_dir = os.path.join(args.path_to_directory, build_types[build_type]) + if not os.path.exists(build_type_dir): + print(f"Unexpected error: {build_type_dir} does not exist, even though it was in the set.") + continue + # loop over the folder in the build_type_dir for dir in dirs: - flatten_tree(os.path.join(args.path_to_directory, dir)) + print(f"Cleaning up {dir}") + # Traverse the directory tree and move all of the files to the root directory + flatten_tree(os.path.join(build_type_dir, dir)) # Now move the symbols files to the symbols folder - symbols_folder = os.path.join(args.path_to_directory, "symbols") + # prep the symbols folder + symbols_folder = os.path.join(build_type_dir, "symbols") os.mkdir(symbols_folder) - # Traverse the directory tree and move all of the files to the root directory - symbol_archives = glob.glob(f"{args.path_to_directory}/**/*_hvk*", recursive=True) + symbol_archives = glob.glob(f"{build_type_dir}/**/*_hvk*", recursive=True) for sym_file in symbol_archives: print(f"Moving {sym_file} to {symbols_folder}") shutil.move(sym_file, symbols_folder) - symbol_archives = glob.glob(f"{args.path_to_directory}/**/*_oss*", recursive=True) + symbol_archives = glob.glob(f"{build_type_dir}/**/*_oss*", recursive=True) for sym_file in symbol_archives: print(f"Moving {sym_file} to {symbols_folder}") shutil.move(sym_file, symbols_folder) - -file_dict = {} -md5_dict = {} + # While we're at it, let's print the md5 listing + file_dict = {} + md5_dict = {} + platforms_printable = {"windows":"MS Windows", "mac":"MacOS", "linux":"Linux"} + grids_printable = {"SL":"Second Life", "OS":"OpenSim"} -for dir in dirs: - dir = dir.lower() - files = get_files(os.path.join(args.path_to_directory, dir)) - for file in files: - full_file = os.path.join(args.path_to_directory, dir, file) - md5 = get_md5(full_file) - base_name = os.path.basename(file) - if "-Release-" in base_name or "-Beta-" in base_name: - wordsize = "32" - else: - wordsize = "64" - - if "FirestormOS-" in base_name: - grid = "OS" - else: - grid = "SL" - - if dir in dirs: - file_dict[f"{grid}{dir}{wordsize}"] = full_file - md5_dict[f"{grid}{dir}{wordsize}"] = md5 - -download_root_preview = "https://downloads.firestormviewer.org/preview" -download_root_release = "https://downloads.firestormviewer.org/release" - -if args.release: - download_root = download_root_release -else: - download_root = download_root_preview - -print(''' -DOWNLOADS''') - -platforms_printable = {"windows":"MS Windows", "mac":"MacOS", "linux":"Linux"} -grids_printable = {"SL":"Second Life", "OS":"OpenSim"} - -for dir in dirs: - print(f'''------------------------------------------------------------------------------------------------------- -{platforms_printable[dir]} -''') - dir=dir.lower() - wordsize = "64" - platform = f"{platforms_printable[dir]}" - for grid in ["SL", "OS"]: - grid_printable = f"{grids_printable[grid]}" + download_root = "https://downloads.firestormviewer.org/{build_types[build_type]}/" + for dir in dirs: + print(f"Getting files for {dir} in {build_type_dir}") + files = get_files(os.path.join(build_type_dir, dir)) try: - print (f"{platform} for {grid_printable} ({wordsize}-bit)") - print ( "{}/{}/{}".format(download_root,dir,os.path.basename(file_dict[f"{grid}{dir}{wordsize}"])) ) - print () - print ( "MD5: {}".format(md5_dict[f"{grid}{dir}{wordsize}"]) ) - print () - if(dir == "windows"): - # Need to do 32 bit as well - wordsize = "32" - print (f"{platform} for {grid_printable} ({wordsize}-bit)") - print ( "{}/{}/{}".format(download_root,dir,os.path.basename(file_dict[f"{grid}{dir}{wordsize}"])) ) - print () - print ( "MD5: {}".format(md5_dict[f"{grid}{dir}{wordsize}"]) ) - print () - wordsize = "64" - except KeyError: - print (f"{platform} for {grid_printable} ({wordsize}-bit) - NOT AVAILABLE") - print () + for file in files: + full_file = os.path.join(build_type_dir, dir, file) + md5 = get_md5(full_file) + base_name = os.path.basename(file) + if "x64" in base_name: + wordsize = "64" + else: + wordsize = "32" + + if "FirestormOS-" in base_name: + grid = "OS" + else: + grid = "SL" -print(''' --------------------------------------------------------------------------------------------------------''') + if dir in dirs: + file_dict[f"{grid}{dir}{wordsize}"] = full_file + md5_dict[f"{grid}{dir}{wordsize}"] = md5 + except TypeError: + print(f"No files found for {dir} in {build_type_dir}") + + + output += f''' +DOWNLOADS - {build_type} +''' + + output += f'''------------------------------------------------------------------------------------------------------- +{platforms_printable[dir]} +''' + dir = dir.lower() + wordsize = "64" + platform = f"{platforms_printable[dir]}" + for grid in ["SL", "OS"]: + grid_printable = f"{grids_printable[grid]}" + try: + output += f"{platform} for {grid_printable} ({wordsize}-bit)\n" + output += f"{download_root}/{dir}/{os.path.basename(file_dict[f'{grid}{dir}{wordsize}'])}\n" + output += "\n" + output += f"MD5: {md5_dict[f'{grid}{dir}{wordsize}']}\n" + output += "\n" + if dir == "windows": + # Need to do 32 bit as well + wordsize = "32" + output += f"{platform} for {grid_printable} ({wordsize}-bit)\n" + output += f"{download_root}/{dir}/{os.path.basename(file_dict[f'{grid}{dir}{wordsize}'])}\n" + output += "\n" + output += f"MD5: {md5_dict[f'{grid}{dir}{wordsize}']}\n" + output += "\n" + wordsize = "64" + except KeyError: + output += f"{platform} for {grid_printable} ({wordsize}-bit) - NOT AVAILABLE\n" + output += "\n" + output += ''' +------------------------------------------------------------------------------------------------------- +''' + + if args.webhook: + # Add the message to the webhook + webhook.set_content(content=output) + # Send the webhook + response = webhook.execute() + # Print the response + print(f"Webhook response: {response}") + print(output)