Massive overhaul of build publishing

This will doubtless break horribly.

.github/workflows/build_viewer.yml

@@ -391,27 +391,30 @@ jobs:
         with:
           sparse-checkout: |
             fsutils/download_list.py
+            fsutils/build_config.json
+            fsutils/build_config.py
           sparse-checkout-cone-mode: false
           ref: ${{ github.head_ref || github.ref_name || 'master' }}
           fetch-depth: 1
       - name: Install discord-webhook library
         run: pip install discord-webhook
 
-      - name: find channel and webhook from Branch name
+      - name: find channel and webhook from build_matrix outputs
         run: | 
-          if [[ "${{ github.ref_name }}" == Firestorm* ]]; then
+          viewer_release_type=${{ needs.build_matrix.outputs.viewer_release_type }}
+          if [[ "$viewer_release_type" == "Release" ]]; then
             FS_RELEASE_FOLDER=release
             FS_BUILD_WEBHOOK_URL=${{ secrets.RELEASE_WEBHOOK_URL }}
-          elif [[ "${{ github.ref_name }}" == *review* ]]; then
+          elif [[ "$viewer_release_type" == "Beta" ]]; then
             FS_RELEASE_FOLDER=preview
             FS_BUILD_WEBHOOK_URL=${{ secrets.BETA_WEBHOOK_URL }}
-          elif [[ "${{ github.ref_name }}" == *alpha* ]]; then
+          elif [[ "$viewer_release_type" == "Alpha" ]]; then
             FS_RELEASE_FOLDER=test
             FS_BUILD_WEBHOOK_URL=${{ secrets.BETA_WEBHOOK_URL }}
-            elif [[ "${{ github.ref_name }}" == *nightly* ]] || [[ "${{ github.event_name }}" == 'schedule' ]]; then
+            elif [[ "$viewer_release_type" == "Nightly" ]] || [[ "${{ github.event_name }}" == 'schedule' ]]; then
             FS_RELEASE_FOLDER=nightly
             FS_BUILD_WEBHOOK_URL=${{ secrets.NIGHTLY_WEBHOOK_URL }}
-            elif [[ "${{github.event_name }}" == "workflow_dispatch" ]]; then
+            elif [[ "$viewer_release_type" == "Manual" ]]; then
             FS_RELEASE_FOLDER=test
             FS_BUILD_WEBHOOK_URL=${{ secrets.MANUAL_WEBHOOK_URL }}
           else
@@ -430,7 +433,12 @@ jobs:
         working-directory: ${{steps.download.outputs.download-path}}
 
       - name: Reorganise artifacts ready for server upload.
-        run: python ./fsutils/download_list.py -u ${{steps.download.outputs.download-path}} -w ${{ env.FS_BUILD_WEBHOOK_URL }}
+        env:
+          FS_VIEWER_CHANNEL: ${{ needs.build_matrix.outputs.viewer_channel }}
+          FS_VIEWER_VERSION: ${{ needs.build_matrix.outputs.viewer_version }}
+          FS_VIEWER_BUILD: ${{ needs.build_matrix.outputs.viewer_build }}
+          FS_VIEWER_RELEASE_TYPE: ${{ needs.build_matrix.outputs.viewer_release_type }}          
+        run: python ./fsutils/download_list.py ${{steps.download.outputs.download-path}} -w ${{ env.FS_BUILD_WEBHOOK_URL }}
 
       - name: Setup rclone and download the folder
         uses: beqjanus/setup-rclone@main

fsutils/build_config.json

@@ -0,0 +1,47 @@
+{
+    "os_download_dirs": [
+        "windows", "mac", "linux"
+    ],
+    "fs_version_mgr_platform": {
+        "windows": "win",
+        "mac": "mac",
+        "linux": "lin"
+    },
+    "build_type_hosted_folder": {
+        "Release": "release",
+        "Beta": "preview",
+        "Alpha": "test",
+        "Nightly": "nightly"
+    },
+    "os_hosted_folder": {
+        "windows": "windows",
+        "darwin": "mac",
+        "linux": "linux"
+    },
+    "platforms_printable": {
+        "windows": "MS Windows",
+        "mac": "MacOS",
+        "linux": "Linux"
+    },
+    "grids_printable": {
+        "SL": "Second Life",
+        "OS": "OpenSim"
+    },
+    "download_root": "https://downloads.firestormviewer.org",
+    "viewer_channel_mapping": {
+        "Release": "release",
+        "Beta": "beta",
+        "Alpha": "alpha",
+        "Nightly": "nightly"
+    },
+    "grid_type_mapping": {
+        "SecondLife": "sl",
+        "OpenSim": "os"
+    },
+    "build_type_mapping": {
+        "regular": "regular",
+        "avx": "avx",
+        "tracy": "tracy",
+        "arm": "arm"
+    }
+}

fsutils/build_config.py

@@ -0,0 +1,20 @@
+# build_config.py
+
+import json
+
+class BuildConfig:
+    def __init__(self, config_file='config.json'):
+        with open(config_file, 'r') as f:
+            config_data = json.load(f)
+
+        self.supported_os_dirs = config_data.get('os_download_dirs', [])
+        # channel_to_build_type is a map from Beta, Release and Nightly to folder names preview release and nightly
+        self.build_type_hosted_folder = config_data.get('build_types', {})
+        self.fs_version_mgr_platform = config_data.get('fs_version_mgr_platform', {})
+        self.os_hosted_folder = config_data.get('target_folder', {})
+        self.platforms_printable = config_data.get('platforms_printable', {})
+        self.grids_printable = config_data.get('grids_printable', {})
+        self.download_root = config_data.get('download_root', '')
+        self.viewer_channel_mapping = config_data.get('viewer_channel_mapping', {})
+        self.grid_type_mapping = config_data.get('grid_type_mapping', {})
+        self.build_type_mapping = config_data.get('build_type_mapping', {})

fsutils/download_list.py

@@ -6,10 +6,37 @@ import time
 import zipfile
 import glob
 import shutil
+import hashlib
+import pytz
+from datetime import datetime
+import requests
 from discord_webhook import DiscordWebhook
 
+from build_config import BuildConfig
 
+def get_current_date_str():
+    now = datetime.now(pytz.timezone('UTC'))
+    day = now.day
+    month = now.month
+    year = now.year
+    return f"{day}{month}{year}"
 
+def generate_secret(secret_key):
+    current_date = get_current_date_str()
+    data = secret_key + current_date
+    secret_for_api = hashlib.sha1(data.encode()).hexdigest()
+    return secret_for_api
+
+def map_build_type(build_variant):
+    # Map your build_variant to the API's build_type values
+    build_type_mapping = {
+        'Release': 'regular',
+        'Beta': 'beta',
+        'Alpha': 'alpha',
+        'Nightly': 'nightly',
+        # Add other mappings if necessary
+    }
+    return build_type_mapping.get(build_variant, 'regular')
 
 # run a command line subshell and return the output
 
@@ -103,171 +130,300 @@ def flatten_tree(tree_root):
             # Delete the subdirectory and its contents
             shutil.rmtree(subdir_path)
 
+def get_build_variables():
+    """
+    Extracts initial build variables from environment variables.
+    In practice these are set from the outputs of the earlier matrix commands.
+    Returns:
+        dict: A dictionary containing 'version' and 'build_number'.
+    """
+    import os
 
-# parse args first arg optional -r (release) second arg mandatory string path_to_directory
+    version = os.environ.get('FS_VIEWER_VERSION')
+    build_number = os.environ.get('FS_VIEWER_BUILD')
+    release_type = os.environ.get('FS_VIEWER_RELEASE_TYPE')
 
-parser = argparse.ArgumentParser(
+    if not version or not build_number or not release_type:
-    prog="print_download_list",
+        raise ValueError("Environment variables 'FS_VIEWER_VERSION' and 'FS_VIEWER_BUILD' must be set.")
-    description="Prints the list of files for download and their md5 checksums"
-    )
-parser.add_argument("-r", "--release", required=False, default=False, action="store_true", help="use the release folder in the target URL")
-parser.add_argument("-u", "--unzip", required=False, default=False, action="store_true", help="unzip the github artifact first")
-parser.add_argument("-w", "--webhook", help="post details to the webhook")
 
-# add path_to_directory required parameter to parser
+    return {
-parser.add_argument("path_to_directory", help="path to the directory in which we'll look for the files")
+        'version': version,
+        'build_number': build_number,
+        'version_full': f"{version}.{build_number}",
+        'release_type': release_type,
+    }
 
-args = parser.parse_args()
+def get_hosted_folder_for_build_type(build_type, config):
-path_to_directory = args.path_to_directory
+    return config.build_type_hosted_folder.get(
-release = args.release
+        build_type, 
+        config.build_type_hosted_folder.get("Unknown")
+        )
 
-# Create a webhook object with the webhook URL
+def get_supported_build_type(build_type, config):
-if args.webhook:
+    if build_type in config.build_type_hosted_folder:
-    webhook = DiscordWebhook(url=args.webhook)
+        return build_type
+    else:
+        return None
+def get_hosted_folder_for_os_type(os_type, config):
+    return config.build_type_hosted_folder.get(
+        os_type
+        )
 
-dirs = ["windows", "mac", "linux"]
+def get_supported_os(os_name, config):
+    # throws for unexpected os_name
+    return config.os_hosted_folder.get(os_name)
 
-# build_types is a map from Beta, Release and Nightly to folder names preview release and nightly
+def extract_vars_from_zipfile_name(file):
-build_types = {
+    # File is an artifact file sometihng like Nightly-windows-2022-64-sl-artifacts.zip
-    "Alpha": "test",
-    "Beta": "preview",
-    "Release": "release",
-    "Nightly": "nightly",
-    "Unknown": "test"
-}
-
-target_folder = {
-    "ubuntu":"linux",
-    "windows":"windows",
-    "macos":"mac"
-}
-
-# unzip the github artifact for this OS (`dir`) into the folder `dir`
-# get the .zip files in args.path_to_directory using glob 
-print(f"Processing artifacts in {args.path_to_directory}")
-build_types_created = set()
-zips = glob.glob(f"{args.path_to_directory}/*.zip")
-for file in zips:
     # print(f"unzipping {file}")
     #extract first word (delimited by '-' from the file name)
     # build_type is a fullpath but we only want the last folder, remove the leading part of the path leaving just the foldername using basename
     filename = os.path.basename(file)
     build_type = filename.split("-")[0]
     platform = filename.split("-")[1].lower()
+    return filename,build_type, platform
 
-    # print(f"build_type is {build_type}")
-    if build_type not in build_types:
-        print(f"Invalid build_type {build_type} from file {file} using 'Unknown'")
-        build_type = "Unknown"
 
-    build_folder = build_types[build_type]
+def unpack_artifacts(path_to_artifacts_directory, config):
-    
+    build_types_found = {}
-    build_types_created.add(build_type)
+    zips = glob.glob(f"{path_to_artifacts_directory}/*.zip")
+    for file in zips:
+        filename, build_type, platform = extract_vars_from_zipfile_name(file)
 
-    build_type_dir = os.path.join(args.path_to_directory, build_folder)
+        build_type = get_supported_build_type( build_type, config)
+        if build_type == None:
+            print(f"Invalid build_type {build_type} from file {file} using 'Unknown'")
+            build_type = "Unknown"
+        build_folder = get_hosted_folder_for_build_type(build_type, config)
+        build_type_dir = os.path.join(path_to_artifacts_directory, build_folder)
 
-    if platform not in target_folder:
+        try:
-        print(f"Invalid platform {platform} using file {file}")
+            os_folder = get_hosted_folder_for_os_type(platform, config)
-        continue
+        except KeyError:
-    
+            print(f"Invalid OS platform {platform} specified by file {file} - skipping")
-    unpack_folder = os.path.join(build_type_dir, target_folder[platform])
+            continue
-    print(f"unpacking {filename} to {unpack_folder}")
+        except Exception as e:
+            print(f"An error occurred while getting OS folder: {e}")
+            continue
+        
+        unpack_folder = os.path.join(build_type_dir, os_folder)
+        print(f"unpacking {filename} to {unpack_folder}")
+        try:
+            unzip_file(file, unpack_folder)
+        except zipfile.BadZipFile:
+            print(f"Skipping {file} as it is not a valid zip file")
+            continue
+        except Exception as e:
+            print(f"An error occurred while unpacking {file}: {e} , skipping file {filename}")
+            continue
+        
+        if build_type not in build_types_found:
+            build_types_found[build_type] = { 
+                "build_type": build_type,
+                "build_type_folder": build_folder,
+                "build_type_fullpath": build_type_dir,
+                "os_folders": [], 
+            }
+        build_types_found[build_type]["os_folders"].append(os_folder)
+    return build_types_found
 
-    if os.path.isfile(file):
+def restructure_folders(build_type, config):
-        # this is an actual zip file
+    build_type_dir = build_type["build_type_fullpath"]
-        unzip_file(file, unpack_folder)
-    else:
-        # Create the destination folder if it doesn't exist
-        # if not os.path.exists(unpack_folder):
-        #     os.makedirs(unpack_folder)
-        # Copy the contents of the source folder to the destination folder recursively
-        shutil.copytree(file, unpack_folder, dirs_exist_ok=True)
-
-output = ""
-for build_type in build_types_created:
-    build_type_dir = os.path.join(args.path_to_directory, build_types[build_type])
     if not os.path.exists(build_type_dir):
-        print(f"Unexpected error: {build_type_dir} does not exist, even though it was in the set.")
+        print(f"Unexpected error: path {build_type_dir} does not exist, even though it was in the set.")
-        continue
+        raise FileNotFoundError
     # loop over the folder in the build_type_dir
-    for dir in dirs:
+    for platform_folder in build_type["os_folders"]:
-        print(f"Cleaning up {dir}")
+        print(f"Cleaning up {platform_folder}")
         # Traverse the directory tree and move all of the files to the root directory
-        flatten_tree(os.path.join(build_type_dir, dir))
+        flatten_tree(os.path.join(build_type_dir, platform_folder))
     # Now move the symbols files to the symbols folder
-    # prep the symbols folder
+    # Define the folder for symbols
     symbols_folder = os.path.join(build_type_dir, "symbols")
     os.mkdir(symbols_folder)
-    symbol_archives = glob.glob(f"{build_type_dir}/**/*_hvk*", recursive=True)
+    # prep the symbols folder
-    for sym_file in symbol_archives:
+    symbol_patterns = ["*_hvk*", "*_oss*"]
-        print(f"Moving {sym_file} to {symbols_folder}")
-        shutil.move(sym_file, symbols_folder)
-    symbol_archives = glob.glob(f"{build_type_dir}/**/*_oss*", recursive=True)
-    for sym_file in symbol_archives:
-        print(f"Moving {sym_file} to {symbols_folder}")
-        shutil.move(sym_file, symbols_folder)
 
+    # Loop through each pattern, find matching files, and move them
+    for pattern in symbol_patterns:
+        symbol_archives = glob.glob(f"{build_type_dir}/**/{pattern}", recursive=True)
+        for sym_file in symbol_archives:
+            print(f"Moving {sym_file} to {symbols_folder}")
+            shutil.move(sym_file, symbols_folder)
+
+def gather_build_info(build_type, config):
     # While we're at it, let's print the md5 listing 
-    file_dict = {}
+    download_root = f"{config.download_root}/{build_type["build_type_folder"]}"
-    md5_dict = {}
+    # for each os that we have built for 
-    platforms_printable = {"windows":"MS Windows", "mac":"MacOS", "linux":"Linux"}
+    build_type_dir = build_type["build_type_fullpath"]
-    grids_printable = {"SL":"Second Life", "OS":"OpenSim"}
+    for platform_folder in build_type["os_folders"]:
-
+        print(f"Getting files for {platform_folder} in {build_type_dir}")
-    download_root = f"https://downloads.firestormviewer.org/{build_types[build_type]}"
+        build_type_platform_folder = os.path.join(build_type_dir, platform_folder)
-    output += f'''
+        files = get_files(build_type_platform_folder)
-DOWNLOADS - {build_type}
--------------------------------------------------------------------------------------------------------
-'''
-    for dir in dirs:
-        print(f"Getting files for {dir} in {build_type_dir}")
-        files = get_files(os.path.join(build_type_dir, dir))
         try:
             for file in files:
-                full_file = os.path.join(build_type_dir, dir, file)
+                full_file = os.path.join(build_type_platform_folder, file)
-                md5 = get_md5(full_file)
                 base_name = os.path.basename(file)
-                wordsize = "64"
+                file_URI = f"{download_root}/{platform_folder}/{base_name}"
+                md5 = get_md5(full_file)
                 
                 if "FirestormOS-" in base_name:
                     grid = "OS"
                 else:
                     grid = "SL"
 
-                if dir in dirs:
+                file_key = f"{grid}-{platform_folder}"
-                    file_dict[f"{grid}{dir}{wordsize}"] = full_file
+
-                    md5_dict[f"{grid}{dir}{wordsize}"] = md5
+                # if platform_folder in config.os_download_dirs:
+                if "downloadable_artifacts" not in build_type:
+                    build_type["downloadable_artifacts"] = {}
+
+                build_type["downloadable_artifacts"] = { f"{file_key}":{
+                    "file_path": full_file,         
+                    "file_download_URI": file_URI,
+                    "grid": grid,
+                    "fs_ver_mgr_platform": config.fs_version_mgr_platform.get(platform_folder),
+                    "md5": md5,
+                }}
         except TypeError:
-            print(f"No files found for {dir} in {build_type_dir}")
+            print(f"Error processing files for {platform_folder} in {build_type_dir}")
+            continue
+        except Exception as e:
+            print(f"An error occurred while processing files for {platform_folder} in {build_type_dir}: {e}")
+            continue
+    return build_type
 
-
+def create_discord_message(build_info, config):
-
+# Start with a header line            
-        output += f'''
+    text_summary = f'''
-{platforms_printable[dir]}
+DOWNLOADS - {build_info["build_type"]}
+-------------------------------------------------------------------------------------------------------
 '''
-        dir = dir.lower()
+# for each platform we potentailly build for 
-        wordsize = "64"
+# Append platform label in printable form
-        platform = f"{platforms_printable[dir]}"
+    for platform_folder in config.supported_os_dirs:
+        platform_printable = config.platforms_printable[platform_folder]
+        text_summary += f'''
+{platform_printable}
+'''
+        platform_folder = platform_folder.lower()
         for grid in ["SL", "OS"]:
-            grid_printable = f"{grids_printable[grid]}"
+            grid_printable = f"{config.grids_printable[grid]}"
             try:
-                output += f"{platform} for {grid_printable} ({wordsize}-bit)\n"
+                file_key = f"{grid}-{platform_folder}"
-                output += f"{download_root}/{dir}/{os.path.basename(file_dict[f'{grid}{dir}{wordsize}'])}\n"
+                text_summary += f"{platform_printable} for {grid_printable}\n"
-                output += "\n"
+                text_summary += f"{build_info["downloadable_artifacts"][file_key]["file_download_URI"]}\n"
-                output += f"MD5: {md5_dict[f'{grid}{dir}{wordsize}']}\n"
+                text_summary += "\n"
-                output += "\n"
+                text_summary += f"MD5: {build_info["downloadable_artifacts"][file_key]["md5"]}\n"
+                text_summary += "\n"
             except KeyError:
-                output += f"{platform} for {grid_printable} ({wordsize}-bit) - NOT AVAILABLE\n"
+                text_summary += f"{platform_printable} for {grid_printable} - NOT AVAILABLE\n"
-                output += "\n"
+                text_summary += "\n"
-        output += '''-------------------------------------------------------------------------------------------------------
+        text_summary += '''
+-------------------------------------------------------------------------------------------------------
 '''
+    return text_summary
 
-    if args.webhook:
+def update_fs_version_mgr(build_info, config):
-        # Add the message to the webhook
+    # Read the secret key from environment variables
-        webhook.set_content(content=output)
+    secret_key = os.environ.get('FS_VERSION_MGR_KEY')
-        # Send the webhook
+    if not secret_key:
-        response = webhook.execute()
+        print("Error: FS_VERSION_MGR_KEY not set")
-        # Print the response
+        sys.exit(1)
-        if not response.ok:
-            print(f"Webhook Error {response.status_code}: {response.text}")        
-    print(output)
 
+    secret_for_api = generate_secret(secret_key)  
+    build_type = build_info["build_type"]
+    version = os.environ.get('FS_VIEWER_VERSION')
+    channel = os.environ.get('FS_VIEWER_CHANNEL')
+    build_number = os.environ.get('FS_VIEWER_BUILD')
+
+    build_variant = "regular"
+    for file_key in build_info["downloadable_artifacts"]:
+        try:
+            download_link = build_info["downloadable_artifacts"][file_key]["file_download_URI"]
+            md5_checksum = build_info["downloadable_artifacts"][file_key]["md5"]
+            grid = build_info["downloadable_artifacts"][file_key]["grid"].lower()
+            os_name = build_info["downloadable_artifacts"][file_key]["fs_ver_mgr_platform"]
+        except KeyError:
+            print(f"Error: Could not find downloadable artifacts for {file_key}")
+            continue
+
+        payload = {
+            "secret": secret_for_api,
+            "viewer_channel": channel,
+            "grid_type": grid,
+            "operating_system": os_name,
+            "build_type": build_type.lower(),
+            "viewer_version": version,
+            "build_number": int(build_number),
+            "download_link": download_link,
+            "md5_checksum": md5_checksum
+        }
+
+        # Make the API call
+        url = "https://www.firestormviewer.org/set-fs-vrsns-jsn/"
+        headers = {"Content-Type": "application/json"}
+
+        try:
+            response = requests.post(url, json=payload, headers=headers)
+            response.raise_for_status()
+            response_data = response.json()
+            result = response_data.get('result')
+            message = response_data.get('message')
+            if result == 'success':
+                print(f"Version manager updated successfully for {os_name} {build_variant}")
+            else:
+                print(f"Error updating version manager: {message}")
+        except requests.exceptions.RequestException as e:
+            print(f"API request failed: {e}")
+        except ValueError:
+            print("API response is not valid JSON")
+
+# parse args first arg optional -r (release) second arg mandatory string path_to_directory
+def main():
+    try:
+        # Initialise the build configuration
+        config = BuildConfig()
+
+        parser = argparse.ArgumentParser(
+            prog="print_download_list",
+            description="Prints the list of files for download and their md5 checksums"
+            )
+        parser.add_argument("-w", "--webhook", help="post details to the webhook")
+
+        # add path_to_directory required parameter to parser
+        parser.add_argument("path_to_directory", help="path to the directory in which we'll look for the files")
+
+        args = parser.parse_args()
+
+        # Create a webhook object with the webhook URL
+        if args.webhook:
+            webhook = DiscordWebhook(url=args.webhook)
+
+        # unzip the github artifact for this OS (`dir`) into the folder `dir`
+        # get the .zip files in args.path_to_directory using glob 
+        print(f"Processing artifacts in {args.path_to_directory}")
+        build_types_created = unpack_artifacts(args.path_to_directory, config)
+
+        for build_type in build_types_created:
+            restructure_folders(build_type, config)
+            build_info = gather_build_info(build_type, config)
+            update_fs_version_mgr(build_info, config)
+
+            discord_text = create_discord_message(build_info, config)
+            if args.webhook:
+                # Add the message to the webhook
+                webhook.set_content(content=discord_text)
+                # Send the webhook
+                response = webhook.execute()
+                # Print the response
+                if not response.ok:
+                    print(f"Webhook Error {response.status_code}: {response.text}")        
+            print(discord_text)
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        sys.exit(1)        
+
+if __name__ == '__main__':
+    import sys
+    main()