From d64f98b8e3dabbc08c4d29c2ab7ad59233bb7b71 Mon Sep 17 00:00:00 2001 From: Beq Date: Sun, 29 Sep 2024 15:34:05 +0100 Subject: [PATCH] More adjustments to handle expanded build matrix --- .github/workflows/build_viewer.yml | 14 +++---- .github/workflows/deploy_only.yml | 13 +++---- fsutils/build_config.json | 10 ++--- fsutils/build_config.py | 2 +- fsutils/download_list.py | 60 +++++++++++++++--------------- 5 files changed, 49 insertions(+), 50 deletions(-) diff --git a/.github/workflows/build_viewer.yml b/.github/workflows/build_viewer.yml index ac45182703..f6baff2a46 100644 --- a/.github/workflows/build_viewer.yml +++ b/.github/workflows/build_viewer.yml @@ -400,9 +400,16 @@ jobs: sparse-checkout-cone-mode: false ref: ${{ github.head_ref || github.ref_name || 'master' }} fetch-depth: 1 + - name: Install discord-webhook library run: pip install discord-webhook + - name: Download artifacts + uses: actions/download-artifact@v4 + id: download + with: + path: to_deploy + - name: find channel and webhook from build_matrix outputs run: | viewer_release_type=${{ needs.build_matrix.outputs.viewer_release_type }} @@ -427,18 +434,12 @@ jobs: echo "FS_RELEASE_FOLDER=${FS_RELEASE_FOLDER}" >> $GITHUB_ENV echo "FS_BUILD_WEBHOOK_URL=${FS_BUILD_WEBHOOK_URL}" >> $GITHUB_ENV - - name: Download artifacts - uses: actions/download-artifact@v4 - id: download - with: - path: to_deploy - name: List artifacts download run: ls -R working-directory: ${{steps.download.outputs.download-path}} - name: Create Build Info artifact env: - FS_VIEWER_CHANNEL: ${{ needs.build_matrix.outputs.viewer_channel }} FS_VIEWER_VERSION: FS_VIEWER_BUILD: FS_VIEWER_RELEASE_TYPE: @@ -463,7 +464,6 @@ jobs: - name: Reorganise artifacts ready for server upload. env: - FS_VIEWER_CHANNEL: ${{ needs.build_matrix.outputs.viewer_channel }} FS_VIEWER_VERSION: ${{ needs.build_matrix.outputs.viewer_version }} FS_VIEWER_BUILD: ${{ needs.build_matrix.outputs.viewer_build }} FS_VIEWER_RELEASE_TYPE: ${{ needs.build_matrix.outputs.viewer_release_type }} diff --git a/.github/workflows/deploy_only.yml b/.github/workflows/deploy_only.yml index 1374e51c89..a74cb42cd1 100644 --- a/.github/workflows/deploy_only.yml +++ b/.github/workflows/deploy_only.yml @@ -6,10 +6,6 @@ on: build_run_number: description: 'GitHub Run Number (per build_viewer.yml workflow)' required: true - viewer_channel: - description: 'viewer_channel' - required: true - default: 'Releasex64' viewer_version: description: 'viewer version not including build' required: true @@ -44,15 +40,17 @@ jobs: sparse-checkout-cone-mode: false ref: ${{ github.head_ref || github.ref_name || 'master' }} fetch-depth: 1 + + - name: Install discord-webhook library + run: pip install discord-webhook + - name: Download Build Artifacts uses: dawidd6/action-download-artifact@v6 id: download with: workflow: build_viewer.yml run_number: ${{ github.event.inputs.build_run_number }} - path: to_deploy - - name: Install discord-webhook library - run: pip install discord-webhook + path: to_deploy - name: find channel and webhook from build_matrix outputs run: | @@ -84,7 +82,6 @@ jobs: - name: Reorganise artifacts ready for server upload. env: - FS_VIEWER_CHANNEL: ${{ github.event.inputs.viewer_channel }} FS_VIEWER_VERSION: ${{ github.event.inputs.viewer_version }} FS_VIEWER_BUILD: ${{ github.event.inputs.viewer_build }} FS_VIEWER_RELEASE_TYPE: ${{ github.event.inputs.viewer_release_type }} diff --git a/fsutils/build_config.json b/fsutils/build_config.json index 9ca79a8ffe..31fffd248c 100644 --- a/fsutils/build_config.json +++ b/fsutils/build_config.json @@ -35,10 +35,10 @@ "Alpha": "alpha", "Nightly": "nightly" }, - "build_type_mapping": { - "regular": "regular", - "avx": "avx", - "tracy": "tracy", - "arm": "arm" + "variant_printable": { + "regular": "Older CPUs", + "avx": "Modern CPUs (post 2013)", + "tracy": "Tracy profiler", + "arm": "Arm" } } diff --git a/fsutils/build_config.py b/fsutils/build_config.py index c35f89099d..dc8cb5ec0f 100644 --- a/fsutils/build_config.py +++ b/fsutils/build_config.py @@ -16,4 +16,4 @@ class BuildConfig: self.grids_printable = config_data.get('grids_printable', {}) self.download_root = config_data.get('download_root', '') self.viewer_channel_mapping = config_data.get('viewer_channel_mapping', {}) - self.build_type_mapping = config_data.get('build_type_mapping', {}) + self.variant_printable = config_data.get('variant_printable', {}) diff --git a/fsutils/download_list.py b/fsutils/download_list.py index a166145293..5fd3e7180d 100644 --- a/fsutils/download_list.py +++ b/fsutils/download_list.py @@ -286,14 +286,14 @@ def unpack_artifacts(path_to_artifacts_directory, config): print(f"Finished processing artifacts for build_type {build_type}") return build_types_found -def restructure_folders(build_type, config): - print(f"Restructuring folders for build_type {build_type}") - build_type_dir = build_type["build_type_fullpath"] +def restructure_folders(build_type_info, config): + print(f"Restructuring folders for build_type {build_type_info['build_type']}") + build_type_dir = build_type_info["build_type_fullpath"] if not os.path.exists(build_type_dir): print(f"Unexpected error: path {build_type_dir} does not exist, even though it was in the set.") raise FileNotFoundError # loop over the folder in the build_type_dir - for platform_folder in build_type["os_folders"]: + for platform_folder in build_type_info["os_folders"]: print(f"Cleaning up {platform_folder}") # Traverse the directory tree and move all of the files to the root directory flatten_tree(os.path.join(build_type_dir, platform_folder)) @@ -311,13 +311,13 @@ def restructure_folders(build_type, config): print(f"Moving {sym_file} to {symbols_folder}") shutil.move(sym_file, symbols_folder) -def gather_build_info(build_type, config): - print(f"Gathering build info for build_type {build_type}") +def gather_build_info(build_type_info, config): + print(f"Gathering build info for build_type {build_type_info}") # While we're at it, let's print the md5 listing - download_root = f"{config.download_root}/{build_type['build_type_folder']}" + download_root = f"{config.download_root}/{build_type_info['build_type_folder']}" # for each os that we have built for - build_type_dir = build_type["build_type_fullpath"] - for platform_folder in build_type["os_folders"]: + build_type_dir = build_type_info["build_type_fullpath"] + for platform_folder in build_type_info["os_folders"]: print(f"Getting files for {platform_folder} in {build_type_dir}") build_type_platform_folder = os.path.join(build_type_dir, platform_folder) files = get_files(build_type_platform_folder) @@ -341,10 +341,10 @@ def gather_build_info(build_type, config): file_key = f"{grid}-{platform_folder}" # if platform_folder in config.os_download_dirs: - if "downloadable_artifacts" not in build_type: - build_type["downloadable_artifacts"] = {} + if "downloadable_artifacts" not in build_type_info: + build_type_info["downloadable_artifacts"] = {} - build_type["downloadable_artifacts"][f"{file_key}"] = { + build_type_info["downloadable_artifacts"][f"{file_key}"] = { "file_path": full_file, "file_download_URI": file_URI, "grid": grid, @@ -359,8 +359,8 @@ def gather_build_info(build_type, config): except Exception as e: print(f"An error occurred while processing files for {platform_folder} in {build_type_dir}: {e}") continue - print(f"Created build info: {build_type}") - return build_type + print(f"Created build info: {build_type_info}") + return build_type_info def create_discord_message(build_info, config): # Start with a header line @@ -378,16 +378,19 @@ DOWNLOADS - {build_info["build_type"]} platform_folder = platform_folder.lower() for grid in ["SL", "OS"]: grid_printable = f"{config.grids_printable[grid]}" - try: - file_key = f"{grid}-{platform_folder}" - text_summary += f"{platform_printable} for {grid_printable}\n" - text_summary += f"{build_info['downloadable_artifacts'][file_key]['file_download_URI']}\n" - text_summary += "\n" - text_summary += f"MD5: {build_info['downloadable_artifacts'][file_key]['md5']}\n" - text_summary += "\n" - except KeyError: - text_summary += f"{platform_printable} for {grid_printable} - NOT AVAILABLE\n" - text_summary += "\n" + for variant in ["avx", "regular"]: + variant_printable = f"{config.variants_printable[variant]}" + text_summary += f"{variant_printable}\n" + try: + file_key = f"{grid}-{platform_folder}" + text_summary += f"{platform_printable} for {grid_printable}\n" + text_summary += f"{build_info['downloadable_artifacts'][file_key]['file_download_URI']}\n" + text_summary += "\n" + text_summary += f"MD5: {build_info['downloadable_artifacts'][file_key]['md5']}\n" + text_summary += "\n" + except KeyError: + text_summary += f"{platform_printable} for {grid_printable} - NOT AVAILABLE\n" + text_summary += "\n" text_summary += ''' ------------------------------------------------------------------------------------------------------- ''' @@ -404,7 +407,6 @@ def update_fs_version_mgr(build_info, config): secret_for_api = generate_secret(secret_key) build_type = build_info["build_type"].lower() version = os.environ.get('FS_VIEWER_VERSION') - channel = os.environ.get('FS_VIEWER_CHANNEL') build_number = os.environ.get('FS_VIEWER_BUILD') for file_key in build_info["downloadable_artifacts"]: @@ -422,7 +424,7 @@ def update_fs_version_mgr(build_info, config): "viewer_channel": build_type, "grid_type": grid, "operating_system": os_name, - "build_type": build_variant, + "build_type": build_variant, # on the webservice variant is known as build_type. "viewer_version": version, "build_number": int(build_number), "download_link": download_link, @@ -492,10 +494,10 @@ def main(): print(f"Processing artifacts in {args.path_to_directory}") build_types_created = unpack_artifacts(args.path_to_directory, config) print(f"buuild types created: {build_types_created}") - for build_type_key, build_type in build_types_created.items(): + for build_type_key, build_type_info in build_types_created.items(): print(f"Processing {build_type_key}") - restructure_folders(build_type, config) - build_info = gather_build_info(build_type, config) + restructure_folders(build_type_info, config) + build_info = gather_build_info(build_type_info, config) update_fs_version_mgr(build_info, config) discord_text = create_discord_message(build_info, config)