Updated build workflow for avx2

Significant changes include:
- Added a new 'variant' matrix in the GitHub Actions workflow, replacing the previous 'addrsize' matrix.
- Introduced support for AVX2 builds with additional flags.
- Modified artifact naming to include the variant type.
- Updated tag-fs-build.yml to reflect changes in tag creation logic.
- Overhauled extract_vars_from_zipfile_name function in download_list.py to handle new filename format and provide more detailed extraction of variables from zip file names.
- Included variant information in gathered build info and updated fs_version_mgr accordingly.
master
Beq 2024-09-29 00:04:41 +01:00
parent 30aad977b2
commit 0c8dc7d1f8
3 changed files with 87 additions and 25 deletions

View File

@ -21,6 +21,7 @@ env:
XZ_DEFAULTS: -T0 XZ_DEFAULTS: -T0
FS_RELEASE_TYPE: Unknown FS_RELEASE_TYPE: Unknown
platform: Unknown platform: Unknown
addrsize: 64
fallback_platform: ${platform} fallback_platform: ${platform}
FS_RELEASE_CHAN: ${FS_RELEASE_TYPE}x64 FS_RELEASE_CHAN: ${FS_RELEASE_TYPE}x64
FS_GRID: "GRID FLAGS NOT SET" FS_GRID: "GRID FLAGS NOT SET"
@ -32,11 +33,12 @@ jobs:
matrix: matrix:
os: [macos-12,ubuntu-22.04,windows-2022] os: [macos-12,ubuntu-22.04,windows-2022]
grid: [sl,os] grid: [sl,os]
addrsize: [64] variant: [regular, avx]
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
outputs: outputs:
viewer_channel: ${{ steps.channel.outputs.viewer_channel }} viewer_channel: ${{ steps.channel.outputs.viewer_channel }}
viewer_version: ${{ steps.version.outputs.viewer_version }} viewer_version: ${{ steps.version.outputs.viewer_version }}
viewer_variant: ${{ matrix.variant }}
viewer_build: ${{ steps.version.outputs.viewer_build }} viewer_build: ${{ steps.version.outputs.viewer_build }}
viewer_release_type: ${{ steps.version.outputs.viewer_release_type }} viewer_release_type: ${{ steps.version.outputs.viewer_release_type }}
steps: steps:
@ -129,7 +131,7 @@ jobs:
FS_RELEASE_TYPE=Profiling FS_RELEASE_TYPE=Profiling
fi fi
fi fi
if [[ "${{ matrix.addrsize }}" == "64" ]]; then if [[ "${{ matrix.variant }}" == "avx" ]]; then
FS_RELEASE_CHAN="${FS_RELEASE_TYPE}x64" FS_RELEASE_CHAN="${FS_RELEASE_TYPE}x64"
else else
FS_RELEASE_CHAN=${FS_RELEASE_TYPE} FS_RELEASE_CHAN=${FS_RELEASE_TYPE}
@ -157,10 +159,7 @@ jobs:
os_map=( ["Windows"]="windows" ["Linux"]="linux" ["macOS"]="darwin" ) os_map=( ["Windows"]="windows" ["Linux"]="linux" ["macOS"]="darwin" )
platform="${os_map[${{ runner.os}}]}" platform="${os_map[${{ runner.os}}]}"
echo "fallback_platform=${platform}" >> $GITHUB_ENV echo "fallback_platform=${platform}" >> $GITHUB_ENV
if [ ${{ matrix.addrsize }} -ne 32 ]; platform+=${ addrsize }
then
platform+=${{ matrix.addrsize }}
fi
echo "platform=${platform}" >> $GITHUB_ENV echo "platform=${platform}" >> $GITHUB_ENV
shell: bash shell: bash
@ -290,18 +289,22 @@ jobs:
if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.include_tracy == 'true' }} if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.include_tracy == 'true' }}
shell: bash shell: bash
run: echo "EXTRA_ARGS=${{ env.EXTRA_ARGS }} --tracy" >> $GITHUB_ENV run: echo "EXTRA_ARGS=${{ env.EXTRA_ARGS }} --tracy" >> $GITHUB_ENV
- name: Add flags for AVX2 builds
if: ${{ matrix.variant == 'avx' }}
shell: bash
run: echo "EXTRA_ARGS=${{ env.EXTRA_ARGS }} --avx2" >> $GITHUB_ENV
- name: Clean up packages to give more space - name: Clean up packages to give more space
run: rm *${{ env.fallback_platform }}*bz2 run: rm *${{ env.fallback_platform }}*bz2
shell: bash shell: bash
- name: Configure - name: Configure
run: | run: |
autobuild configure -c ReleaseFS -A${{matrix.addrsize}} -- --package --chan ${{env.FS_RELEASE_CHAN}} ${{env.EXTRA_ARGS}} ${{env.FS_GRID}} autobuild configure -c ReleaseFS -A${ addrsize } -- --package --chan ${{env.FS_RELEASE_CHAN}} ${{env.EXTRA_ARGS}} ${{env.FS_GRID}}
echo "BUGSPLAT_DB=$BUGSPLAT_DB" >> $GITHUB_ENV echo "BUGSPLAT_DB=$BUGSPLAT_DB" >> $GITHUB_ENV
shell: bash shell: bash
- name: build - name: build
id: build id: build
run: autobuild build -c ReleaseFS -A${{matrix.addrsize}} --no-configure run: autobuild build -c ReleaseFS -A${ addrsize } --no-configure
shell: bash shell: bash
- name: Extract version number - name: Extract version number
id: version id: version
@ -357,7 +360,7 @@ jobs:
if: runner.os == 'Windows' if: runner.os == 'Windows'
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ env.FS_RELEASE_TYPE }}-${{ matrix.os }}-${{ matrix.addrsize }}-${{ matrix.grid }}-artifacts.zip name: ${{ env.FS_RELEASE_TYPE }}-${{ matrix.os }}-${{ matrix.variant }}-${{ matrix.grid }}-artifacts.zip
path: | path: |
build-*/newview/Release/*Setup.exe build-*/newview/Release/*Setup.exe
build-*/newview/Release/*.xz build-*/newview/Release/*.xz
@ -366,7 +369,7 @@ jobs:
if: runner.os == 'Linux' if: runner.os == 'Linux'
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ env.FS_RELEASE_TYPE }}-${{ matrix.os }}-${{matrix.addrsize}}-${{matrix.grid}}-artifacts.zip name: ${{ env.FS_RELEASE_TYPE }}-${{ matrix.os }}-${{ matrix.variant }}-${{matrix.grid}}-artifacts.zip
path: | path: |
build-linux-*/newview/*.xz build-linux-*/newview/*.xz
build-linux-*/newview/*.bz2 build-linux-*/newview/*.bz2
@ -375,7 +378,7 @@ jobs:
if: runner.os == 'macOS' if: runner.os == 'macOS'
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ env.FS_RELEASE_TYPE }}-${{ matrix.os }}-${{matrix.addrsize}}-${{matrix.grid}}-artifacts.zip name: ${{ env.FS_RELEASE_TYPE }}-${{ matrix.os }}-${{ matrix.variant }}-${{matrix.grid}}-artifacts.zip
path: | path: |
build-darwin-*/newview/*.dmg build-darwin-*/newview/*.dmg
build-darwin-*/newview/*.bz2 build-darwin-*/newview/*.bz2
@ -451,7 +454,7 @@ jobs:
EOF EOF
echo "Build info created: $(cat build_info.json)" echo "Build info created: $(cat build_info.json)"
# Upload Build Info Artifact # Upload Build Info Artifact (note that this file is expected to be identical for each matrix run, so items like OS and VARIANT cannot be in this file.)
- name: Upload Tag Info - name: Upload Tag Info
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:

View File

@ -155,7 +155,7 @@ jobs:
- name: Tagging Confirmation - name: Tagging Confirmation
run: | run: |
if [ "${{ steps.get_inputs.outputs.dry_run }}" == "true" ]; then if [ "${{ steps.get_inputs.outputs.dry_run }}" == "true" ]; then
echo "Dry run mode enabled. Tag '$TAG_NAME' was not created or pushed." echo "Dry run mode enabled. Tag '${{ steps.check_tag.outputs.tag_exists }}' was not created or pushed."
elif [ "${{ steps.check_tag.outputs.tag_exists }}" == "true" ]; then elif [ "${{ steps.check_tag.outputs.tag_exists }}" == "true" ]; then
echo "Tag '${{ steps.get_tag.outputs.tag_name }}' already exists. No new tag was created." echo "Tag '${{ steps.get_tag.outputs.tag_name }}' already exists. No new tag was created."
else else

View File

@ -11,6 +11,8 @@ import pytz
from datetime import datetime from datetime import datetime
import requests import requests
from discord_webhook import DiscordWebhook from discord_webhook import DiscordWebhook
from pathlib import Path
from typing import Tuple
from build_config import BuildConfig from build_config import BuildConfig
@ -162,24 +164,75 @@ def get_supported_os(os_name, config):
# throws for unexpected os_name # throws for unexpected os_name
return config.os_hosted_folder.get(os_name) return config.os_hosted_folder.get(os_name)
def extract_vars_from_zipfile_name(file): def extract_vars_from_zipfile_name(file_path: str) -> Tuple[str, str, str, str, str, str]:
# File is an artifact file sometihng like Nightly-windows-2022-64-sl-artifacts.zip """
# print(f"unzipping {file}") Extract variables from an artifact zip file name.
#extract first word (delimited by '-' from the file name)
# build_type is a fullpath but we only want the last folder, remove the leading part of the path leaving just the foldername using basename
filename = os.path.basename(file)
build_type = filename.split("-")[0]
platform = filename.split("-")[1].lower()
return filename,build_type, platform
The expected filename format is:
{build_Type}-{platform}-{platform_ver}-{variant}-{grid}-artifacts.zip
Args:
file_path (str): The full path to the zip file.
Returns:
Tuple containing:
filename (str): The basename of the file.
build_type (str)
platform (str)
platform_ver (str)
variant (str)
grid (str)
Raises:
ValueError: If the filename does not conform to the expected format.
"""
# Use pathlib for path manipulations
path = Path(file_path)
filename = path.name
# Remove the .zip extension
name_without_ext = path.stem
# Expected suffix after the main parts
expected_suffix = "artifacts"
parts = name_without_ext.split('-')
if len(parts) < 6:
raise ValueError(f"Filename '{filename}' does not have enough parts separated by '-'.")
# Unpack the parts
build_type, platform, platform_ver, variant, grid, suffix = parts[:6]
if suffix.lower() != expected_suffix:
raise ValueError(f"Filename '{filename}' does not end with '-{expected_suffix}'.")
# Return the extracted variables, applying lower() where needed
return (
filename,
build_type,
platform.lower(),
platform_ver.lower(),
variant.lower(),
grid.lower()
)
def unpack_artifacts(path_to_artifacts_directory, config): def unpack_artifacts(path_to_artifacts_directory, config):
build_types_found = {} build_types_found = {}
zips = glob.glob(f"{path_to_artifacts_directory}/*.zip") zips = glob.glob(f"{path_to_artifacts_directory}/*.zip")
for file in zips: for file in zips:
print(f"Processing zip file {file}") print(f"Processing zip file {file}")
filename, build_type, platform = extract_vars_from_zipfile_name(file) try:
print(f"Identified filename {filename}, build_type {build_type} and platform {platform} from file {file}") filename, build_type, platform, platform_ver, variant, grid = extract_vars_from_zipfile_name(file)
# print(f"Filename: {filename}")
# print(f"Build Type: {build_type}")
# print(f"Platform: {platform}")
# print(f"Platform Version: {platform_ver}")
# print(f"Variant: {variant}")
# print(f"Grid: {grid}")
except ValueError as e:
print(f"Error extracting vars from zipfile name: {e}")
continue
print(f"Identified filename {filename} with build_type {build_type} and platform {platform}({platform_ver}) grid {grid} and variant {variant} from file {file}")
if is_supported_build_type( build_type, config) == False: if is_supported_build_type( build_type, config) == False:
print(f"Invalid build_type {build_type} from file {file} using 'Unknown' instead") print(f"Invalid build_type {build_type} from file {file} using 'Unknown' instead")
build_type = "Unknown" build_type = "Unknown"
@ -280,6 +333,11 @@ def gather_build_info(build_type, config):
else: else:
grid = "SL" grid = "SL"
if "x64" in base_name:
variant = "avx"
else:
variant = "regular"
file_key = f"{grid}-{platform_folder}" file_key = f"{grid}-{platform_folder}"
# if platform_folder in config.os_download_dirs: # if platform_folder in config.os_download_dirs:
@ -290,6 +348,7 @@ def gather_build_info(build_type, config):
"file_path": full_file, "file_path": full_file,
"file_download_URI": file_URI, "file_download_URI": file_URI,
"grid": grid, "grid": grid,
"variant": variant,
"fs_ver_mgr_platform": config.fs_version_mgr_platform.get(platform_folder), "fs_ver_mgr_platform": config.fs_version_mgr_platform.get(platform_folder),
"md5": md5, "md5": md5,
} }
@ -348,7 +407,7 @@ def update_fs_version_mgr(build_info, config):
channel = os.environ.get('FS_VIEWER_CHANNEL') channel = os.environ.get('FS_VIEWER_CHANNEL')
build_number = os.environ.get('FS_VIEWER_BUILD') build_number = os.environ.get('FS_VIEWER_BUILD')
build_variant = "regular" build_variant = build_info["variant"]
for file_key in build_info["downloadable_artifacts"]: for file_key in build_info["downloadable_artifacts"]:
try: try:
download_link = build_info["downloadable_artifacts"][file_key]["file_download_URI"] download_link = build_info["downloadable_artifacts"][file_key]["file_download_URI"]