diff --git a/fsutils/translation_strings_auditing_tool.py b/fsutils/translation_strings_auditing_tool.py index effac01ba5..00ae3c4634 100644 --- a/fsutils/translation_strings_auditing_tool.py +++ b/fsutils/translation_strings_auditing_tool.py @@ -4,11 +4,11 @@ import html import re import xml.etree.ElementTree as ET from dataclasses import dataclass, field +from datetime import date from pathlib import Path REPO = Path(__file__).resolve().parent.parent -EN_DIR = REPO / "indra/newview/skins/default/xui/en" -PL_DIR = REPO / "indra/newview/skins/default/xui/pl" +SKINS_ROOT = REPO / "indra/newview/skins" OUT_FILE = REPO / "_audit_result.html" TRANSLATABLE_ATTRS = frozenset({ @@ -44,6 +44,38 @@ BRAND_STRINGS = frozenset({ "CAPITALIZED_APP_NAME", "CURRENT_GRID", "SECOND_LIFE", "DOWNLOAD_URL", }) SKIP_FILES = frozenset({"xui_version.xml"}) +TRANSLATE_FALSE_VALUES = frozenset({"false", "0", "no"}) + +# Common misspellings of real XUI translation attributes. +ATTR_TYPO_HINTS: dict[str, str] = { + "tooltip": "tool_tip", + "toolTip": "tool_tip", + "tool-tip": "tool_tip", + "ToolTip": "tool_tip", + "Tooltip": "tool_tip", + "helptext": "help_text", + "helpText": "help_text", + "shortdesc": "short_desc", + "longdesc": "long_desc", + "shortDesc": "short_desc", + "longDesc": "long_desc", + "yes_text": "yestext", + "no_text": "notext", + "cancel_text": "canceltext", +} + +TRANSLATION_ATTR_KEYWORDS = ( + "tip", "label", "title", "text", "message", "desc", "help", +) + +ATTR_EN_ALTERNATES: dict[str, tuple[str, ...]] = { + "label": ("value", "title"), + "title": ("label", "value"), + "value": ("label",), + "text": ("label", "value"), + "tool_tip": ("help_text",), + "help_text": ("tool_tip",), +} @dataclass @@ -53,11 +85,70 @@ class Entry: kind: str +@dataclass +class AttrIssue: + element: str + attr: str + value: str + hint: str = "" + + @dataclass class FileReport: rel: str missing: list[Entry] = field(default_factory=list) redundant: list[Entry] = field(default_factory=list) + extra_attrs: list[AttrIssue] = field(default_factory=list) + + +@dataclass +class LocaleReport: + locale: str + en_count: int + locale_count: int + missing_files: list[str] = field(default_factory=list) + redundant_files: list[str] = field(default_factory=list) + file_reports: list[FileReport] = field(default_factory=list) + + @property + def total_missing(self) -> int: + return sum(len(r.missing) for r in self.file_reports) + + @property + def total_redundant(self) -> int: + return sum(len(r.redundant) for r in self.file_reports) + + @property + def total_extra_attrs(self) -> int: + return sum(len(r.extra_attrs) for r in self.file_reports) + + @property + def files_with_missing(self) -> int: + return sum(1 for r in self.file_reports if r.missing) + + @property + def files_with_redundant(self) -> int: + return sum(1 for r in self.file_reports if r.redundant) + + @property + def files_with_extra_attrs(self) -> int: + return sum(1 for r in self.file_reports if r.extra_attrs) + + @property + def has_issues(self) -> bool: + return bool( + self.missing_files or self.redundant_files or self.file_reports, + ) + + +@dataclass +class SkinReport: + skin: str + locale_reports: list[LocaleReport] = field(default_factory=list) + + @property + def en_count(self) -> int: + return self.locale_reports[0].en_count if self.locale_reports else 0 def normalise_text(text: str) -> str: @@ -119,7 +210,7 @@ def is_trivial(value: str, key: str = "") -> bool: return False -def needs_pl_entry(value: str, key: str) -> bool: +def needs_translation_entry(value: str, key: str) -> bool: if is_trivial(value, key): return False if key.startswith("string:") and value == key[7:]: @@ -131,6 +222,45 @@ def needs_pl_entry(value: str, key: str) -> bool: return True +def is_translate_disabled(elem: ET.Element) -> bool: + return elem.attrib.get("translate", "").strip().lower() in TRANSLATE_FALSE_VALUES + + +def collect_no_translate_targets(root: ET.Element, rel: str) -> frozenset[str]: + targets: set[str] = set() + + if rel == "strings.xml": + for child in root: + tag = child.tag.split("}")[-1] + if tag != "string": + continue + if not is_translate_disabled(child): + continue + name = child.attrib.get("name", "").strip() + if name: + targets.add(f"string:{name}") + return frozenset(targets) + + def walk(elem: ET.Element) -> None: + ekey = element_map_key(elem) + if ekey and is_translate_disabled(elem): + targets.add(ekey) + for child in elem: + walk(child) + + walk(root) + return frozenset(targets) + + +def entry_is_non_translatable(key: str, no_translate: frozenset[str]) -> bool: + if key in no_translate: + return True + if key.startswith("@"): + element = key.split("::", 1)[0] + return element in no_translate + return False + + def element_identity(elem: ET.Element) -> str: name = elem.attrib.get("name", "").strip() if name: @@ -152,6 +282,109 @@ def entry_key(elem: ET.Element, attr: str) -> str: return f"{tag}::{attr}" +def element_map_key(elem: ET.Element) -> str | None: + ident = element_identity(elem) + if ident: + return f"@{ident}" + return None + + +def is_xml_meta_attr(attr: str) -> bool: + return attr.startswith("{") or attr.startswith("xmlns") + + +def looks_translation_attr(attr: str) -> bool: + if attr in TRANSLATABLE_ATTRS or attr in ATTR_TYPO_HINTS: + return True + lower = attr.lower() + return any(keyword in lower for keyword in TRANSLATION_ATTR_KEYWORDS) + + +def should_report_extra_attr(attr: str, value: str, en_attrs: dict[str, str]) -> bool: + if attr in ATTR_TYPO_HINTS: + return True + if attr in TRANSLATABLE_ATTRS: + return True + if looks_translation_attr(attr): + return True + key = f"@extra::{attr}" + return not is_trivial(value, key) + + +def extra_attr_hint(attr: str, en_attrs: dict[str, str]) -> str: + canonical = ATTR_TYPO_HINTS.get(attr) + if canonical and canonical in en_attrs: + return f"EN uses {canonical} on this element" + if canonical: + return f"Did you mean {canonical}?" + + for alternate in ATTR_EN_ALTERNATES.get(attr, ()): + if alternate in en_attrs: + return f"EN uses {alternate} on this element, not {attr}" + + return "Attribute not present on EN element" + + +def attr_entry_key(element: str, attr: str) -> str: + return f"{element}::{attr}" + + +def extract_element_attrs(root: ET.Element, rel: str) -> dict[str, dict[str, str]]: + elements: dict[str, dict[str, str]] = {} + + if rel == "strings.xml": + return elements + + def walk(elem: ET.Element) -> None: + ekey = element_map_key(elem) + if ekey: + bucket = elements.setdefault(ekey, {}) + for attr, value in elem.attrib.items(): + if is_xml_meta_attr(attr): + continue + bucket[attr] = normalise_text(value) + for child in elem: + walk(child) + + walk(root) + return elements + + +def compare_extra_attributes( + en_elements: dict[str, dict[str, str]], + locale_elements: dict[str, dict[str, str]], + skip_keys: frozenset[str] | None = None, + skip_elements: frozenset[str] | None = None, +) -> list[AttrIssue]: + issues: list[AttrIssue] = [] + skip = skip_keys or frozenset() + skip_elems = skip_elements or frozenset() + + for ekey, locale_attrs in locale_elements.items(): + if ekey in skip_elems: + continue + en_attrs = en_elements.get(ekey) + if en_attrs is None: + continue + for attr, value in locale_attrs.items(): + key = attr_entry_key(ekey, attr) + if key in skip: + continue + if attr in en_attrs: + continue + if not should_report_extra_attr(attr, value, en_attrs): + continue + issues.append(AttrIssue( + element=ekey, + attr=attr, + value=value, + hint=extra_attr_hint(attr, en_attrs), + )) + + issues.sort(key=lambda i: (i.element, i.attr)) + return issues + + def extract_entries(root: ET.Element, rel: str) -> dict[str, Entry]: entries: dict[str, Entry] = {} @@ -160,6 +393,8 @@ def extract_entries(root: ET.Element, rel: str) -> dict[str, Entry]: tag = child.tag.split("}")[-1] if tag != "string": continue + if is_translate_disabled(child): + continue name = child.attrib.get("name", "").strip() if not name: continue @@ -172,19 +407,20 @@ def extract_entries(root: ET.Element, rel: str) -> dict[str, Entry]: tag = elem.tag.split("}")[-1] if "}" in elem.tag else elem.tag ident = element_identity(elem) - for attr in TRANSLATABLE_ATTRS: - if attr in elem.attrib: - value = normalise_text(elem.attrib[attr]) - key = entry_key(elem, attr) - entries[key] = Entry(key, value, attr) + if not is_translate_disabled(elem): + for attr in TRANSLATABLE_ATTRS: + if attr in elem.attrib: + value = normalise_text(elem.attrib[attr]) + key = entry_key(elem, attr) + entries[key] = Entry(key, value, attr) - body = normalise_text("".join(elem.itertext())) - if ident and body and tag in { - "global", "notification", "text", "string", "ignore", "button", - }: - key = entry_key(elem, "#text") - if key not in entries: - entries[key] = Entry(key, body, "#text") + body = normalise_text("".join(elem.itertext())) + if ident and body and tag in { + "global", "notification", "text", "string", "ignore", "button", + }: + key = entry_key(elem, "#text") + if key not in entries: + entries[key] = Entry(key, body, "#text") for child in elem: walk(child) @@ -203,6 +439,8 @@ def parse_xml(path: Path) -> ET.Element | None: def collect_files(base: Path) -> dict[str, Path]: files: dict[str, Path] = {} + if not base.is_dir(): + return files for path in base.rglob("*.xml"): rel = path.relative_to(base).as_posix() if rel not in SKIP_FILES: @@ -210,172 +448,553 @@ def collect_files(base: Path) -> dict[str, Path]: return files -def compare_file(rel: str, en_path: Path, pl_path: Path | None) -> FileReport: +def compare_file(rel: str, en_path: Path, locale_path: Path | None) -> FileReport: report = FileReport(rel=rel) en_root = parse_xml(en_path) if en_root is None: return report en_all = extract_entries(en_root, rel) - en_entries = {k: v for k, v in en_all.items() if needs_pl_entry(v.value, k)} + en_no_translate = collect_no_translate_targets(en_root, rel) + en_entries = { + k: v for k, v in en_all.items() + if needs_translation_entry(v.value, k) + and not entry_is_non_translatable(k, en_no_translate) + } - if pl_path is None or not pl_path.exists(): + if locale_path is None or not locale_path.exists(): report.missing = list(en_entries.values()) return report - pl_root = parse_xml(pl_path) - if pl_root is None: + locale_root = parse_xml(locale_path) + if locale_root is None: report.missing = list(en_entries.values()) return report - pl_all = extract_entries(pl_root, rel) - pl_entries = {k: v for k, v in pl_all.items() if not is_trivial(v.value, k)} + locale_all = extract_entries(locale_root, rel) + locale_entries = { + k: v for k, v in locale_all.items() if not is_trivial(v.value, k) + } for key, entry in en_entries.items(): - if key not in pl_all: + if key not in locale_all: report.missing.append(entry) - for key, entry in pl_entries.items(): + for key, entry in locale_entries.items(): if key not in en_all: + if entry_is_non_translatable(key, en_no_translate): + continue report.redundant.append(entry) + redundant_keys = frozenset(e.key for e in report.redundant) + en_elements = extract_element_attrs(en_root, rel) + locale_elements = extract_element_attrs(locale_root, rel) + report.extra_attrs = compare_extra_attributes( + en_elements, + locale_elements, + skip_keys=redundant_keys, + skip_elements=en_no_translate, + ) + return report -def render_html( - en_count: int, - pl_count: int, - missing_files: list[str], - redundant_files: list[str], - file_reports: list[FileReport], -) -> str: - total_missing = sum(len(r.missing) for r in file_reports) - total_redundant = sum(len(r.redundant) for r in file_reports) - files_with_missing = sum(1 for r in file_reports if r.missing) - files_with_redundant = sum(1 for r in file_reports if r.redundant) +def discover_skins() -> list[str]: + skins: list[str] = [] + if not SKINS_ROOT.is_dir(): + return skins + for path in sorted(SKINS_ROOT.iterdir()): + if path.is_dir() and (path / "xui" / "en").is_dir(): + skins.append(path.name) + return skins + + +def discover_locales(skin: str) -> list[str]: + xui = SKINS_ROOT / skin / "xui" + if not xui.is_dir(): + return [] + return sorted( + d.name for d in xui.iterdir() + if d.is_dir() and d.name != "en" + ) + + +def audit_locale(skin: str, locale: str) -> LocaleReport: + en_dir = SKINS_ROOT / skin / "xui" / "en" + locale_dir = SKINS_ROOT / skin / "xui" / locale + + en_files = collect_files(en_dir) + locale_files = collect_files(locale_dir) + + en_rels = set(en_files) + locale_rels = set(locale_files) + + report = LocaleReport( + locale=locale, + en_count=len(en_files), + locale_count=len(locale_files), + missing_files=sorted(en_rels - locale_rels), + redundant_files=sorted(locale_rels - en_rels), + ) + + for rel in sorted(en_rels & locale_rels): + file_report = compare_file(rel, en_files[rel], locale_files.get(rel)) + if file_report.missing or file_report.redundant or file_report.extra_attrs: + report.file_reports.append(file_report) + + for rel in report.missing_files: + en_root = parse_xml(en_files[rel]) + if en_root is None: + continue + en_all = extract_entries(en_root, rel) + entries = [ + e for e in en_all.values() if needs_translation_entry(e.value, e.key) + ] + if entries: + report.file_reports.append(FileReport(rel=rel, missing=entries)) + + report.file_reports.sort(key=lambda r: r.rel) + return report + + +def audit_skin(skin: str) -> SkinReport: + locales = discover_locales(skin) + return SkinReport( + skin=skin, + locale_reports=[audit_locale(skin, loc) for loc in locales], + ) + + +def badge(count: int, kind: str) -> str: + if count == 0: + return f"{count}" + return f"{count}" + + +def render_file_report(report: FileReport, locale: str) -> str: + parts = [ + "
{html.escape(report.rel)}",
+ ]
+ if report.missing:
+ parts.append(badge(len(report.missing), "missing"))
+ if report.redundant:
+ parts.append(badge(len(report.redundant), "redundant"))
+ if report.extra_attrs:
+ parts.append(badge(len(report.extra_attrs), "extra-attr"))
+ parts.append("| Key | EN value |
|---|---|
{html.escape(entry.key)} | "
+ f"{html.escape(entry.value)} |
| Key | " + f"{html.escape(locale)} value |
|---|---|
{html.escape(entry.key)} | "
+ f"{html.escape(entry.value)} |
| Element | Attribute | " + f"{html.escape(locale)} value | Note |
|---|---|---|---|
{html.escape(issue.element)} | "
+ f"{html.escape(issue.attr)} | "
+ f"{html.escape(display_value)} | " + f"{html.escape(issue.hint)} |
| {html.escape(label)} | {value} |
|---|
{html.escape(rel)}{html.escape(rel)}No string differences found.
") + + parts.append("{html.escape(skin)}Base: "
+ f"indra/newview/skins/{html.escape(skin)}/xui/
No non-EN locales found for this skin.
", + ) + else: + for locale_report in skin_report.locale_reports: + parts.append(render_locale_report(locale_report)) + + parts.append("Missing string counts per skin and locale. " + "Click a skin tab for detail.
", + "| Skin | ", + ] + for loc in all_locales: + parts.append(f"{html.escape(loc)} | ") + parts.append("||
|---|---|---|---|
{html.escape(skin_report.skin)} | ")
+ for loc in all_locales:
+ locale_report = lookup.get((skin_report.skin, loc))
+ if locale_report is None:
+ parts.append("- | ") + elif not locale_report.has_issues: + parts.append("0 | ") + else: + missing = locale_report.total_missing + extra = ( + locale_report.total_redundant + + locale_report.total_extra_attrs + ) + cell = f"{missing}" + if extra: + cell += f" (+{extra})" + parts.append(f"{cell} | ") + parts.append("
Compared indra/newview/skins/default/xui/pl/ against "
- "indra/newview/skins/default/xui/en/ on 2026-06-13.
Entries are matched by widget name (viewer layered-XML rules). "
- "Omitted as trivial: integers, placeholder-only text, keyboard labels, slash commands, "
- "font names, brand constants, and EN labels identical to the widget name.
| EN XML files | {en_count} |
|---|---|
| PL XML files | {pl_count} |
| Missing PL files | {len(missing_files)} |
| Redundant PL-only files | {len(redundant_files)} |
| Shared files with missing strings | {files_with_missing} |
| Shared files with redundant strings | {files_with_redundant} |
| Total missing entries | {total_missing} |
| Total redundant entries | {total_redundant} |
All skins under indra/newview/skins/*/xui/ compared "
+ f"EN against every other locale present. Generated {audit_date}.
Entries are matched by widget name (viewer "
+ "layered-XML rules). Each locale element is also checked attribute-by-attribute "
+ "against its EN counterpart: attributes present in the translation but absent "
+ "on EN (e.g. tooltip instead of tool_tip) are flagged. "
+ "Elements with translate=\"false\" are skipped (direct content only, "
+ "not children). Omitted as trivial: integers, placeholder-only text, keyboard labels, slash "
+ "commands, font names, brand constants, and EN labels identical to the widget "
+ "name.
{html.escape(rel)}{html.escape(rel)}| Key | EN value |
|---|---|
{html.escape(entry.key)} | "
- f"{html.escape(entry.value)} |
| Key | PL value |
|---|---|
{html.escape(entry.key)} | "
- f"{html.escape(entry.value)} |