From 4583a743be8ef800d2b5a77671c264548360c8c4 Mon Sep 17 00:00:00 2001 From: Mykola Grymalyuk Date: Sun, 21 Jul 2024 11:54:54 -0600 Subject: [PATCH] sucatalog: Publish initial version --- opencore_legacy_patcher/sucatalog/__init__.py | 111 +++++ .../sucatalog/constants.py | 57 +++ opencore_legacy_patcher/sucatalog/products.py | 402 ++++++++++++++++++ opencore_legacy_patcher/sucatalog/url.py | 175 ++++++++ .../support/macos_installer_handler.py | 307 +------------ .../wx_gui/gui_macos_installer_download.py | 72 ++-- 6 files changed, 785 insertions(+), 339 deletions(-) create mode 100644 opencore_legacy_patcher/sucatalog/__init__.py create mode 100644 opencore_legacy_patcher/sucatalog/constants.py create mode 100644 opencore_legacy_patcher/sucatalog/products.py create mode 100644 opencore_legacy_patcher/sucatalog/url.py diff --git a/opencore_legacy_patcher/sucatalog/__init__.py b/opencore_legacy_patcher/sucatalog/__init__.py new file mode 100644 index 000000000..2545c2534 --- /dev/null +++ b/opencore_legacy_patcher/sucatalog/__init__.py @@ -0,0 +1,111 @@ +""" +sucatalog: Python module for querying Apple's Software Update Catalog, supporting Tiger through Sequoia. + +------------------- + +## Usage + +### Get Software Update Catalog URL + +```python +>>> import sucatalog + +>>> # Defaults to PublicRelease seed +>>> url = sucatalog.CatalogURL().url +"https://swscan.apple.com/.../index-15-14-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog" + +>>> url = sucatalog.CatalogURL(seed=sucatalog.SeedType.DeveloperSeed).url +"https://swscan.apple.com/.../index-15seed-15-14-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog" + +>>> url = sucatalog.CatalogURL(version=sucatalog.CatalogVersion.HIGH_SIERRA).url +"https://swscan.apple.com/.../index-10.13-10.12-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog" +``` + + +### Parse Software Update Catalog - InstallAssistants only + +>>> import sucatalog + +>>> # Pass contents of URL (as dictionary) +>>> catalog = plistlib.loads(requests.get(url).content) + +>>> products = sucatalog.CatalogProducts(catalog).products +[ + { + 'Build': '22G720', + 'Catalog': , + 'InstallAssistant': { + 'IntegrityDataSize': 42008, + 'IntegrityDataURL': 'https://swcdn.apple.com/.../InstallAssistant.pkg.integrityDataV1', + 'Size': 12210304673, + 'URL': 'https://swcdn.apple.com/.../InstallAssistant.pkg' + }, + 'PostDate': datetime.datetime(2024, 5, 20, 17, 18, 21), + 'ProductID': '052-96247', + 'Title': 'macOS Ventura', + 'Version': '13.6.7' + } +] + +### Parse Software Update Catalog - All products + +By default, `CatalogProducts` will only return InstallAssistants. To get all products, set `install_assistants_only=False`. + +>>> import sucatalog + +>>> # Pass contents of URL (as dictionary) +>>> products = sucatalog.CatalogProducts(catalog, install_assistants_only=False).products +[ + { + 'Build': None, + 'Catalog': None, + 'Packages': [ + { + 'MetadataURL': 'https://swdist.apple.com/.../iLifeSlideshow_v2.pkm', + 'Size': 116656956, + 'URL': 'http://swcdn.apple.com/.../iLifeSlideshow_v2.pkg' + }, + { + 'MetadataURL': 'https://swdist.apple.com/.../iPhoto9.2.3ContentUpdate.pkm', + 'Size': 59623907, + 'URL': 'http://swcdn.apple.com/.../iPhoto9.2.3ContentUpdate.pkg' + }, + { + 'MetadataURL': 'https://swdist.apple.com/.../iPhoto9.2.3Update.pkm', + 'Size': 197263405, + 'URL': 'http://swcdn.apple.com/.../iPhoto9.2.3Update.pkg' + } + ], + 'PostDate': datetime.datetime(2019, 10, 23, 0, 2, 42), + 'ProductID': '041-85230', + 'Title': 'iPhoto Update', + 'Version': '9.2.3' + }, + { + 'Build': None, + 'Catalog': None, + 'Packages': [ + { + 'Digest': '9aba109078feec7ea841529e955440b63d7755a0', + 'MetadataURL': 'https://swdist.apple.com/.../iPhoto9.4.3Update.pkm', + 'Size': 555246460, + 'URL': 'http://swcdn.apple.com/.../iPhoto9.4.3Update.pkg' + }, + { + 'Digest': '0bb013221ca2df5e178d950cb229f41b8e680d00', + 'MetadataURL': 'https://swdist.apple.com/.../iPhoto9.4.3ContentUpdate.pkm', + 'Size': 213073666, + 'URL': 'http://swcdn.apple.com/.../iPhoto9.4.3ContentUpdate.pkg' + } + ], + 'PostDate': datetime.datetime(2019, 10, 13, 3, 23, 14), + 'ProductID': '041-88859', + 'Title': 'iPhoto Update', + 'Version': '9.4.3' + } +] +""" + +from .url import CatalogURL +from .constants import CatalogVersion, SeedType +from .products import CatalogProducts \ No newline at end of file diff --git a/opencore_legacy_patcher/sucatalog/constants.py b/opencore_legacy_patcher/sucatalog/constants.py new file mode 100644 index 000000000..2d3df0463 --- /dev/null +++ b/opencore_legacy_patcher/sucatalog/constants.py @@ -0,0 +1,57 @@ +""" +constants.py: Enumerations for sucatalog-py +""" + +from enum import StrEnum + + +class SeedType(StrEnum): + """ + Enum for catalog types + + Variants: + DeveloperSeed: Developer Beta (Part of the Apple Developer Program) + PublicSeed: Public Beta + CustomerSeed: AppleSeed Program (Generally mirrors DeveloperSeed) + PublicRelease: Public Release + """ + DeveloperSeed: str = "seed" + PublicSeed: str = "beta" + CustomerSeed: str = "customerseed" + PublicRelease: str = "" + + +class CatalogVersion(StrEnum): + """ + Enum for macOS versions + + Used for generating sucatalog URLs + """ + SEQUOIA: str = "15" + SONOMA: str = "14" + VENTURA: str = "13" + MONTEREY: str = "12" + BIG_SUR: str = "11" + BIG_SUR_LEGACY: str = "10.16" + CATALINA: str = "10.15" + MOJAVE: str = "10.14" + HIGH_SIERRA: str = "10.13" + SIERRA: str = "10.12" + EL_CAPITAN: str = "10.11" + YOSEMITE: str = "10.10" + MAVERICKS: str = "10.9" + MOUNTAIN_LION: str = "mountainlion" + LION: str = "lion" + SNOW_LEOPARD: str = "snowleopard" + LEOPARD: str = "leopard" + TIGER: str = "" + + +class CatalogExtension(StrEnum): + """ + Enum for catalog extensions + + Used for generating sucatalog URLs + """ + PLIST: str = ".sucatalog" + GZIP: str = ".sucatalog.gz" \ No newline at end of file diff --git a/opencore_legacy_patcher/sucatalog/products.py b/opencore_legacy_patcher/sucatalog/products.py new file mode 100644 index 000000000..4beba6934 --- /dev/null +++ b/opencore_legacy_patcher/sucatalog/products.py @@ -0,0 +1,402 @@ +""" +products.py: Parse products from Software Update Catalog +""" + +import re +import plistlib + +import packaging.version +import xml.etree.ElementTree as ET + +from pathlib import Path +from functools import cached_property + +from .url import CatalogURL +from .constants import CatalogVersion, SeedType + +from ..support import network_handler + + +class CatalogProducts: + """ + Args: + catalog (dict): Software Update Catalog (contents of CatalogURL's URL) + install_assistants_only (bool): Only list InstallAssistant products + only_vmm_install_assistants (bool): Only list VMM-x86_64-compatible InstallAssistant products + max_install_assistant_version (CatalogVersion): Maximum InstallAssistant version to list + """ + def __init__(self, + catalog: dict, + install_assistants_only: bool = True, + only_vmm_install_assistants: bool = True, + max_install_assistant_version: CatalogVersion = CatalogVersion.SONOMA + ) -> None: + self.catalog: dict = catalog + self.ia_only: bool = install_assistants_only + self.vmm_only: bool = only_vmm_install_assistants + self.max_ia_version: packaging = packaging.version.parse(f"{max_install_assistant_version.value}.99.99") + self.max_ia_catalog: CatalogVersion = max_install_assistant_version + + + def _legacy_parse_info_plist(self, data: dict) -> dict: + """ + Legacy version of parsing for installer details through Info.plist + """ + + if "MobileAssetProperties" not in data: + return {} + if "SupportedDeviceModels" not in data["MobileAssetProperties"]: + return {} + if "OSVersion" not in data["MobileAssetProperties"]: + return {} + if "Build" not in data["MobileAssetProperties"]: + return {} + + # Ensure Apple Silicon specific Installers are not listed + if "VMM-x86_64" not in data["MobileAssetProperties"]["SupportedDeviceModels"]: + if self.vmm_only: + return {} + + version = data["MobileAssetProperties"]["OSVersion"] + build = data["MobileAssetProperties"]["Build"] + + catalog = "" + try: + catalog = data["MobileAssetProperties"]["BridgeVersionInfo"]["CatalogURL"] + except KeyError: + pass + + if any([version, build]) is None: + return {} + + return { + "Version": version, + "Build": build, + "Catalog": CatalogURL().catalog_url_to_seed(catalog), + } + + + def _parse_mobile_asset_plist(self, data: dict) -> dict: + """ + Parses the MobileAsset plist for installer details + + With macOS Sequoia, the Info.plist is no longer present in the InstallAssistant's assets + """ + + for entry in data["Assets"]: + if "SupportedDeviceModels" not in entry: + continue + if "OSVersion" not in entry: + continue + if "Build" not in entry: + continue + if "VMM-x86_64" not in entry["SupportedDeviceModels"]: + if self.vmm_only: + continue + + build = entry["Build"] + version = entry["OSVersion"] + + catalog_url = "" + try: + catalog_url = entry["BridgeVersionInfo"]["CatalogURL"] + except KeyError: + pass + + return { + "Version": version, + "Build": build, + "Catalog": CatalogURL().catalog_url_to_seed(catalog_url), + } + + return {} + + + def _parse_english_distributions(self, data: bytes) -> dict: + """ + Resolve Title, Build and Version from the English distribution file + """ + try: + plist_contents = plistlib.loads(data) + except plistlib.InvalidFileException: + plist_contents = None + + try: + xml_contents = ET.fromstring(data) + except ET.ParseError: + xml_contents = None + + _product_map = { + "Title": None, + "Build": None, + "Version": None, + } + + if plist_contents: + if "macOSProductBuildVersion" in plist_contents: + _product_map["Build"] = plist_contents["macOSProductBuildVersion"] + if "macOSProductVersion" in plist_contents: + _product_map["Version"] = plist_contents["macOSProductVersion"] + if "BUILD" in plist_contents: + _product_map["Build"] = plist_contents["BUILD"] + if "VERSION" in plist_contents: + _product_map["Version"] = plist_contents["VERSION"] + + if xml_contents: + # Fetch item title + item_title = xml_contents.find(".//title").text + if item_title in ["SU_TITLE", "MANUAL_TITLE", "MAN_TITLE"]: + # regex search the contents for the title + title_search = re.search(r'"SU_TITLE"\s*=\s*"(.*)";', data.decode("utf-8")) + if title_search: + item_title = title_search.group(1) + + _product_map["Title"] = item_title + + return _product_map + + + def _build_installer_name(self, version: str, catalog: SeedType) -> str: + """ + Builds the installer name based on the version and catalog + """ + try: + marketing_name = CatalogVersion(version.split(".")[0]).name + except ValueError: + marketing_name = "Unknown" + + # Replace _ with space + marketing_name = marketing_name.replace("_", " ") + + # Convert to upper for each word + marketing_name = "macOS " + " ".join([word.capitalize() for word in marketing_name.split()]) + + # Append Beta if needed + if catalog in [SeedType.DeveloperSeed, SeedType.PublicSeed, SeedType.CustomerSeed]: + marketing_name += " Beta" + + return marketing_name + + + def _list_latest_installers_only(self, products: list) -> list: + """ + List only the latest installers per macOS version + + macOS versions capped at n-3 (n being the latest macOS version) + """ + + supported_versions = [] + + # Build list of supported versions (n to n-3, where n is the latest macOS version set) + did_find_latest = False + for version in CatalogVersion: + if did_find_latest is False: + if version != self.max_ia_catalog: + continue + did_find_latest = True + + supported_versions.append(version) + + if len(supported_versions) == 4: + break + + # invert the list + supported_versions = supported_versions[::-1] + + # Remove all but the newest version + for version in supported_versions: + _newest_version = packaging.version.parse("0.0.0") + + # First, determine largest version + for installer in products: + if installer["Version"] is None: + continue + if not installer["Version"].startswith(version.value): + continue + if installer["Catalog"] in [SeedType.CustomerSeed, SeedType.DeveloperSeed, SeedType.PublicSeed]: + continue + try: + if packaging.version.parse(installer["Version"]) > _newest_version: + _newest_version = packaging.version.parse(installer["Version"]) + except packaging.version.InvalidVersion: + pass + + # Next, remove all installers that are not the newest version + for installer in products: + if installer["Version"] is None: + continue + if not installer["Version"].startswith(version.value): + continue + try: + if packaging.version.parse(installer["Version"]) < _newest_version: + products.remove(installer) + except packaging.version.InvalidVersion: + pass + + # Remove Betas if there's a non-beta version available + for installer in products: + if installer["Catalog"] in [SeedType.CustomerSeed, SeedType.DeveloperSeed, SeedType.PublicSeed]: + for installer_2 in products: + if installer_2["Version"].split(".")[0] == installer["Version"].split(".")[0] and installer_2["Catalog"] not in [SeedType.CustomerSeed, SeedType.DeveloperSeed, SeedType.PublicSeed]: + products.remove(installer) + + # Remove EOL versions (older than n-3) + for installer in products: + if installer["Version"].split(".")[0] < supported_versions[-4].value: + products.remove(installer) + + return products + + + @cached_property + def products(self) -> None: + """ + Returns a list of products from the sucatalog + """ + + catalog = self.catalog + + _products = [] + + for product in catalog["Products"]: + + # InstallAssistants.pkgs (macOS Installers) will have the following keys: + if self.ia_only: + if "ExtendedMetaInfo" not in catalog["Products"][product]: + continue + if "InstallAssistantPackageIdentifiers" not in catalog["Products"][product]["ExtendedMetaInfo"]: + continue + if "SharedSupport" not in catalog["Products"][product]["ExtendedMetaInfo"]["InstallAssistantPackageIdentifiers"]: + continue + + _product_map = { + "ProductID": product, + "PostDate": catalog["Products"][product]["PostDate"], + "Title": None, + "Build": None, + "Version": None, + "Catalog": None, + + # Optional keys if not InstallAssistant only: + # "Packages": None, + + # Optional keys if InstallAssistant found: + # "InstallAssistant": { + # "URL": None, + # "Size": None, + # "XNUMajor": None, + # "IntegrityDataURL": None, + # "IntegrityDataSize": None + # }, + } + + # InstallAssistant logic + if "Packages" in catalog["Products"][product]: + # Add packages to product map if not InstallAssistant only + if self.ia_only is False: + _product_map["Packages"] = catalog["Products"][product]["Packages"] + for package in catalog["Products"][product]["Packages"]: + if "URL" in package: + if Path(package["URL"]).name == "InstallAssistant.pkg": + _product_map["InstallAssistant"] = { + "URL": package["URL"], + "Size": package["Size"], + "IntegrityDataURL": package["IntegrityDataURL"], + "IntegrityDataSize": package["IntegrityDataSize"] + } + + if Path(package["URL"]).name not in ["Info.plist", "com_apple_MobileAsset_MacSoftwareUpdate.plist"]: + continue + + net_obj = network_handler.NetworkUtilities().get(package["URL"]) + if net_obj is None: + continue + + contents = net_obj.content + try: + plist_contents = plistlib.loads(contents) + except plistlib.InvalidFileException: + continue + + if plist_contents: + if Path(package["URL"]).name == "Info.plist": + _product_map.update(self._legacy_parse_info_plist(plist_contents)) + else: + _product_map.update(self._parse_mobile_asset_plist(plist_contents)) + + if _product_map["Version"] is not None: + _product_map["Title"] = self._build_installer_name(_product_map["Version"], _product_map["Catalog"]) + + # Fall back to English distribution if no version is found + if _product_map["Version"] is None: + url = None + if "Distributions" in catalog["Products"][product]: + if "English" in catalog["Products"][product]["Distributions"]: + url = catalog["Products"][product]["Distributions"]["English"] + elif "en" in catalog["Products"][product]["Distributions"]: + url = catalog["Products"][product]["Distributions"]["en"] + + if url is None: + continue + + net_obj = network_handler.NetworkUtilities().get(url) + if net_obj is None: + continue + + contents = net_obj.content + + _product_map.update(self._parse_english_distributions(contents)) + + if _product_map["Version"] is None: + if "ServerMetadataURL" in catalog["Products"][product]: + server_metadata_url = catalog["Products"][product]["ServerMetadataURL"] + + net_obj = network_handler.NetworkUtilities().get(server_metadata_url) + if net_obj is None: + continue + + server_metadata_contents = net_obj.content + + try: + server_metadata_plist = plistlib.loads(server_metadata_contents) + except plistlib.InvalidFileException: + pass + + if "CFBundleShortVersionString" in server_metadata_plist: + _product_map["Version"] = server_metadata_plist["CFBundleShortVersionString"] + + + if _product_map["Version"] is not None: + # Check if version is newer than the max version + if self.ia_only: + try: + if packaging.version.parse(_product_map["Version"]) > self.max_ia_version: + continue + except packaging.version.InvalidVersion: + pass + + if _product_map["Build"] is not None: + if "InstallAssistant" in _product_map: + try: + # Grab first 2 characters of build + _product_map["InstallAssistant"]["XNUMajor"] = int(_product_map["Build"][:2]) + except ValueError: + pass + + # If version is still None, set to 0.0.0 + if _product_map["Version"] is None: + _product_map["Version"] = "0.0.0" + + _products.append(_product_map) + + _products = sorted(_products, key=lambda x: x["Version"]) + + return _products + + + @cached_property + def latest_products(self) -> list: + """ + Returns a list of the latest products from the sucatalog + """ + return self._list_latest_installers_only(self.products) \ No newline at end of file diff --git a/opencore_legacy_patcher/sucatalog/url.py b/opencore_legacy_patcher/sucatalog/url.py new file mode 100644 index 000000000..7c3be128f --- /dev/null +++ b/opencore_legacy_patcher/sucatalog/url.py @@ -0,0 +1,175 @@ +""" +url.py: Generate URL for Software Update Catalog + +Usage: +>>> import sucatalog +>>> catalog_url = sucatalog.CatalogURL().url +https://swscan.apple.com/content/catalogs/others/index-15seed-15-14-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog +""" + +import logging +import plistlib + +from .constants import ( + SeedType, + CatalogVersion, + CatalogExtension +) + +from ..support import network_handler + + +class CatalogURL: + """ + Provides URL generation for Software Update Catalog + + Args: + version (CatalogVersion): Version of macOS + seed (SeedType): Seed type + extension (CatalogExtension): Extension for the catalog URL + """ + def __init__(self, + version: CatalogVersion = CatalogVersion.SONOMA, + seed: SeedType = SeedType.PublicRelease, + extension: CatalogExtension = CatalogExtension.PLIST + ) -> None: + self.version = version + self.seed = seed + self.extension = extension + + self.seed = self._fix_seed_type() + self.version = self._fix_version() + + + def _fix_seed_type(self) -> SeedType: + """ + Fixes seed type for URL generation + """ + # Pre-Mountain Lion lacked seed types + if self.version in [CatalogVersion.LION, CatalogVersion.SNOW_LEOPARD, CatalogVersion.LEOPARD, CatalogVersion.TIGER]: + if self.seed != SeedType.PublicRelease: + logging.warning(f"{self.seed.name} not supported for {self.version.name}, defaulting to PublicRelease") + return SeedType.PublicRelease + + # Pre-Yosemite lacked PublicSeed/CustomerSeed, thus override to DeveloperSeed + if self.version in [CatalogVersion.MAVERICKS, CatalogVersion.MOUNTAIN_LION]: + if self.seed in [SeedType.PublicSeed, SeedType.CustomerSeed]: + logging.warning(f"{self.seed.name} not supported for {self.version.name}, defaulting to DeveloperSeed") + return SeedType.DeveloperSeed + + return self.seed + + + def _fix_version(self) -> CatalogVersion: + """ + Fixes version for URL generation + """ + if self.version == CatalogVersion.BIG_SUR: + return CatalogVersion.BIG_SUR_LEGACY + + return self.version + + + def _fetch_versions_for_url(self) -> list: + """ + Fetches versions for URL generation + """ + versions: list = [] + + _did_hit_variant: bool = False + for variant in CatalogVersion: + + # Avoid appending versions newer than the current version + if variant == self.version: + _did_hit_variant = True + if _did_hit_variant is False: + continue + + # Skip invalid version + if variant in [CatalogVersion.BIG_SUR, CatalogVersion.TIGER]: + continue + + versions.append(variant.value) + + if self.version == CatalogVersion.SNOW_LEOPARD: + # Reverse list pre-Lion (ie. just Snow Leopard, since Lion is a list of one) + versions = versions[::-1] + + return versions + + + def _construct_catalog_url(self) -> str: + """ + Constructs the catalog URL based on the seed type + """ + + url: str = "https://swscan.apple.com/content/catalogs" + + if self.version == CatalogVersion.TIGER: + url += "/index" + else: + url += "/others/index" + + if self.seed in [SeedType.DeveloperSeed, SeedType.PublicSeed, SeedType.CustomerSeed]: + url += f"-{self.version.value}" + if self.version == CatalogVersion.MAVERICKS and self.seed == SeedType.CustomerSeed: + # Apple previously used 'publicseed' for CustomerSeed in Mavericks + url += "publicseed" + else: + url += f"{self.seed.value}" + + # 10.10 and older don't append versions for CustomerSeed + if self.seed == SeedType.CustomerSeed and self.version in [ + CatalogVersion.YOSEMITE, + CatalogVersion.MAVERICKS, + CatalogVersion.MOUNTAIN_LION, + CatalogVersion.LION, + CatalogVersion.SNOW_LEOPARD, + CatalogVersion.LEOPARD + ]: + pass + else: + for version in self._fetch_versions_for_url(): + url += f"-{version}" + + if self.version != CatalogVersion.TIGER: + url += ".merged-1" + url += self.extension.value + + return url + + + def catalog_url_to_seed(self, catalog_url: str) -> SeedType: + """ + Converts the Catalog URL to a SeedType + """ + if "beta" in catalog_url: + return SeedType.PublicSeed + elif "customerseed" in catalog_url: + return SeedType.CustomerSeed + elif "seed" in catalog_url: + return SeedType.DeveloperSeed + return SeedType.PublicRelease + + + @property + def url(self) -> str: + """ + Generate URL for Software Update Catalog + + Returns: + str: URL for Software Update Catalog + """ + return self._construct_catalog_url() + + + @property + def url_contents(self) -> dict: + """ + Return URL contents + """ + try: + return plistlib.loads(network_handler.NetworkUtilities().get(self.url).content) + except Exception as e: + logging.error(f"Failed to fetch URL contents: {e}") + return None diff --git a/opencore_legacy_patcher/support/macos_installer_handler.py b/opencore_legacy_patcher/support/macos_installer_handler.py index 56ed658b9..64b32c1dd 100644 --- a/opencore_legacy_patcher/support/macos_installer_handler.py +++ b/opencore_legacy_patcher/support/macos_installer_handler.py @@ -1,20 +1,17 @@ """ -macos_installer_handler.py: Handler for macOS installers, both local and remote +macos_installer_handler.py: Handler for local macOS installers """ -import enum import logging import plistlib import tempfile import subprocess -import applescript from pathlib import Path from ..datasets import os_data from . import ( - network_handler, utilities, subprocess_wrapper ) @@ -22,26 +19,6 @@ from . import ( APPLICATION_SEARCH_PATH: str = "/Applications" SFR_SOFTWARE_UPDATE_PATH: str = "SFR/com_apple_MobileAsset_SFRSoftwareUpdate/com_apple_MobileAsset_SFRSoftwareUpdate.xml" -CATALOG_URL_BASE: str = "https://swscan.apple.com/content/catalogs/others/index" -CATALOG_URL_EXTENSION: str = ".merged-1.sucatalog" -CATALOG_URL_VARIANTS: list = [ - "15", - "14", - "13", - "12", - "10.16", - "10.15", - "10.14", - "10.13", - "10.12", - "10.11", - "10.10", - "10.9", - "mountainlion", - "lion", - "snowleopard", - "leopard", -] tmp_dir = tempfile.TemporaryDirectory() @@ -221,288 +198,6 @@ fi return list_disks -class SeedType(enum.IntEnum): - """ - Enum for catalog types - - Variants: - DeveloperSeed: Developer Beta (Part of the Apple Developer Program) - PublicSeed: Public Beta - CustomerSeed: AppleSeed Program (Generally mirrors DeveloperSeed) - PublicRelease: Public Release - """ - DeveloperSeed: int = 0 - PublicSeed: int = 1 - CustomerSeed: int = 2 - PublicRelease: int = 3 - - -class RemoteInstallerCatalog: - """ - Parses Apple's Software Update catalog and finds all macOS installers. - """ - - def __init__(self, seed_override: SeedType = SeedType.PublicRelease, os_override: int = os_data.os_data.sonoma) -> None: - - self.catalog_url: str = self._construct_catalog_url(seed_override, os_override) - - self.available_apps: dict = self._parse_catalog() - self.available_apps_latest: dict = self._list_newest_installers_only() - - - def _construct_catalog_url(self, seed_type: SeedType, os_kernel: int) -> str: - """ - Constructs the catalog URL based on the seed type - - Parameters: - seed_type (SeedType): The seed type to use - - Returns: - str: The catalog URL - """ - - url: str = CATALOG_URL_BASE - - os_version: str = os_data.os_conversion.kernel_to_os(os_kernel) - os_version = "10.16" if os_version == "11" else os_version - if os_version not in CATALOG_URL_VARIANTS: - logging.error(f"OS version {os_version} is not supported, defaulting to latest") - os_version = CATALOG_URL_VARIANTS[0] - - url += f"-{os_version}" - if seed_type == SeedType.DeveloperSeed: - url += f"seed" - elif seed_type == SeedType.PublicSeed: - url += f"beta" - elif seed_type == SeedType.CustomerSeed: - url += f"customerseed" - - did_find_variant: bool = False - for variant in CATALOG_URL_VARIANTS: - if variant in url: - did_find_variant = True - if did_find_variant: - url += f"-{variant}" - - url += f"{CATALOG_URL_EXTENSION}" - - return url - - - def _fetch_catalog(self) -> dict: - """ - Fetches the catalog from Apple's servers - - Returns: - dict: The catalog as a dictionary - """ - - catalog: dict = {} - - if network_handler.NetworkUtilities(self.catalog_url).verify_network_connection() is False: - return catalog - - try: - catalog = plistlib.loads(network_handler.NetworkUtilities().get(self.catalog_url).content) - except plistlib.InvalidFileException: - return {} - - return catalog - - def _parse_catalog(self) -> dict: - """ - Parses the catalog and returns a dictionary of available installers - - Returns: - dict: Dictionary of available installers - """ - available_apps: dict = {} - - catalog: dict = self._fetch_catalog() - if not catalog: - return available_apps - - if "Products" not in catalog: - return available_apps - - for product in catalog["Products"]: - if "ExtendedMetaInfo" not in catalog["Products"][product]: - continue - if "Packages" not in catalog["Products"][product]: - continue - if "InstallAssistantPackageIdentifiers" not in catalog["Products"][product]["ExtendedMetaInfo"]: - continue - if "SharedSupport" not in catalog["Products"][product]["ExtendedMetaInfo"]["InstallAssistantPackageIdentifiers"]: - continue - if "BuildManifest" not in catalog["Products"][product]["ExtendedMetaInfo"]["InstallAssistantPackageIdentifiers"]: - continue - - for bm_package in catalog["Products"][product]["Packages"]: - if "Info.plist" not in bm_package["URL"]: - continue - if "InstallInfo.plist" in bm_package["URL"]: - continue - - try: - build_plist = plistlib.loads(network_handler.NetworkUtilities().get(bm_package["URL"]).content) - except plistlib.InvalidFileException: - continue - - if "MobileAssetProperties" not in build_plist: - continue - if "SupportedDeviceModels" not in build_plist["MobileAssetProperties"]: - continue - if "OSVersion" not in build_plist["MobileAssetProperties"]: - continue - if "Build" not in build_plist["MobileAssetProperties"]: - continue - - # Ensure Apple Silicon specific Installers are not listed - if "VMM-x86_64" not in build_plist["MobileAssetProperties"]["SupportedDeviceModels"]: - continue - - version = build_plist["MobileAssetProperties"]["OSVersion"] - build = build_plist["MobileAssetProperties"]["Build"] - - try: - catalog_url = build_plist["MobileAssetProperties"]["BridgeVersionInfo"]["CatalogURL"] - if "beta" in catalog_url: - catalog_url = "PublicSeed" - elif "customerseed" in catalog_url: - catalog_url = "CustomerSeed" - elif "seed" in catalog_url: - catalog_url = "DeveloperSeed" - else: - catalog_url = "Public" - except KeyError: - # Assume Public if no catalog URL is found - catalog_url = "Public" - - download_link = None - integrity = None - size = None - date = catalog["Products"][product]["PostDate"] - - for ia_package in catalog["Products"][product]["Packages"]: - if "InstallAssistant.pkg" not in ia_package["URL"]: - continue - if "URL" not in ia_package: - continue - if "IntegrityDataURL" not in ia_package: - continue - - download_link = ia_package["URL"] - integrity = ia_package["IntegrityDataURL"] - size = ia_package["Size"] if ia_package["Size"] else 0 - - - if any([version, build, download_link, size, integrity]) is None: - continue - - available_apps.update({ - product: { - "Version": version, - "Build": build, - "Link": download_link, - "Size": size, - "integrity": integrity, - "Source": "Apple Inc.", - "Variant": catalog_url, - "OS": os_data.os_conversion.os_to_kernel(version), - "Models": build_plist["MobileAssetProperties"]["SupportedDeviceModels"], - "Date": date - } - }) - - available_apps = {k: v for k, v in sorted(available_apps.items(), key=lambda x: x[1]['Version'])} - - return available_apps - - - def _list_newest_installers_only(self) -> dict: - """ - Returns a dictionary of the newest macOS installers only. - Primarily used to avoid overwhelming the user with a list of - installers that are not the newest version. - - Returns: - dict: A dictionary of the newest macOS installers only. - """ - - if self.available_apps is None: - return {} - - newest_apps: dict = self.available_apps.copy() - supported_versions = ["10.13", "10.14", "10.15", "11", "12", "13", "14"] - - for version in supported_versions: - remote_version_minor = 0 - remote_version_security = 0 - os_builds = [] - - # First determine the largest version - for ia in newest_apps: - if newest_apps[ia]["Version"].startswith(version): - if newest_apps[ia]["Variant"] not in ["CustomerSeed", "DeveloperSeed", "PublicSeed"]: - remote_version = newest_apps[ia]["Version"].split(".") - if remote_version[0] == "10": - remote_version.pop(0) - remote_version.pop(0) - else: - remote_version.pop(0) - if int(remote_version[0]) > remote_version_minor: - remote_version_minor = int(remote_version[0]) - remote_version_security = 0 # Reset as new minor version found - if len(remote_version) > 1: - if int(remote_version[1]) > remote_version_security: - remote_version_security = int(remote_version[1]) - - # Now remove all versions that are not the largest - for ia in list(newest_apps): - # Don't use Beta builds to determine latest version - if newest_apps[ia]["Variant"] in ["CustomerSeed", "DeveloperSeed", "PublicSeed"]: - continue - - if newest_apps[ia]["Version"].startswith(version): - remote_version = newest_apps[ia]["Version"].split(".") - if remote_version[0] == "10": - remote_version.pop(0) - remote_version.pop(0) - else: - remote_version.pop(0) - if int(remote_version[0]) < remote_version_minor: - newest_apps.pop(ia) - continue - if int(remote_version[0]) == remote_version_minor: - if len(remote_version) > 1: - if int(remote_version[1]) < remote_version_security: - newest_apps.pop(ia) - continue - else: - if remote_version_security > 0: - newest_apps.pop(ia) - continue - - # Remove duplicate builds - # ex. macOS 12.5.1 has 2 builds in the Software Update Catalog - # ref: https://twitter.com/classicii_mrmac/status/1560357471654379522 - if newest_apps[ia]["Build"] in os_builds: - newest_apps.pop(ia) - continue - - os_builds.append(newest_apps[ia]["Build"]) - - # Remove Betas if there's a non-beta version available - for ia in list(newest_apps): - if newest_apps[ia]["Variant"] in ["CustomerSeed", "DeveloperSeed", "PublicSeed"]: - for ia2 in newest_apps: - if newest_apps[ia2]["Version"].split(".")[0] == newest_apps[ia]["Version"].split(".")[0] and newest_apps[ia2]["Variant"] not in ["CustomerSeed", "DeveloperSeed", "PublicSeed"]: - newest_apps.pop(ia) - break - - return newest_apps - - class LocalInstallerCatalog: """ Finds all macOS installers on the local machine. diff --git a/opencore_legacy_patcher/wx_gui/gui_macos_installer_download.py b/opencore_legacy_patcher/wx_gui/gui_macos_installer_download.py index 152b6a34c..27da9501f 100644 --- a/opencore_legacy_patcher/wx_gui/gui_macos_installer_download.py +++ b/opencore_legacy_patcher/wx_gui/gui_macos_installer_download.py @@ -10,7 +10,10 @@ import webbrowser from pathlib import Path -from .. import constants +from .. import ( + constants, + sucatalog +) from ..datasets import ( os_data, @@ -46,7 +49,7 @@ class macOSInstallerDownloadFrame(wx.Frame): self.available_installers = None self.available_installers_latest = None - self.catalog_seed: macos_installer_handler.SeedType = macos_installer_handler.SeedType.DeveloperSeed + self.catalog_seed: sucatalog.SeedType = sucatalog.SeedType.DeveloperSeed self.frame_modal = wx.Dialog(parent, title=title, size=(330, 200)) @@ -132,10 +135,16 @@ class macOSInstallerDownloadFrame(wx.Frame): # Grab installer catalog def _fetch_installers(): - logging.info(f"Fetching installer catalog: {macos_installer_handler.SeedType(self.catalog_seed).name}") - remote_obj = macos_installer_handler.RemoteInstallerCatalog(seed_override=self.catalog_seed) - self.available_installers = remote_obj.available_apps - self.available_installers_latest = remote_obj.available_apps_latest + logging.info(f"Fetching installer catalog: {sucatalog.SeedType.DeveloperSeed.name}") + + sucatalog_contents = sucatalog.CatalogURL(seed=sucatalog.SeedType.DeveloperSeed).url_contents + if sucatalog_contents is None: + logging.error("Failed to download Installer Catalog from Apple") + return + + self.available_installers = sucatalog.CatalogProducts(sucatalog_contents).products + self.available_installers_latest = sucatalog.CatalogProducts(sucatalog_contents).latest_products + thread = threading.Thread(target=_fetch_installers) thread.start() @@ -157,7 +166,7 @@ class macOSInstallerDownloadFrame(wx.Frame): bundles = [wx.BitmapBundle.FromBitmaps(icon) for icon in self.icons] self.frame_modal.Destroy() - self.frame_modal = wx.Dialog(self, title="Select macOS Installer", size=(460, 500)) + self.frame_modal = wx.Dialog(self, title="Select macOS Installer", size=(505, 500)) # Title: Select macOS Installer title_label = wx.StaticText(self.frame_modal, label="Select macOS Installer", pos=(-1,-1)) @@ -169,35 +178,31 @@ class macOSInstallerDownloadFrame(wx.Frame): self.list = wx.ListCtrl(self.frame_modal, id, style=wx.LC_REPORT | wx.LC_SINGLE_SEL | wx.LC_NO_HEADER | wx.BORDER_SUNKEN) self.list.SetSmallImages(bundles) - self.list.InsertColumn(0, "Version") - self.list.InsertColumn(1, "Size") - self.list.InsertColumn(2, "Release Date") + self.list.InsertColumn(0, "Title", width=175) + self.list.InsertColumn(1, "Version", width=50) + self.list.InsertColumn(2, "Build", width=75) + self.list.InsertColumn(3, "Size", width=75) + self.list.InsertColumn(4, "Release Date", width=100) installers = self.available_installers_latest if show_full is False else self.available_installers if show_full is False: - self.frame_modal.SetSize((460, 370)) + self.frame_modal.SetSize((490, 370)) if installers: locale.setlocale(locale.LC_TIME, '') logging.info(f"Available installers on SUCatalog ({'All entries' if show_full else 'Latest only'}):") for item in installers: - extra = " Beta" if installers[item]['Variant'] in ["DeveloperSeed" , "PublicSeed"] else "" - logging.info(f"- macOS {installers[item]['Version']} ({installers[item]['Build']}):\n - Size: {utilities.human_fmt(installers[item]['Size'])}\n - Source: {installers[item]['Source']}\n - Variant: {installers[item]['Variant']}\n - Link: {installers[item]['Link']}\n") - index = self.list.InsertItem(self.list.GetItemCount(), f"macOS {installers[item]['Version']} {os_data.os_conversion.convert_kernel_to_marketing_name(int(installers[item]['Build'][:2]))}{extra} ({installers[item]['Build']})") - self.list.SetItemImage(index, self._macos_version_to_icon(int(installers[item]['Build'][:2]))) - self.list.SetItem(index, 1, utilities.human_fmt(installers[item]['Size'])) - self.list.SetItem(index, 2, installers[item]['Date'].strftime("%x")) + logging.info(f"- {item['Title']} ({item['Version']} - {item['Build']}):\n - Size: {utilities.human_fmt(item['InstallAssistant']['Size'])}\n - Link: {item['InstallAssistant']['URL']}\n") + index = self.list.InsertItem(self.list.GetItemCount(), f"{item['Title']}") + self.list.SetItemImage(index, self._macos_version_to_icon(int(item['Build'][:2]))) + self.list.SetItem(index, 1, item['Version']) + self.list.SetItem(index, 2, item['Build']) + self.list.SetItem(index, 3, utilities.human_fmt(item['InstallAssistant']['Size'])) + self.list.SetItem(index, 4, item['PostDate'].strftime("%x")) else: logging.error("No installers found on SUCatalog") wx.MessageDialog(self.frame_modal, "Failed to download Installer Catalog from Apple", "Error", wx.OK | wx.ICON_ERROR).ShowModal() - self.list.SetColumnWidth(0, 280) - self.list.SetColumnWidth(1, 65) - if show_full is True: - self.list.SetColumnWidth(2, 80) - else: - self.list.SetColumnWidth(2, 94) # Hack to get the highlight to fill the ListCtrl - if show_full is False: self.list.Select(-1) @@ -256,7 +261,7 @@ class macOSInstallerDownloadFrame(wx.Frame): if not clipboard.IsOpened(): clipboard.Open() - clipboard.SetData(wx.TextDataObject(list(installers.values())[selected_item]['Link'])) + clipboard.SetData(wx.TextDataObject(installers[selected_item]['InstallAssistant']['URL'])) clipboard.Close() @@ -278,14 +283,15 @@ class macOSInstallerDownloadFrame(wx.Frame): selected_item = self.list.GetFirstSelected() if selected_item != -1: + selected_installer = installers[selected_item] - logging.info(f"Selected macOS {list(installers.values())[selected_item]['Version']} ({list(installers.values())[selected_item]['Build']})") + logging.info(f"Selected macOS {selected_installer['Version']} ({selected_installer['Build']})") # Notify user whether their model is compatible with the selected installer problems = [] model = self.constants.custom_model or self.constants.computer.real_model if model in smbios_data.smbios_dictionary: - if list(installers.values())[selected_item]["OS"] >= os_data.os_data.ventura: + if selected_installer["InstallAssistant"]["XNUMajor"] >= os_data.os_data.ventura: if smbios_data.smbios_dictionary[model]["CPU Generation"] <= cpu_data.CPUGen.penryn or model in ["MacPro4,1", "MacPro5,1", "Xserve3,1"]: if model.startswith("MacBook"): problems.append("Lack of internal Keyboard/Trackpad in macOS installer.") @@ -293,7 +299,7 @@ class macOSInstallerDownloadFrame(wx.Frame): problems.append("Lack of internal Keyboard/Mouse in macOS installer.") if problems: - logging.warning(f"Potential issues with {model} and {list(installers.values())[selected_item]['Version']} ({list(installers.values())[selected_item]['Build']}): {problems}") + logging.warning(f"Potential issues with {model} and {selected_installer['Version']} ({selected_installer['Build']}): {problems}") problems = "\n".join(problems) dlg = wx.MessageDialog(self.frame_modal, f"Your model ({model}) may not be fully supported by this installer. You may encounter the following issues:\n\n{problems}\n\nFor more information, see associated page. Otherwise, we recommend using macOS Monterey", "Potential Issues", wx.YES_NO | wx.CANCEL | wx.ICON_WARNING) dlg.SetYesNoCancelLabels("View Github Issue", "Download Anyways", "Cancel") @@ -305,7 +311,7 @@ class macOSInstallerDownloadFrame(wx.Frame): return host_space = utilities.get_free_space() - needed_space = list(installers.values())[selected_item]['Size'] * 2 + needed_space = selected_installer['InstallAssistant']['Size'] * 2 if host_space < needed_space: logging.error(f"Insufficient space to download and extract: {utilities.human_fmt(host_space)} available vs {utilities.human_fmt(needed_space)} required") dlg = wx.MessageDialog(self.frame_modal, f"You do not have enough free space to download and extract this installer. Please free up some space and try again\n\n{utilities.human_fmt(host_space)} available vs {utilities.human_fmt(needed_space)} required", "Insufficient Space", wx.OK | wx.ICON_WARNING) @@ -314,22 +320,22 @@ class macOSInstallerDownloadFrame(wx.Frame): self.frame_modal.Close() - download_obj = network_handler.DownloadObject(list(installers.values())[selected_item]['Link'], self.constants.payload_path / "InstallAssistant.pkg") + download_obj = network_handler.DownloadObject(selected_installer['InstallAssistant']['URL'], self.constants.payload_path / "InstallAssistant.pkg") gui_download.DownloadFrame( self, title=self.title, global_constants=self.constants, download_obj=download_obj, - item_name=f"macOS {list(installers.values())[selected_item]['Version']} ({list(installers.values())[selected_item]['Build']})", - download_icon=self.constants.icons_path[self._macos_version_to_icon(int(list(installers.values())[selected_item]['Build'][:2]))] + item_name=f"macOS {selected_installer['Version']} ({selected_installer['Build']})", + download_icon=self.constants.icons_path[self._macos_version_to_icon(selected_installer["InstallAssistant"]["XNUMajor"])] ) if download_obj.download_complete is False: self.on_return_to_main_menu() return - self._validate_installer(list(installers.values())[selected_item]['integrity']) + self._validate_installer(selected_installer['InstallAssistant']['IntegrityDataURL']) def _validate_installer(self, chunklist_link: str) -> None: