products_appledb.py: Add AppleDB as InstallAssistant source

opencore_legacy_patcher/datasets/os_data.py

@@ -30,6 +30,7 @@ class os_data(enum.IntEnum):
     ventura =       22
     sonoma =        23
     sequoia =       24
+    tahoe =         25
     max_os =        99
 
 

opencore_legacy_patcher/sucatalog/__init__.py

@@ -108,4 +108,5 @@ By default, `CatalogProducts` will only return InstallAssistants. To get all pro
 
 from .url       import CatalogURL
 from .constants import CatalogVersion, SeedType
-from .products  import CatalogProducts
+from .products  import CatalogProducts
+from .products_appledb import AppleDBProducts

opencore_legacy_patcher/sucatalog/constants.py

@@ -27,6 +27,7 @@ class CatalogVersion(StrEnum):
 
     Used for generating sucatalog URLs
     """
+    TAHOE:          str = "26"
     SEQUOIA:        str = "15"
     SONOMA:         str = "14"
     VENTURA:        str = "13"

opencore_legacy_patcher/sucatalog/products_appledb.py

@@ -0,0 +1,182 @@
+"""
+products.py: Parse products from Software Update Catalog
+"""
+
+import datetime
+import hashlib
+import logging
+import zoneinfo
+
+import packaging.version
+
+from functools import cached_property
+
+from opencore_legacy_patcher import constants
+from opencore_legacy_patcher.datasets.os_data import os_data
+
+from ..support import network_handler
+
+
+APPLEDB_API_URL = "https://api.appledb.dev/ios/macOS/main.json"
+
+
+class AppleDBProducts:
+    """
+    Fetch InstallAssistants from AppleDB
+    """
+
+    def __init__(
+        self,
+        global_constants: constants.Constants,
+        max_install_assistant_version: os_data = os_data.sequoia,
+    ) -> None:
+        self.constants: constants.Constants = global_constants
+
+        try:
+            self.data = (
+                network_handler.NetworkUtilities()
+                .get(APPLEDB_API_URL, headers={"User-Agent": f"OCLP/{self.constants.patcher_version}"})
+                .json()
+            )
+        except Exception as e:
+            self.data = []
+            logging.error(f"Failed to fetch AppleDB API response: {e}")
+            return
+
+        self.max_ia: os_data = max_install_assistant_version
+
+    def _build_installer_name(self, xnu_major: int, beta: bool) -> str:
+        """
+        Builds the installer name based on the version and catalog
+        """
+        try:
+            return f"macOS {os_data(xnu_major).name.replace('_', ' ').title()}{' Beta' if beta else ''}"
+        except ValueError:
+            return f"macOS{' Beta' if beta else ''}"
+
+    def _list_latest_installers_only(self, products: list) -> list:
+        """
+        List only the latest installers per macOS version
+
+        macOS versions capped at n-3 (n being the latest macOS version)
+        """
+
+        supported_versions = {
+            os_data(i): [v for v in products if v["InstallAssistant"]["XNUMajor"] == i] for i in range(self.max_ia - 3, self.max_ia + 1)
+        }
+
+        for versions in supported_versions.values():
+            versions.sort(key=lambda v: (not v["Beta"], packaging.version.parse(v["RawVersion"])), reverse=True)
+
+        return [next(iter(versions)) for versions in supported_versions.values() if versions]
+
+    @cached_property
+    def products(self) -> None:
+        """
+        Returns a list of products from the sucatalog
+        """
+
+        _products = []
+
+        for firmware in self.data:
+            if firmware.get("internal") or firmware.get("sdk") or firmware.get("rsr"):
+                continue
+
+            # AppleDB does not track whether an installer supports the VMM pseudo-identifier,
+            # so we will use MacPro7,1, which supports all macOS versions that we care about.
+            if "MacPro7,1" not in firmware["deviceMap"]:
+                continue
+
+            firmware["raw_version"] = firmware["version"].partition(" ")[0]
+
+            xnu_major = int(firmware["build"][:2])
+            beta = firmware.get("beta") or firmware.get("rc")
+
+            details = {
+                # Dates in AppleDB are in Cupertino time. There are no times, so pin to 10 AM
+                "PostDate": datetime.datetime.fromisoformat(firmware["released"]).replace(
+                    # hour=10,
+                    # minute=0,
+                    # second=0,
+                    tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles"),
+                ),
+                "Title": f"{self._build_installer_name(xnu_major, beta)}",
+                "Build": firmware["build"],
+                "RawVersion": firmware["raw_version"],
+                "Version": firmware["version"],
+                "Beta": beta,
+                "InstallAssistant": {"XNUMajor": xnu_major},
+            }
+
+            if xnu_major > self.max_ia:
+                continue
+
+            for source in firmware.get("sources", []):
+                if source["type"] != "installassistant":
+                    continue
+
+                if "MacPro7,1" not in source["deviceMap"]:
+                    continue
+
+                for link in source["links"]:
+                    if not link["active"]:
+                        continue
+
+                    if not network_handler.NetworkUtilities(link["url"]).validate_link():
+                        continue
+
+                    details["InstallAssistant"] |= {
+                        "URL": link["url"],
+                        "Size": source.get("size", 0),
+                        "Checksum": source.get("hashes"),
+                    }
+                    break
+                else:
+                    continue
+
+                break
+
+            else:
+                # No applicable InstallAssistants, or no active sources
+                continue
+
+            _products.append(details)
+
+        _products = sorted(_products, key=lambda x: x["Beta"])
+        _deduplicated_products = []
+        _seen_builds = set()
+
+        # Prevent RCs that were the final release from showing up
+        for product in _products:
+            if product["Beta"] and product["Build"] in _seen_builds:
+                continue
+            _deduplicated_products.append(product)
+            _seen_builds.add(product["Build"])
+
+        _deduplicated_products = sorted(
+            _deduplicated_products, key=lambda x: (packaging.version.parse(x["RawVersion"]), x["Build"], not x["Beta"])
+        )
+
+        return _deduplicated_products
+
+    @cached_property
+    def latest_products(self) -> list:
+        """
+        Returns a list of the latest products from the sucatalog
+        """
+        return self._list_latest_installers_only(self.products)
+
+    def checksum_for_product(self, product: dict):
+        """
+        Returns the checksum and algorithm for a given product
+        """
+        HASH_TO_ALGO = {"md5": hashlib.md5, "sha1": hashlib.sha1, "sha2-256": hashlib.sha256, "sha2-512": hashlib.sha512}
+
+        if not product.get("InstallAssistant", {}).get("Checksum"):
+            return None, None
+
+        for algo, hash_func in HASH_TO_ALGO.items():
+            if algo in product["InstallAssistant"]["Checksum"]:
+                return product["InstallAssistant"]["Checksum"][algo], hash_func()
+
+        return None, None

opencore_legacy_patcher/wx_gui/gui_macos_installer_download.py

@@ -46,6 +46,7 @@ class macOSInstallerDownloadFrame(wx.Frame):
         self.title: str = title
         self.parent: wx.Frame = parent
 
+        self.catalog_products = None
         self.available_installers = None
         self.available_installers_latest = None
 
@@ -142,8 +143,10 @@ class macOSInstallerDownloadFrame(wx.Frame):
                 logging.error("Failed to download Installer Catalog from Apple")
                 return
 
-            self.available_installers        = sucatalog.CatalogProducts(sucatalog_contents).products
+            self.catalog_products = sucatalog.AppleDBProducts(self.constants)
-            self.available_installers_latest = sucatalog.CatalogProducts(sucatalog_contents).latest_products
+
+            self.available_installers        = self.catalog_products.products
+            self.available_installers_latest = self.catalog_products.latest_products
 
 
         thread = threading.Thread(target=_fetch_installers)
@@ -165,7 +168,7 @@ class macOSInstallerDownloadFrame(wx.Frame):
         bundles = [wx.BitmapBundle.FromBitmaps(icon) for icon in self.icons]
 
         self.frame_modal.Destroy()
-        self.frame_modal = wx.Dialog(self, title="Select macOS Installer", size=(505, 500))
+        self.frame_modal = wx.Dialog(self, title="Select macOS Installer", size=(550, 500))
 
         # Title: Select macOS Installer
         title_label = wx.StaticText(self.frame_modal, label="Select macOS Installer", pos=(-1,-1))
@@ -177,15 +180,15 @@ class macOSInstallerDownloadFrame(wx.Frame):
         self.list = wx.ListCtrl(self.frame_modal, id, style=wx.LC_REPORT | wx.LC_SINGLE_SEL | wx.LC_NO_HEADER | wx.BORDER_SUNKEN)
         self.list.SetSmallImages(bundles)
 
-        self.list.InsertColumn(0, "Title",        width=175)
+        self.list.InsertColumn(0, "Title",        width=190 if show_full else 150)
-        self.list.InsertColumn(1, "Version",      width=50)
+        self.list.InsertColumn(1, "Version",      width=80 if show_full else 50)
         self.list.InsertColumn(2, "Build",        width=75)
         self.list.InsertColumn(3, "Size",         width=75)
         self.list.InsertColumn(4, "Release Date", width=100)
 
         installers = self.available_installers_latest if show_full is False else self.available_installers
         if show_full is False:
-            self.frame_modal.SetSize((490, 370))
+            self.frame_modal.SetSize((480, 370))
 
         if installers:
             locale.setlocale(locale.LC_TIME, '')
@@ -319,7 +322,11 @@ class macOSInstallerDownloadFrame(wx.Frame):
 
             self.frame_modal.Close()
 
-            download_obj = network_handler.DownloadObject(selected_installer['InstallAssistant']['URL'], self.constants.payload_path / "InstallAssistant.pkg")
+            expected_checksum, checksum_algo = self.catalog_products.checksum_for_product(selected_installer)
+
+            download_obj = network_handler.DownloadObject(
+                selected_installer["InstallAssistant"]["URL"], self.constants.payload_path / "InstallAssistant.pkg", checksum_algo=checksum_algo
+            )
 
             gui_download.DownloadFrame(
                 self,
@@ -334,24 +341,31 @@ class macOSInstallerDownloadFrame(wx.Frame):
                 self.on_return_to_main_menu()
                 return
 
-            self._validate_installer(selected_installer['InstallAssistant']['IntegrityDataURL'])
+            self._validate_installer(expected_checksum, download_obj.checksum)
 
 
-    def _validate_installer(self, chunklist_link: str) -> None:
+    def _validate_installer(self, expected_checksum: str, calculated_checksum: str) -> None:
         """
         Validate macOS installer
         """
+
+        if expected_checksum != calculated_checksum:
+            logging.error(f"Checksum validation failed: Expected {expected_checksum}, got {calculated_checksum}")
+            wx.MessageBox(f"Checksum validation failed!\n\nThis generally happens when downloading on unstable connections such as WiFi or cellular.\n\nPlease try redownloading again on a stable connection (ie. Ethernet)", "Corrupted Installer!", wx.OK | wx.ICON_ERROR)
+            self.on_return_to_main_menu()
+            return
+
         self.SetSize((300, 200))
         for child in self.GetChildren():
             child.Destroy()
 
-        # Title: Validating macOS Installer
+        logging.info("macOS installer validated")
-        title_label = wx.StaticText(self, label="Validating macOS Installer", pos=(-1,5))
+
+        title_label = wx.StaticText(self, label="Extracting macOS Installer", pos=(-1,5))
         title_label.SetFont(gui_support.font_factory(19, wx.FONTWEIGHT_BOLD))
         title_label.Centre(wx.HORIZONTAL)
 
-        # Label: Validating chunk 0 of 0
+        chunk_label = wx.StaticText(self, label="May take a few minutes...", pos=(-1, title_label.GetPosition()[1] + title_label.GetSize()[1] + 5))
-        chunk_label = wx.StaticText(self, label="Validating chunk 0 of 0", pos=(-1, title_label.GetPosition()[1] + title_label.GetSize()[1] + 5))
         chunk_label.SetFont(gui_support.font_factory(13, wx.FONTWEIGHT_NORMAL))
         chunk_label.Centre(wx.HORIZONTAL)
 
@@ -363,39 +377,6 @@ class macOSInstallerDownloadFrame(wx.Frame):
         self.SetSize((-1, progress_bar.GetPosition()[1] + progress_bar.GetSize()[1] + 40))
         self.Show()
 
-        chunklist_stream = network_handler.NetworkUtilities().get(chunklist_link).content
-        if chunklist_stream:
-            logging.info("Validating macOS installer")
-            utilities.disable_sleep_while_running()
-            chunk_obj = integrity_verification.ChunklistVerification(self.constants.payload_path / Path("InstallAssistant.pkg"), chunklist_stream)
-            if chunk_obj.chunks:
-                progress_bar.SetValue(chunk_obj.current_chunk)
-                progress_bar.SetRange(chunk_obj.total_chunks)
-
-                wx.App.Get().Yield()
-                chunk_obj.validate()
-
-                while chunk_obj.status == integrity_verification.ChunklistStatus.IN_PROGRESS:
-                    progress_bar.SetValue(chunk_obj.current_chunk)
-                    chunk_label.SetLabel(f"Validating chunk {chunk_obj.current_chunk} of {chunk_obj.total_chunks}")
-                    chunk_label.Centre(wx.HORIZONTAL)
-                    wx.App.Get().Yield()
-
-                if chunk_obj.status == integrity_verification.ChunklistStatus.FAILURE:
-                    logging.error(f"Chunklist validation failed: Hash mismatch on {chunk_obj.current_chunk}")
-                    wx.MessageBox(f"Chunklist validation failed: Hash mismatch on {chunk_obj.current_chunk}\n\nThis generally happens when downloading on unstable connections such as WiFi or cellular.\n\nPlease try redownloading again on a stable connection (ie. Ethernet)", "Corrupted Installer!", wx.OK | wx.ICON_ERROR)
-                    self.on_return_to_main_menu()
-                    return
-
-        logging.info("macOS installer validated")
-
-        # Extract installer
-        title_label.SetLabel("Extracting macOS Installer")
-        title_label.Centre(wx.HORIZONTAL)
-
-        chunk_label.SetLabel("May take a few minutes...")
-        chunk_label.Centre(wx.HORIZONTAL)
-
         progress_bar_animation = gui_support.GaugePulseCallback(self.constants, progress_bar)
         progress_bar_animation.start_pulse()