Merge pull request #1026 from dortania/kdk-updates

kdk_handler.py: Switch to KdkSupportPkg as primary source

resources/kdk_handler.py

@@ -20,7 +20,7 @@ class kernel_debug_kit_handler:
         self.constants = constants
 
     def get_available_kdks(self):
-        KDK_API_LINK = "https://kdk-api.dhinak.net/v1"
+        KDK_API_LINK = "https://raw.githubusercontent.com/dortania/KdkSupportPkg/gh-pages/manifest.json"
 
         print("- Fetching available KDKs")
 
@@ -36,93 +36,6 @@ class kernel_debug_kit_handler:
 
         return sorted(results.json(), key=lambda x: (packaging.version.parse(x["version"]), datetime.datetime.fromisoformat(x["date"])), reverse=True)
 
-    def get_closest_match_legacy(self, host_version: str, host_build: str):
-        # Get the closest match to the provided version
-        # KDKs are generally a few days late, so we'll rely on N-1 matching
-
-        # Note: AppleDB is manually updated, so this is not a perfect solution
-
-        OS_DATABASE_LINK = "https://api.appledb.dev/main.json"
-        VERSION_PATTERN = re.compile(r"\d+\.\d+(\.\d+)?")
-
-        parsed_host_version = cast(packaging.version.Version, packaging.version.parse(host_version))
-
-        print(f"- Checking closest match for: {host_version} build {host_build}")
-
-        try:
-            results = utilities.SESSION.get(OS_DATABASE_LINK)
-        except (requests.exceptions.Timeout, requests.exceptions.TooManyRedirects, requests.exceptions.ConnectionError):
-            print("- Could not contact AppleDB")
-            return None, "", ""
-
-        if results.status_code != 200:
-            print("- Could not fetch database")
-            return None, "", ""
-
-        macos_builds = [i for i in results.json()["ios"] if i["osType"] == "macOS"]
-        # If the version is borked, put it at the bottom of the list
-        # Would omit it, but can't do that in this lambda
-        macos_builds.sort(key=lambda x: (packaging.version.parse(VERSION_PATTERN.match(x["version"]).group() if VERSION_PATTERN.match(x["version"]) else "0.0.0"), datetime.datetime.fromisoformat(x["released"] if x["released"] != "" else "1984-01-01")), reverse=True)  # type: ignore
-
-        # Iterate through, find build that is closest to the host version
-        # Use date to determine which is closest
-        for build_info in macos_builds:
-            if build_info["osType"] == "macOS":
-                raw_version = VERSION_PATTERN.match(build_info["version"])
-                if not raw_version:
-                    # Skip if version is borked
-                    continue
-                version = cast(packaging.version.Version, packaging.version.parse(raw_version.group()))
-                build = build_info["build"]
-                if build == host_build:
-                    # Skip, as we want the next closest match
-                    continue
-                elif version <= parsed_host_version and version.major == parsed_host_version.major and version.minor == parsed_host_version.minor:
-                    # The KDK list is already sorted by date then version, so the first match is the closest
-                    print(f"- Closest match: {version} build {build}")
-                    return self.generate_kdk_link(str(version), build), str(version), build
-
-        print("- Could not find a match")
-        return None, "", ""
-
-    def generate_kdk_link(self, version: str, build: str):
-        return f"https://download.developer.apple.com/macOS/Kernel_Debug_Kit_{version}_build_{build}/Kernel_Debug_Kit_{version}_build_{build}.dmg"
-
-    def verify_apple_developer_portal(self, link):
-        # Determine whether Apple Developer Portal is up
-        # and if the requested file is available
-
-        # Returns following:
-        # 0: Portal is up and file is available
-        # 1: Portal is up but file is not available
-        # 2: Portal is down
-        # 3: Network error
-
-        if utilities.verify_network_connection("https://developerservices2.apple.com/services/download") is False:
-            print("- Could not connect to the network")
-            return 3
-
-        TOKEN_URL_BASE = "https://developerservices2.apple.com/services/download"
-        remote_path = urllib.parse.urlparse(link).path
-        token_url = urllib.parse.urlunparse(urllib.parse.urlparse(TOKEN_URL_BASE)._replace(query=urllib.parse.urlencode({"path": remote_path})))
-
-        try:
-            response = utilities.SESSION.get(token_url, timeout=5)
-        except (requests.exceptions.Timeout, requests.exceptions.TooManyRedirects, requests.exceptions.ConnectionError):
-            print("- Could not contact Apple download servers")
-            return 2
-
-        try:
-            response.raise_for_status()
-        except requests.exceptions.HTTPError:
-            if response.status_code == 400 and "The path specified is invalid" in response.text:
-                print("- File does not exist on Apple download servers")
-                return 1
-            else:
-                print("- Could not request download authorization from Apple download servers")
-                return 2
-        return 0
-
     def download_kdk(self, version: str, build: str):
         detected_build = build
 
@@ -145,26 +58,23 @@ class kernel_debug_kit_handler:
                 kdk_version = cast(packaging.version.Version, packaging.version.parse(kdk["version"]))
                 if kdk["build"] == build:
                     download_link = kdk["url"]
-                elif not closest_match_download_link and kdk_version <= parsed_version and kdk_version.major == parsed_version.major and (kdk_version.minor == parsed_version.minor or kdk_version.minor == parsed_version.minor - 1):
+                elif not closest_match_download_link and kdk_version <= parsed_version and kdk_version.major == parsed_version.major and (kdk_version.minor in range(parsed_version.minor - 1, parsed_version.minor + 1)):
-                    # The KDK list is already sorted by date then version, so the first match is the closest
+                    # The KDK list is already sorted by version then date, so the first match is the closest
                     closest_match_download_link = kdk["url"]
                     closest_version = kdk["version"]
                     closest_build = kdk["build"]
         else:
-            print("- Could not fetch KDK list, falling back to brute force")
+            msg = "Could not fetch KDK list"
-            download_link = self.generate_kdk_link(version, build)
+            print(f"- {msg}")
-            closest_match_download_link, closest_version, closest_build = self.get_closest_match_legacy(version, build)
+            return False, msg, ""
 
         print(f"- Checking for KDK matching macOS {version} build {build}")
         # download_link is None if no matching KDK is found, so we'll fall back to the closest match
-        result = self.verify_apple_developer_portal(download_link) if download_link else 1
+        if not download_link:
-        if result == 0:
-            print("- Downloading KDK")
-        elif result == 1:
             print("- Could not find KDK, finding closest match")
 
             if self.is_kdk_installed(closest_build) is True:
-                print(f"- Closet Build ({closest_build}) already installed")
+                print(f"- Closest build ({closest_build}) already installed")
                 self.remove_unused_kdks(exclude_builds=[detected_build, closest_build])
                 return True, "", closest_build
 
@@ -174,47 +84,22 @@ class kernel_debug_kit_handler:
                 return False, msg, ""
 
             print(f"- Closest match: {closest_version} build {closest_build}")
-            result = self.verify_apple_developer_portal(closest_match_download_link)
+            download_link = closest_match_download_link
 
-            if result == 0:
+        if utilities.verify_network_connection(download_link):
-                print("- Downloading KDK")
+            print("- Downloading KDK")
-                download_link = closest_match_download_link
+        else:
-            elif result == 1:
+            msg = "Could not contact download site"
-                msg = "Could not find KDK for host on Apple's servers, nor closest match"
-                print(f"- {msg}")
-                return False, msg, ""
-            elif result == 2:
-                msg = "Could not contact Apple download servers"
-                download_link = self.kdk_backup_site(closest_build)
-                if download_link is None:
-                    msg += " and could not find a backup copy online"
-                    print(f"- {msg}")
-                    return False, msg, ""
-            else:
-                msg = "Unknown error"
-                print(f"- {msg}")
-                return False, msg, ""
-        elif result == 2:
-            msg = "Could not contact Apple download servers"
-            download_link = self.kdk_backup_site(build)
-            if download_link is None:
-                msg += " and could not find a backup copy online"
-                print(f"- {msg}")
-                return False, msg, ""
-        elif result == 3:
-            msg = "Failed to connect to the internet"
             print(f"- {msg}")
             return False, msg, ""
 
-        if "github" in download_link:
+        result = utilities.download_file(download_link, self.constants.kdk_download_path)
-            result = utilities.download_file(download_link, self.constants.kdk_download_path)
-        else:
-            result = utilities.download_apple_developer_portal(download_link, self.constants.kdk_download_path)
 
         if result:
+            # TODO: should we use the checksum from the API?
             result = subprocess.run(["hdiutil", "verify", self.constants.kdk_download_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             if result.returncode != 0:
-                print(f"Error: Kernel Debug Kit checksum verification failed!")
+                print("Error: Kernel Debug Kit checksum verification failed!")
                 print(f"Output: {result.stderr}")
                 msg = "Kernel Debug Kit checksum verification failed, please try again.\n\nIf this continues to fail, ensure you're downloading on a stable network connection (ie. Ethernet)"
                 print(f"- {msg}")

resources/utilities.py

@@ -407,12 +407,15 @@ def download_file(link, location, is_gui=None, verify_checksum=False):
         dl = 0
         total_downloaded_string = ""
         global clear
+        checksum = hashlib.sha256() if verify_checksum else None
         with location.open("wb") as file:
             count = 0
             start = time.perf_counter()
             for chunk in response.iter_content(1024 * 1024 * 4):
                 dl += len(chunk)
                 file.write(chunk)
+                if checksum:
+                    checksum.update(chunk)
                 count += len(chunk)
                 if is_gui is None:
                     if clear:
@@ -425,19 +428,8 @@ def download_file(link, location, is_gui=None, verify_checksum=False):
                     total_downloaded_string = f" ({round(float(dl / total_file_size * 100), 2)}%)"
                 print(f"{round(count / 1024 / 1024, 2)}MB Downloaded{file_size_string}{total_downloaded_string}\nAverage Download Speed: {round(dl//(time.perf_counter() - start) / 100000 / 8, 2)} MB/s")
 
-        if verify_checksum is True:
-            # Verify checksum
-            # Note that this can be quite taxing on slower Macs
-            checksum = hashlib.sha256()
-            with location.open("rb") as file:
-                chunk = file.read(1024 * 1024 * 16)
-                while chunk:
-                    checksum.update(chunk)
-                    chunk = file.read(1024 * 1024 * 16)
-            enable_sleep_after_running()
-            return checksum
         enable_sleep_after_running()
-        return True
+        return checksum.hexdigest() if checksum else True
     else:
         cls()
         header = "# Could not establish Network Connection with provided link! #"
@@ -455,29 +447,6 @@ def download_file(link, location, is_gui=None, verify_checksum=False):
         return None
 
 
-def download_apple_developer_portal(link, location, is_gui=None, verify_checksum=False):
-    TOKEN_URL_BASE = "https://developerservices2.apple.com/services/download?path="
-    remote_path = urllib.parse.urlparse(link).path
-    token_url = urllib.parse.urlunparse(urllib.parse.urlparse(TOKEN_URL_BASE)._replace(query=urllib.parse.urlencode({"path": remote_path})))
-
-    try:
-        response = SESSION.get(token_url, timeout=5)
-    except (requests.exceptions.Timeout, requests.exceptions.TooManyRedirects, requests.exceptions.ConnectionError):
-        print(" - Could not contact Apple download servers")
-        return None
-
-    try:
-        response.raise_for_status()
-    except requests.exceptions.HTTPError:
-        if response.status_code == 400 and "The path specified is invalid" in response.text:
-            print(" - File does not exist on Apple download servers")
-        else:
-            print(" - Could not request download authorization from Apple download servers")
-        return None
-
-    return download_file(link, location, is_gui, verify_checksum)
-
-
 def dump_constants(constants):
     with open(os.path.join(os.path.expanduser('~'), 'Desktop', 'internal_data.txt'), 'w') as f:
         f.write(str(vars(constants)))