From 90092a296d40125428403bce467b60163f3d17c6 Mon Sep 17 00:00:00 2001 From: Mykola Grymalyuk Date: Thu, 1 Aug 2024 11:16:00 -0600 Subject: [PATCH] Implement getattrlist for improved CoW detection --- ci_tooling/build_modules/application.py | 3 +- ci_tooling/build_modules/shim.py | 5 +- .../support/kdk_handler.py | 3 +- .../support/macos_installer_handler.py | 17 +-- .../sys_patch/sys_patch.py | 15 +-- .../sys_patch/sys_patch_auto.py | 3 +- .../sys_patch/sys_patch_helpers.py | 3 +- opencore_legacy_patcher/volume/__init__.py | 46 ++++++++ opencore_legacy_patcher/volume/copy.py | 35 ++++++ opencore_legacy_patcher/volume/properties.py | 110 ++++++++++++++++++ .../wx_gui/gui_macos_installer_flash.py | 5 +- 11 files changed, 222 insertions(+), 23 deletions(-) create mode 100644 opencore_legacy_patcher/volume/__init__.py create mode 100644 opencore_legacy_patcher/volume/copy.py create mode 100644 opencore_legacy_patcher/volume/properties.py diff --git a/ci_tooling/build_modules/application.py b/ci_tooling/build_modules/application.py index e18197607..a7ef601b3 100644 --- a/ci_tooling/build_modules/application.py +++ b/ci_tooling/build_modules/application.py @@ -5,6 +5,7 @@ import subprocess from pathlib import Path +from opencore_legacy_patcher.volume import generate_copy_arguments from opencore_legacy_patcher.support import subprocess_wrapper @@ -157,7 +158,7 @@ class GenerateApplication: print("Embedding resources") for file in Path("payloads/Icon/AppIcons").glob("*.icns"): subprocess_wrapper.run_and_verify( - ["/bin/cp", str(file), self._application_output / "Contents" / "Resources/"], + generate_copy_arguments(str(file), self._application_output / "Contents" / "Resources/"), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) diff --git a/ci_tooling/build_modules/shim.py b/ci_tooling/build_modules/shim.py index cec6e07e1..0f05bef73 100644 --- a/ci_tooling/build_modules/shim.py +++ b/ci_tooling/build_modules/shim.py @@ -4,6 +4,7 @@ shim.py: Generate Update Shim from pathlib import Path +from opencore_legacy_patcher.volume import generate_copy_arguments from opencore_legacy_patcher.support import subprocess_wrapper @@ -25,9 +26,9 @@ class GenerateShim: if Path(self._shim_pkg).exists(): Path(self._shim_pkg).unlink() - subprocess_wrapper.run_and_verify(["/bin/cp", "-R", self._build_pkg, self._shim_pkg]) + subprocess_wrapper.run_and_verify(generate_copy_arguments(self._build_pkg, self._shim_pkg)) if Path(self._output_shim).exists(): Path(self._output_shim).unlink() - subprocess_wrapper.run_and_verify(["/bin/cp", "-R", self._shim_path, self._output_shim]) + subprocess_wrapper.run_and_verify(generate_copy_arguments(self._shim_path, self._output_shim)) diff --git a/opencore_legacy_patcher/support/kdk_handler.py b/opencore_legacy_patcher/support/kdk_handler.py index 4fe3526c2..bb904f5cc 100644 --- a/opencore_legacy_patcher/support/kdk_handler.py +++ b/opencore_legacy_patcher/support/kdk_handler.py @@ -15,6 +15,7 @@ from pathlib import Path from .. import constants from ..datasets import os_data +from ..volume import generate_copy_arguments from . import ( network_handler, @@ -667,7 +668,7 @@ class KernelDebugKitUtilities: logging.info("Backup already exists, skipping") return - result = subprocess_wrapper.run_as_root(["/bin/cp", "-R", kdk_path, kdk_dst_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + result = subprocess_wrapper.run_as_root(generate_copy_arguments(kdk_path, kdk_dst_path), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if result.returncode != 0: logging.info("Failed to create KDK backup:") subprocess_wrapper.log(result) \ No newline at end of file diff --git a/opencore_legacy_patcher/support/macos_installer_handler.py b/opencore_legacy_patcher/support/macos_installer_handler.py index 64b32c1dd..fc764c2d3 100644 --- a/opencore_legacy_patcher/support/macos_installer_handler.py +++ b/opencore_legacy_patcher/support/macos_installer_handler.py @@ -16,6 +16,11 @@ from . import ( subprocess_wrapper ) +from ..volume import ( + can_copy_on_write, + generate_copy_arguments +) + APPLICATION_SEARCH_PATH: str = "/Applications" SFR_SOFTWARE_UPDATE_PATH: str = "SFR/com_apple_MobileAsset_SFRSoftwareUpdate/com_apple_MobileAsset_SFRSoftwareUpdate.xml" @@ -90,13 +95,9 @@ class InstallerCreation(): for file in Path(ia_tmp).glob("*"): subprocess.run(["/bin/rm", "-rf", str(file)]) - # Copy installer to tmp (use CoW to avoid extra disk writes) - args = ["/bin/cp", "-cR", installer_path, ia_tmp] - if utilities.check_filesystem_type() != "apfs": - # HFS+ disks do not support CoW - args[1] = "-R" - - # Ensure we have enough space for the duplication + # Copy installer to tmp + if can_copy_on_write(installer_path, ia_tmp) is False: + # Ensure we have enough space for the duplication when CoW is not supported space_available = utilities.get_free_space() space_needed = Path(ia_tmp).stat().st_size if space_available < space_needed: @@ -104,7 +105,7 @@ class InstallerCreation(): logging.info(f"{utilities.human_fmt(space_available)} available, {utilities.human_fmt(space_needed)} required") return False - subprocess.run(args) + subprocess.run(generate_copy_arguments(installer_path, ia_tmp)) # Adjust installer_path to point to the copied installer installer_path = Path(ia_tmp) / Path(Path(installer_path).name) diff --git a/opencore_legacy_patcher/sys_patch/sys_patch.py b/opencore_legacy_patcher/sys_patch/sys_patch.py index 928d45157..b29267404 100644 --- a/opencore_legacy_patcher/sys_patch/sys_patch.py +++ b/opencore_legacy_patcher/sys_patch/sys_patch.py @@ -46,6 +46,7 @@ from datetime import datetime from .. import constants from ..datasets import os_data +from ..volume import generate_copy_arguments from ..support import ( utilities, @@ -137,7 +138,7 @@ class PatchSysVolume: if not mounted_system_version.exists(): logging.error("- Failed to find SystemVersion.plist on mounted root volume") return False - + try: mounted_data = plistlib.load(open(mounted_system_version, "rb")) if mounted_data["ProductBuildVersion"] != self.constants.detected_os_build: @@ -149,7 +150,7 @@ class PatchSysVolume: except: logging.error("- Failed to parse SystemVersion.plist") return False - + return True @@ -234,7 +235,7 @@ class PatchSysVolume: if save_hid_cs is True and cs_path.exists(): logging.info("- Backing up IOHIDEventDriver CodeSignature") # Note it's a folder, not a file - subprocess_wrapper.run_as_root(["/bin/cp", "-r", cs_path, f"{self.constants.payload_path}/IOHIDEventDriver_CodeSignature.bak"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + subprocess_wrapper.run_as_root(generate_copy_arguments(cs_path, f"{self.constants.payload_path}/IOHIDEventDriver_CodeSignature.bak"), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) logging.info(f"- Merging KDK with Root Volume: {kdk_path.name}") subprocess_wrapper.run_as_root( @@ -256,7 +257,7 @@ class PatchSysVolume: if not cs_path.exists(): logging.info(" - CodeSignature folder missing, creating") subprocess_wrapper.run_as_root(["/bin/mkdir", "-p", cs_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - subprocess_wrapper.run_as_root(["/bin/cp", "-r", f"{self.constants.payload_path}/IOHIDEventDriver_CodeSignature.bak", cs_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + subprocess_wrapper.run_as_root(generate_copy_arguments(f"{self.constants.payload_path}/IOHIDEventDriver_CodeSignature.bak", cs_path), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) subprocess_wrapper.run_as_root(["/bin/rm", "-rf", f"{self.constants.payload_path}/IOHIDEventDriver_CodeSignature.bak"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -533,7 +534,7 @@ class PatchSysVolume: logging.info("- Writing patchset information to Root Volume") if Path(destination_path_file).exists(): subprocess_wrapper.run_as_root_and_verify(["/bin/rm", destination_path_file], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - subprocess_wrapper.run_as_root_and_verify(["/bin/cp", f"{self.constants.payload_path}/{file_name}", destination_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + subprocess_wrapper.run_as_root_and_verify(generate_copy_arguments(f"{self.constants.payload_path}/{file_name}", destination_path), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) def _add_auxkc_support(self, install_file: str, source_folder_path: str, install_patch_directory: str, destination_folder_path: str) -> str: @@ -782,7 +783,7 @@ class PatchSysVolume: subprocess_wrapper.run_as_root_and_verify(["/bin/rm", "-R", f"{destination_folder}/{file_name}"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) else: logging.info(f" - Installing: {file_name}") - subprocess_wrapper.run_as_root_and_verify(["/bin/cp", "-R", f"{source_folder}/{file_name}", destination_folder], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + subprocess_wrapper.run_as_root_and_verify(generate_copy_arguments(f"{source_folder}/{file_name}", destination_folder), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) self._fix_permissions(destination_folder + "/" + file_name) else: # Assume it's an individual file, replace as normal @@ -791,7 +792,7 @@ class PatchSysVolume: subprocess_wrapper.run_as_root_and_verify(["/bin/rm", f"{destination_folder}/{file_name}"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) else: logging.info(f" - Installing: {file_name}") - subprocess_wrapper.run_as_root_and_verify(["/bin/cp", f"{source_folder}/{file_name}", destination_folder], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + subprocess_wrapper.run_as_root_and_verify(generate_copy_arguments(f"{source_folder}/{file_name}", destination_folder), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) self._fix_permissions(destination_folder + "/" + file_name) diff --git a/opencore_legacy_patcher/sys_patch/sys_patch_auto.py b/opencore_legacy_patcher/sys_patch/sys_patch_auto.py index ca6d0bab8..1141ad448 100644 --- a/opencore_legacy_patcher/sys_patch/sys_patch_auto.py +++ b/opencore_legacy_patcher/sys_patch/sys_patch_auto.py @@ -20,6 +20,7 @@ from . import sys_patch_detect from .. import constants from ..datasets import css_data +from ..volume import generate_copy_arguments from ..wx_gui import ( gui_entry, @@ -350,7 +351,7 @@ Please check the Github page for more information about this release.""" if not Path(services[service]).parent.exists(): logging.info(f" - Creating {Path(services[service]).parent} directory") subprocess_wrapper.run_as_root_and_verify(["/bin/mkdir", "-p", Path(services[service]).parent], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - subprocess_wrapper.run_as_root_and_verify(["/bin/cp", service, services[service]], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + subprocess_wrapper.run_as_root_and_verify(generate_copy_arguments(service, services[service]), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # Set the permissions on the service subprocess_wrapper.run_as_root_and_verify(["/bin/chmod", "644", services[service]], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) diff --git a/opencore_legacy_patcher/sys_patch/sys_patch_helpers.py b/opencore_legacy_patcher/sys_patch/sys_patch_helpers.py index aafcc7288..a8b4acc1c 100644 --- a/opencore_legacy_patcher/sys_patch/sys_patch_helpers.py +++ b/opencore_legacy_patcher/sys_patch/sys_patch_helpers.py @@ -14,6 +14,7 @@ from datetime import datetime from .. import constants from ..datasets import os_data +from ..volume import generate_copy_arguments from ..support import ( generate_smbios, @@ -232,6 +233,6 @@ class SysPatchHelpers: src_dir = f"{LIBRARY_DIR}/{file.name}" if not Path(f"{DEST_DIR}/lib").exists(): - subprocess_wrapper.run_as_root_and_verify(["/bin/cp", "-cR", f"{src_dir}/lib", f"{DEST_DIR}/"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + subprocess_wrapper.run_as_root_and_verify(generate_copy_arguments(f"{src_dir}/lib", f"{DEST_DIR}/"), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) break \ No newline at end of file diff --git a/opencore_legacy_patcher/volume/__init__.py b/opencore_legacy_patcher/volume/__init__.py new file mode 100644 index 000000000..249af9692 --- /dev/null +++ b/opencore_legacy_patcher/volume/__init__.py @@ -0,0 +1,46 @@ +""" +volume: Volume utilities for macOS + +------------------------------------------------------------------------------- + +Usage - Checking if Copy on Write is supported between source and destination: + +>>> from volume import can_copy_on_write + +>>> source = "/path/to/source" +>>> destination = "/path/to/destination" + +>>> can_copy_on_write(source, destination) +True + +------------------------------------------------------------------------------- + +Usage - Generating copy arguments: + +>>> from volume import generate_copy_arguments + +>>> source = "/path/to/source" +>>> destination = "/path/to/destination" + +>>> _command = generate_copy_arguments(source, destination) +>>> _command +['/bin/cp', '-c', '/path/to/source', '/path/to/destination'] + +------------------------------------------------------------------------------- + +Usage - Querying volume properties: + +>>> from volume import PathAttributes + +>>> path = "/path/to/file" +>>> obj = PathAttributes(path) + +>>> obj.mount_point() +"/" + +>>> obj.supports_clonefile() +True +""" + +from .properties import PathAttributes +from .copy import can_copy_on_write, generate_copy_arguments \ No newline at end of file diff --git a/opencore_legacy_patcher/volume/copy.py b/opencore_legacy_patcher/volume/copy.py new file mode 100644 index 000000000..eef1e3bcc --- /dev/null +++ b/opencore_legacy_patcher/volume/copy.py @@ -0,0 +1,35 @@ +""" +copy.py: Generate performant '/bin/cp' arguments for macOS +""" + +from pathlib import Path + +from .properties import PathAttributes + + +def can_copy_on_write(source: str, destination: str) -> bool: + """ + Check if Copy on Write is supported between source and destination + """ + source_obj = PathAttributes(source) + return source_obj.mount_point() == PathAttributes(str(Path(destination).parent)).mount_point() and source_obj.supports_clonefile() + + +def generate_copy_arguments(source: str, destination: str) -> list: + """ + Generate performant '/bin/cp' arguments for macOS + """ + _command = ["/bin/cp", source, destination] + if not Path(source).exists(): + raise FileNotFoundError(f"Source file not found: {source}") + if not Path(destination).parent.exists(): + raise FileNotFoundError(f"Destination directory not found: {destination}") + + # Check if Copy on Write is supported. + if can_copy_on_write(source, destination): + _command.insert(1, "-c") + + if Path(source).is_dir(): + _command.insert(1, "-R") + + return _command \ No newline at end of file diff --git a/opencore_legacy_patcher/volume/properties.py b/opencore_legacy_patcher/volume/properties.py new file mode 100644 index 000000000..3cc03fbad --- /dev/null +++ b/opencore_legacy_patcher/volume/properties.py @@ -0,0 +1,110 @@ +""" +properties.py: Query volume properties for a given path using macOS's getattrlist. +""" + +import ctypes + + +class attrreference_t(ctypes.Structure): + _fields_ = [ + ("attr_dataoffset", ctypes.c_int32), + ("attr_length", ctypes.c_uint32) + ] + +class attrlist_t(ctypes.Structure): + _fields_ = [ + ("bitmapcount", ctypes.c_ushort), + ("reserved", ctypes.c_uint16), + ("commonattr", ctypes.c_uint), + ("volattr", ctypes.c_uint), + ("dirattr", ctypes.c_uint), + ("fileattr", ctypes.c_uint), + ("forkattr", ctypes.c_uint) + ] + +class volattrbuf(ctypes.Structure): + _fields_ = [ + ("length", ctypes.c_uint32), + ("mountPoint", attrreference_t), + ("volCapabilities", ctypes.c_uint64), + ("mountPointSpace", ctypes.c_char * 1024), + ] + + +class PathAttributes: + + def __init__(self, path: str) -> None: + self._path = path + if not isinstance(self._path, str): + try: + self._path = str(self._path) + except: + raise ValueError(f"Invalid path: {path}") + + _libc = ctypes.CDLL("/usr/lib/libc.dylib") + + # Reference: + # https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/getattrlist.2.html + try: + self._getattrlist = _libc.getattrlist + except AttributeError: + return + + self._getattrlist.argtypes = [ + ctypes.c_char_p, # Path + ctypes.POINTER(attrlist_t), # Attribute list + ctypes.c_void_p, # Attribute buffer + ctypes.c_ulong, # Attribute buffer size + ctypes.c_ulong # Options + ] + self._getattrlist.restype = ctypes.c_int + + # Reference: + # https://github.com/apple-oss-distributions/xnu/blob/xnu-10063.121.3/bsd/sys/attr.h + ATTR_BIT_MAP_COUNT = 0x00000005 + ATTR_VOL_MOUNTPOINT = 0x00001000 + ATTR_VOL_CAPABILITIES = 0x00020000 + + attrList = attrlist_t() + attrList.bitmapcount = ATTR_BIT_MAP_COUNT + attrList.volattr = ATTR_VOL_MOUNTPOINT | ATTR_VOL_CAPABILITIES + + volAttrBuf = volattrbuf() + + if self._getattrlist(self._path.encode(), ctypes.byref(attrList), ctypes.byref(volAttrBuf), ctypes.sizeof(volAttrBuf), 0) != 0: + return + + self._volAttrBuf = volAttrBuf + + + def supports_clonefile(self) -> bool: + """ + Verify if path provided supports Apple's clonefile function. + + Equivalent to checking for Copy on Write support. + """ + VOL_CAP_INT_CLONE = 0x00010000 + + if not hasattr(self, "_volAttrBuf"): + return False + + if self._volAttrBuf.volCapabilities & VOL_CAP_INT_CLONE: + return True + + return False + + + def mount_point(self) -> str: + """ + Return mount point of path. + """ + + if not hasattr(self, "_volAttrBuf"): + return "" + + mount_point_ptr = ctypes.cast( + ctypes.addressof(self._volAttrBuf.mountPoint) + self._volAttrBuf.mountPoint.attr_dataoffset, + ctypes.POINTER(ctypes.c_char * self._volAttrBuf.mountPoint.attr_length) + ) + + return mount_point_ptr.contents.value.decode() \ No newline at end of file diff --git a/opencore_legacy_patcher/wx_gui/gui_macos_installer_flash.py b/opencore_legacy_patcher/wx_gui/gui_macos_installer_flash.py index 6dd448326..b3fe856fc 100644 --- a/opencore_legacy_patcher/wx_gui/gui_macos_installer_flash.py +++ b/opencore_legacy_patcher/wx_gui/gui_macos_installer_flash.py @@ -15,6 +15,7 @@ from pathlib import Path from .. import constants from ..datasets import os_data +from ..volume import generate_copy_arguments from ..wx_gui import ( gui_main_menu, @@ -460,7 +461,7 @@ class macOSInstallerFlashFrame(wx.Frame): return subprocess.run(["/bin/mkdir", "-p", f"{path}/Library/Packages/"]) - subprocess.run(["/bin/cp", "-r", self.constants.installer_pkg_path, f"{path}/Library/Packages/"]) + subprocess.run(generate_copy_arguments(self.constants.installer_pkg_path, f"{path}/Library/Packages/")) self._kdk_chainload(os_version["ProductBuildVersion"], os_version["ProductVersion"], Path(path + "/Library/Packages/")) @@ -530,7 +531,7 @@ class macOSInstallerFlashFrame(wx.Frame): return logging.info("Copying KDK") - subprocess.run(["/bin/cp", "-r", f"{mount_point}/KernelDebugKit.pkg", kdk_pkg_path]) + subprocess.run(generate_copy_arguments(f"{mount_point}/KernelDebugKit.pkg", kdk_pkg_path)) logging.info("Unmounting KDK") result = subprocess.run(["/usr/bin/hdiutil", "detach", mount_point], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)