Merge pull request #1047 from dortania/post-analytics

Implement back-end for Crash and Analytics Reporting

.github/workflows/build-app-wxpython.yml

@@ -18,10 +18,12 @@ jobs:
       commitdate: ${{ github.event.head_commit.timestamp }}${{ github.event.release.published_at }}
       MAC_NOTARIZATION_USERNAME: ${{ secrets.MAC_NOTARIZATION_USERNAME }}
       MAC_NOTARIZATION_PASSWORD: ${{ secrets.MAC_NOTARIZATION_PASSWORD }}
+      ANALYTICS_KEY: ${{ secrets.ANALYTICS_KEY }}
+      ANALYTICS_SITE: ${{ secrets.ANALYTICS_SITE }}
 
     steps:
       - uses: actions/checkout@v3
-      - run: /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 Build-Binary.command --reset_binaries --branch "${{ env.branch }}" --commit "${{ env.commiturl }}" --commit_date "${{ env.commitdate }}"
+      - run: /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 Build-Binary.command --reset_binaries --branch "${{ env.branch }}" --commit "${{ env.commiturl }}" --commit_date "${{ env.commitdate }}" --key "${{ env.ANALYTICS_KEY }}" --site "${{ env.ANALYTICS_SITE }}"
       - run: 'codesign -s "Developer ID Application: Mykola Grymalyuk (S74BDJXQMD)" -v --force --deep --timestamp --entitlements ./payloads/entitlements.plist -o runtime "dist/OpenCore-Patcher.app"'
       - run: cd dist; ditto -c -k --sequesterRsrc --keepParent OpenCore-Patcher.app ../OpenCore-Patcher-wxPython.app.zip
       - run: xcrun altool --notarize-app --primary-bundle-id "com.dortania.opencore-legacy-patcher" --username "${{ env.MAC_NOTARIZATION_USERNAME }}" --password "${{ env.MAC_NOTARIZATION_PASSWORD }}" --file OpenCore-Patcher-wxPython.app.zip

Build-Binary.command

@@ -61,6 +61,8 @@ class CreateBinary:
         parser.add_argument('--commit', type=str, help='Git commit URL')
         parser.add_argument('--commit_date', type=str, help='Git commit date')
         parser.add_argument('--reset_binaries', action='store_true', help='Force redownload and imaging of payloads')
+        parser.add_argument('--key', type=str, help='Developer key for signing')
+        parser.add_argument('--site', type=str, help='Path to server')
         args = parser.parse_args()
         return args
 
@@ -132,17 +134,85 @@ class CreateBinary:
                 print(rm_output.stderr.decode('utf-8'))
                 raise Exception("Remove failed")
 
+        self._embed_key()
 
         print("- Building GUI binary...")
         build_args = [self.pyinstaller_path, "./OpenCore-Patcher-GUI.spec", "--noconfirm"]
 
         build_result = subprocess.run(build_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        self._strip_key()
+
         if build_result.returncode != 0:
             print("- Build failed")
             print(build_result.stderr.decode('utf-8'))
             raise Exception("Build failed")
 
 
+
+
+    def _embed_key(self):
+        """
+        Embed developer key into binary
+        """
+
+        if not self.args.key:
+            print("- No developer key provided, skipping...")
+            return
+        if not self.args.site:
+            print("- No site provided, skipping...")
+            return
+
+        print("- Embedding developer key...")
+        if not Path("./resources/analytics_handler.py").exists():
+            print("- analytics_handler.py not found")
+            return
+
+        lines = []
+        with open("./resources/analytics_handler.py", "r") as f:
+            lines = f.readlines()
+
+        for i, line in enumerate(lines):
+            if line.startswith("SITE_KEY:         str = "):
+                lines[i] = f"SITE_KEY:         str = \"{self.args.key}\"\n"
+            elif line.startswith("ANALYTICS_SERVER: str = "):
+                lines[i] = f"ANALYTICS_SERVER: str = \"{self.args.site}\"\n"
+
+        with open("./resources/analytics_handler.py", "w") as f:
+            f.writelines(lines)
+
+
+    def _strip_key(self):
+        """
+        Strip developer key from binary
+        """
+
+        if not self.args.key:
+            print("- No developer key provided, skipping...")
+            return
+        if not self.args.site:
+            print("- No site provided, skipping...")
+            return
+
+        print("- Stripping developer key...")
+        if not Path("./resources/analytics_handler.py").exists():
+            print("- analytics_handler.py not found")
+            return
+
+        lines = []
+        with open("./resources/analytics_handler.py", "r") as f:
+            lines = f.readlines()
+
+        for i, line in enumerate(lines):
+            if line.startswith("SITE_KEY:         str = "):
+                lines[i] = f"SITE_KEY:         str = \"\"\n"
+            elif line.startswith("ANALYTICS_SERVER: str = "):
+                lines[i] = f"ANALYTICS_SERVER: str = \"\"\n"
+
+        with open("./resources/analytics_handler.py", "w") as f:
+            f.writelines(lines)
+
+
     def _delete_extra_binaries(self):
         """
         Delete extra binaries from payloads directory

CHANGELOG.md

@@ -1,6 +1,10 @@
 # OpenCore Legacy Patcher changelog
 
 ## 0.6.4
+- Backend changes:
+  - Implement new analytics_handler.py module
+    - Adds support for anonymous analytics including host info (and crash reports in the future)
+    - Can be disabled via GUI or `defaults write com.dortania.opencore-legacy-patcher DisableCrashAndAnalyticsReporting -bool true`
 
 ## 0.6.3
 - Update non-Metal Binaries:

PRIVACY.md

@@ -0,0 +1,25 @@
+# Privacy Policy
+
+OpenCore Legacy Patcher may collect pseudo-anonymized data about the host system and the OpenCore Legacy Patcher application. This data is used to improve the project and to help diagnose issues. The data collected is as follows:
+
+* System's UUID as a SHA1 hash
+  * This is used to identify the system and to prevent duplicate reports
+  * Cannot be used to identify the system without the user providing the UUID
+* Application name and version
+* System's OS version
+* System's model name, GPUs present and firmware vendor
+  * May include more hardware information in the future (ex. CPU, WiFi, etc)
+* General country code of system's reported region
+  * ex. `US`, `CA`, etc
+
+Identifiable data such as IP addresses, MAC addresses, serial numbers, etc. are not collected.
+
+In the future, crash logs may also be collected to help with diagnosing issues.
+----------
+
+Users who wish to opt-out can do so either via the application's preferences or via the following command:
+```
+defaults write com.dortania.opencore-legacy-patcher DisableCrashAndAnalyticsReporting -bool true
+```
+
+To have your data removed, please contact us via our [Discord server](https://discord.gg/rqdPgH8xSN) and provide the UUID of your system.

resources/analytics_handler.py

@@ -0,0 +1,97 @@
+import datetime
+import plistlib
+from pathlib import Path
+import json
+
+from resources import network_handler, constants, global_settings
+
+
+DATE_FORMAT:      str = "%Y-%m-%d %H-%M-%S"
+ANALYTICS_SERVER: str = ""
+SITE_KEY:         str = ""
+
+VALID_ENTRIES: dict = {
+    'KEY':                 str,               # Prevent abuse (embedded at compile time)
+    'UNIQUE_IDENTITY':     str,               # Host's UUID as SHA1 hash
+    'APPLICATION_NAME':    str,               # ex. OpenCore Legacy Patcher
+    'APPLICATION_VERSION': str,               # ex. 0.2.0
+    'OS_VERSION':          str,               # ex. 10.15.7
+    'MODEL':               str,               # ex. MacBookPro11,5
+    'GPUS':                list,              # ex. ['Intel Iris Pro', 'AMD Radeon R9 M370X']
+    'FIRMWARE':            str,               # ex. APPLE
+    'LOCATION':            str,               # ex. 'US' (just broad region, don't need to be specific)
+    'TIMESTAMP':           datetime.datetime, # ex. 2021-09-01-12-00-00
+}
+
+
+class Analytics:
+
+    def __init__(self, global_constants: constants.Constants) -> None:
+        self.constants: constants.Constants = global_constants
+
+        if global_settings.GlobalEnviromentSettings().read_property("DisableCrashAndAnalyticsReporting") is True:
+            return
+
+        self._generate_base_data()
+        self._post_data()
+
+
+    def _get_country(self) -> str:
+        # Get approximate country from .GlobalPreferences.plist
+        path = "/Library/Preferences/.GlobalPreferences.plist"
+        if not Path(path).exists():
+            return "US"
+
+        try:
+            result = plistlib.load(Path(path).open("rb"))
+        except:
+            return "US"
+
+        if "Country" not in result:
+            return "US"
+
+        return result["Country"]
+
+
+    def _generate_base_data(self) -> None:
+
+        self.unique_identity = str(self.constants.computer.uuid_sha1)
+        self.application = str("OpenCore Legacy Patcher")
+        self.version = str(self.constants.patcher_version)
+        self.os = str( self.constants.detected_os_version)
+        self.model = str(self.constants.computer.real_model)
+        self.gpus = []
+
+        self.firmware = str(self.constants.computer.firmware_vendor)
+        self.location = str(self._get_country())
+
+        for gpu in self.constants.computer.gpus:
+            self.gpus.append(str(gpu.arch))
+
+        self.data = {
+            'KEY':                 SITE_KEY,
+            'UNIQUE_IDENTITY':     self.unique_identity,
+            'APPLICATION_NAME':    self.application,
+            'APPLICATION_VERSION': self.version,
+            'OS_VERSION':          self.os,
+            'MODEL':               self.model,
+            'GPUS':                self.gpus,
+            'FIRMWARE':            self.firmware,
+            'LOCATION':            self.location,
+            'TIMESTAMP':           str(datetime.datetime.now().strftime(DATE_FORMAT)),
+        }
+
+        # convert to JSON:
+        self.data = json.dumps(self.data)
+
+
+    def _post_data(self) -> None:
+        # Post data to analytics server
+        if ANALYTICS_SERVER == "":
+            return
+        if SITE_KEY == "":
+            return
+        network_handler.NetworkUtilities().post(ANALYTICS_SERVER, json = self.data)
+
+
+

resources/device_probe.py

@@ -6,6 +6,7 @@ import enum
 import itertools
 import subprocess
 import plistlib
+import hashlib
 from pathlib import Path
 from dataclasses import dataclass, field
 from typing import Any, ClassVar, Optional, Type, Union
@@ -491,6 +492,7 @@ class Computer:
     reported_model: Optional[str] = None
     reported_board_id: Optional[str] = None
     build_model: Optional[str] = None
+    uuid_sha1: Optional[str] = None
     gpus: list[GPU] = field(default_factory=list)
     igpu: Optional[GPU] = None  # Shortcut for IGPU
     dgpu: Optional[GPU] = None  # Shortcut for GFX0
@@ -719,6 +721,8 @@ class Computer:
         else:
             board = "board-id"
         self.reported_board_id = ioreg.corefoundation_to_native(ioreg.IORegistryEntryCreateCFProperty(entry, board, ioreg.kCFAllocatorDefault, ioreg.kNilOptions)).strip(b"\0").decode()  # type: ignore
+        self.uuid_sha1 = ioreg.corefoundation_to_native(ioreg.IORegistryEntryCreateCFProperty(entry, "IOPlatformUUID", ioreg.kCFAllocatorDefault, ioreg.kNilOptions))  # type: ignore
+        self.uuid_sha1 = hashlib.sha1(self.uuid_sha1.encode()).hexdigest()
         ioreg.IOObjectRelease(entry)
 
         # Real model

resources/gui/gui_main.py

@@ -2980,12 +2980,23 @@ class wx_python_gui:
             self.delete_unused_kdks_checkbox.GetPosition().y + self.delete_unused_kdks_checkbox.GetSize().height))
         self.set_ignore_app_updates_checkbox.SetToolTip(wx.ToolTip("This will set whether OpenCore will ignore App Updates on launch.\nEnable this option if you do not want to be prompted for App Updates"))
 
+        # Set Disable Analytics
+        res = global_settings.GlobalEnviromentSettings().read_property("DisableCrashAndAnalyticsReporting")
+        res = False if res is None else res
+        self.set_disable_analytics_checkbox = wx.CheckBox(self.frame_modal, label="Disable Crash/Analytics")
+        self.set_disable_analytics_checkbox.SetValue(res)
+        self.set_disable_analytics_checkbox.Bind(wx.EVT_CHECKBOX, self.set_disable_analytics_click)
+        self.set_disable_analytics_checkbox.SetPosition(wx.Point(
+            self.set_ignore_app_updates_checkbox.GetPosition().x,
+            self.set_ignore_app_updates_checkbox.GetPosition().y + self.set_ignore_app_updates_checkbox.GetSize().height))
+        self.set_disable_analytics_checkbox.SetToolTip(wx.ToolTip("Sets whether anonymized analytics are sent to the Dortania team.\nThis is used to help improve the application and is completely optional."))
+
         # Button: Developer Debug Info
         self.debug_button = wx.Button(self.frame_modal, label="Developer Debug Info")
         self.debug_button.Bind(wx.EVT_BUTTON, self.additional_info_menu)
         self.debug_button.SetPosition(wx.Point(
-            self.set_ignore_app_updates_checkbox.GetPosition().x,
-            self.set_ignore_app_updates_checkbox.GetPosition().y + self.set_ignore_app_updates_checkbox.GetSize().height + 5))
+            self.set_disable_analytics_checkbox.GetPosition().x,
+            self.set_disable_analytics_checkbox.GetPosition().y + self.set_disable_analytics_checkbox.GetSize().height + 5))
         self.debug_button.Center(wx.HORIZONTAL)
 
         # Button: return to main menu
@@ -3038,6 +3049,9 @@ class wx_python_gui:
         else:
             global_settings.GlobalEnviromentSettings().write_property("IgnoreAppUpdates", False)
 
+    def set_disable_analytics_click(self, event):
+        global_settings.GlobalEnviromentSettings().write_property("DisableCrashAndAnalyticsReporting", self.set_disable_analytics_checkbox.GetValue())
+
     def firewire_click(self, event=None):
         if self.firewire_boot_checkbox.GetValue():
             logging.info("Firewire Enabled")

resources/main.py

@@ -16,7 +16,8 @@ from resources import (
     arguments,
     reroute_payloads,
     commit_info,
-    logging_handler
+    logging_handler,
+    analytics_handler,
 )
 
 
@@ -89,6 +90,7 @@ class OpenCoreLegacyPatcher:
 
         # Generate defaults
         defaults.GenerateDefaults(self.computer.real_model, True, self.constants)
+        threading.Thread(target=analytics_handler.Analytics, args=(self.constants,)).start()
 
         if utilities.check_cli_args() is None:
             logging.info(f"- No arguments present, loading {'GUI' if self.constants.wxpython_variant is True else 'TUI'} mode")

resources/network_handler.py

@@ -109,6 +109,35 @@ class NetworkUtilities:
 
         return result
 
+    def post(self, url: str, **kwargs) -> requests.Response:
+        """
+        Wrapper for requests's post method
+        Implement additional error handling
+
+        Parameters:
+            url (str): URL to post
+            **kwargs: Additional parameters for requests.post
+
+        Returns:
+            requests.Response: Response object from requests.post
+        """
+
+        result: requests.Response = None
+
+        try:
+            result = SESSION.post(url, **kwargs)
+        except (
+            requests.exceptions.Timeout,
+            requests.exceptions.TooManyRedirects,
+            requests.exceptions.ConnectionError,
+            requests.exceptions.HTTPError
+        ) as error:
+            logging.warn(f"Error calling requests.post: {error}")
+            # Return empty response object
+            return requests.Response()
+
+        return result
+
 
 class DownloadObject:
     """