Repository: corpnewt/gibMacOS Branch: master Commit: 7e3d1b75f6e1 Files: 16 Total size: 225.5 KB Directory structure: gitextract_4n5ry1cq/ ├── BuildmacOSInstallApp.command ├── BuildmacOSInstallApp.py ├── LICENSE ├── MakeInstall.bat ├── MakeInstall.py ├── Readme.md ├── Scripts/ │ ├── __init__.py │ ├── disk.py │ ├── diskwin.py │ ├── downloader.py │ ├── plist.py │ ├── run.py │ └── utils.py ├── gibMacOS.bat ├── gibMacOS.command └── gibMacOS.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: BuildmacOSInstallApp.command ================================================ #!/usr/bin/env bash # Get the curent directory, the script name # and the script name with "py" substituted for the extension. args=( "$@" ) dir="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" script="${0##*/}" target="${script%.*}.py" # use_py3: # TRUE = Use if found, use py2 otherwise # FALSE = Use py2 # FORCE = Use py3 use_py3="TRUE" # We'll parse if the first argument passed is # --install-python and if so, we'll just install # Can optionally take a version number as the # second arg - i.e. --install-python 3.13.1 just_installing="FALSE" tempdir="" compare_to_version () { # Compares our OS version to the passed OS version, and # return a 1 if we match the passed compare type, or a 0 if we don't. # $1 = 0 (equal), 1 (greater), 2 (less), 3 (gequal), 4 (lequal) # $2 = OS version to compare ours to if [ -z "$1" ] || [ -z "$2" ]; then # Missing info - bail. return fi local current_os= comp= current_os="$(sw_vers -productVersion 2>/dev/null)" comp="$(vercomp "$current_os" "$2")" # Check gequal and lequal first if [[ "$1" == "3" && ("$comp" == "1" || "$comp" == "0") ]] || [[ "$1" == "4" && ("$comp" == "2" || "$comp" == "0") ]] || [[ "$comp" == "$1" ]]; then # Matched echo "1" else # No match echo "0" fi } set_use_py3_if () { # Auto sets the "use_py3" variable based on # conditions passed # $1 = 0 (equal), 1 (greater), 2 (less), 3 (gequal), 4 (lequal) # $2 = OS version to compare # $3 = TRUE/FALSE/FORCE in case of match if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then # Missing vars - bail with no changes. return fi if [ "$(compare_to_version "$1" "$2")" == "1" ]; then use_py3="$3" fi } get_remote_py_version () { local pyurl= py_html= py_vers= py_num="3" pyurl="https://www.python.org/downloads/macos/" py_html="$(curl -L $pyurl --compressed 2>&1)" if [ -z "$use_py3" ]; then use_py3="TRUE" fi if [ "$use_py3" == "FALSE" ]; then py_num="2" fi py_vers="$(echo "$py_html" | grep -i "Latest Python $py_num Release" | awk '{print $8}' | cut -d'<' -f1)" echo "$py_vers" } download_py () { local vers="$1" url= clear echo " ### ###" echo " # Downloading Python #" echo "### ###" echo if [ -z "$vers" ]; then echo "Gathering latest version..." vers="$(get_remote_py_version)" if [ -z "$vers" ]; then if [ "$just_installing" == "TRUE" ]; then echo " - Failed to get info!" exit 1 else # Didn't get it still - bail print_error fi fi echo "Located Version: $vers" else # Got a version passed echo "User-Provided Version: $vers" fi echo "Building download url..." url="$(\ curl -L https://www.python.org/downloads/release/python-${vers//./}/ --compressed 2>&1 | \ grep -iE "python-$vers-macos.*.pkg\"" | \ grep -iE "a href=" | \ awk -F'"' '{ print $2 }' | \ head -n 1\ )" if [ -z "$url" ]; then if [ "$just_installing" == "TRUE" ]; then echo " - Failed to build download url!" exit 1 else # Couldn't get the URL - bail print_error fi fi echo " - $url" echo "Downloading..." # Create a temp dir and download to it tempdir="$(mktemp -d 2>/dev/null || mktemp -d -t 'tempdir')" curl "$url" -o "$tempdir/python.pkg" if [ "$?" != "0" ]; then echo " - Failed to download python installer!" exit $? fi echo echo "Running python install package..." echo sudo installer -pkg "$tempdir/python.pkg" -target / if [ "$?" != "0" ]; then echo " - Failed to install python!" exit $? fi echo # Now we expand the package and look for a shell update script pkgutil --expand "$tempdir/python.pkg" "$tempdir/python" if [ -e "$tempdir/python/Python_Shell_Profile_Updater.pkg/Scripts/postinstall" ]; then # Run the script echo "Updating PATH..." echo "$tempdir/python/Python_Shell_Profile_Updater.pkg/Scripts/postinstall" echo fi vers_folder="Python $(echo "$vers" | cut -d'.' -f1 -f2)" if [ -f "/Applications/$vers_folder/Install Certificates.command" ]; then # Certs script exists - let's execute that to make sure our certificates are updated echo "Updating Certificates..." echo "/Applications/$vers_folder/Install Certificates.command" echo fi echo "Cleaning up..." cleanup if [ "$just_installing" == "TRUE" ]; then echo echo "Done." else # Now we check for py again downloaded="TRUE" clear main fi } cleanup () { if [ -d "$tempdir" ]; then rm -Rf "$tempdir" fi } print_error() { clear cleanup echo " ### ###" echo " # Python Not Found #" echo "### ###" echo echo "Python is not installed or not found in your PATH var." echo if [ "$kernel" == "Darwin" ]; then echo "Please go to https://www.python.org/downloads/macos/ to" echo "download and install the latest version, then try again." else echo "Please install python through your package manager and" echo "try again." fi echo exit 1 } print_target_missing() { clear cleanup echo " ### ###" echo " # Target Not Found #" echo "### ###" echo echo "Could not locate $target!" echo exit 1 } format_version () { local vers="$1" echo "$(echo "$1" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }')" } vercomp () { # Modified from: https://apple.stackexchange.com/a/123408/11374 local ver1="$(format_version "$1")" ver2="$(format_version "$2")" if [ $ver1 -gt $ver2 ]; then echo "1" elif [ $ver1 -lt $ver2 ]; then echo "2" else echo "0" fi } get_local_python_version() { # $1 = Python bin name (defaults to python3) # Echoes the path to the highest version of the passed python bin if any local py_name="$1" max_version= python= python_version= python_path= if [ -z "$py_name" ]; then py_name="python3" fi py_list="$(which -a "$py_name" 2>/dev/null)" # Walk that newline separated list while read python; do if [ -z "$python" ]; then # Got a blank line - skip continue fi if [ "$check_py3_stub" == "1" ] && [ "$python" == "/usr/bin/python3" ]; then # See if we have a valid developer path xcode-select -p > /dev/null 2>&1 if [ "$?" != "0" ]; then # /usr/bin/python3 path - but no valid developer dir continue fi fi python_version="$(get_python_version $python)" if [ -z "$python_version" ]; then # Didn't find a py version - skip continue fi # Got the py version - compare to our max if [ -z "$max_version" ] || [ "$(vercomp "$python_version" "$max_version")" == "1" ]; then # Max not set, or less than the current - update it max_version="$python_version" python_path="$python" fi done <<< "$py_list" echo "$python_path" } get_python_version() { local py_path="$1" py_version= # Get the python version by piping stderr into stdout (for py2), then grepping the output for # the word "python", getting the second element, and grepping for an alphanumeric version number py_version="$($py_path -V 2>&1 | grep -i python | cut -d' ' -f2 | grep -E "[A-Za-z\d\.]+")" if [ ! -z "$py_version" ]; then echo "$py_version" fi } prompt_and_download() { if [ "$downloaded" != "FALSE" ] || [ "$kernel" != "Darwin" ]; then # We already tried to download, or we're not on macOS - just bail print_error fi clear echo " ### ###" echo " # Python Not Found #" echo "### ###" echo target_py="Python 3" printed_py="Python 2 or 3" if [ "$use_py3" == "FORCE" ]; then printed_py="Python 3" elif [ "$use_py3" == "FALSE" ]; then target_py="Python 2" printed_py="Python 2" fi echo "Could not locate $printed_py!" echo echo "This script requires $printed_py to run." echo while true; do read -p "Would you like to install the latest $target_py now? (y/n): " yn case $yn in [Yy]* ) download_py;break;; [Nn]* ) print_error;; esac done } main() { local python= version= # Verify our target exists if [ ! -f "$dir/$target" ]; then # Doesn't exist print_target_missing fi if [ -z "$use_py3" ]; then use_py3="TRUE" fi if [ "$use_py3" != "FALSE" ]; then # Check for py3 first python="$(get_local_python_version python3)" fi if [ "$use_py3" != "FORCE" ] && [ -z "$python" ]; then # We aren't using py3 explicitly, and we don't already have a path python="$(get_local_python_version python2)" if [ -z "$python" ]; then # Try just looking for "python" python="$(get_local_python_version python)" fi fi if [ -z "$python" ]; then # Didn't ever find it - prompt prompt_and_download return 1 fi # Found it - start our script and pass all args "$python" "$dir/$target" "${args[@]}" } # Keep track of whether or not we're on macOS to determine if # we can download and install python for the user as needed. kernel="$(uname -s)" # Check to see if we need to force based on # macOS version. 10.15 has a dummy python3 version # that can trip up some py3 detection in other scripts. # set_use_py3_if "3" "10.15" "FORCE" downloaded="FALSE" # Check for the aforementioned /usr/bin/python3 stub if # our OS version is 10.15 or greater. check_py3_stub="$(compare_to_version "3" "10.15")" trap cleanup EXIT if [ "$1" == "--install-python" ] && [ "$kernel" == "Darwin" ]; then just_installing="TRUE" download_py "$2" else main fi ================================================ FILE: BuildmacOSInstallApp.py ================================================ #!/usr/bin/env python from Scripts import * import os, datetime, shutil, time, sys, argparse # Using the techniques outlined by wolfmannight here: https://www.insanelymac.com/forum/topic/338810-create-legit-copy-of-macos-from-apple-catalog/ class buildMacOSInstallApp: def __init__(self): self.r = run.Run() self.u = utils.Utils("Build macOS Install App") self.target_files = [ "BaseSystem.dmg", "BaseSystem.chunklist", "InstallESDDmg.pkg", "InstallInfo.plist", "AppleDiagnostics.dmg", "AppleDiagnostics.chunklist" ] # Verify we're on macOS - this doesn't work anywhere else if not sys.platform == "darwin": self.u.head("WARNING") print("") print("This script only runs on macOS!") print("") exit(1) def mount_dmg(self, dmg, no_browse = False): # Mounts the passed dmg and returns the mount point(s) args = ["/usr/bin/hdiutil", "attach", dmg, "-plist", "-noverify"] if no_browse: args.append("-nobrowse") out = self.r.run({"args":args}) if out[2] != 0: # Failed! raise Exception("Mount Failed!", "{} failed to mount:\n\n{}".format(os.path.basename(dmg), out[1])) # Get the plist data returned, and locate the mount points try: plist_data = plist.loads(out[0]) mounts = [x["mount-point"] for x in plist_data.get("system-entities", []) if "mount-point" in x] return mounts except: raise Exception("Mount Failed!", "No mount points returned from {}".format(os.path.basename(dmg))) def unmount_dmg(self, mount_point): # Unmounts the passed dmg or mount point - retries with force if failed # Can take either a single point or a list if not type(mount_point) is list: mount_point = [mount_point] unmounted = [] for m in mount_point: args = ["/usr/bin/hdiutil", "detach", m] out = self.r.run({"args":args}) if out[2] != 0: # Polite failed, let's crush this b! args.append("-force") out = self.r.run({"args":args}) if out[2] != 0: # Oh... failed again... onto the next... print(out[1]) continue unmounted.append(m) return unmounted def main(self): while True: self.u.head() print("") print("Q. Quit") print("") fold = self.u.grab("Please drag and drop the output folder from gibMacOS here: ") print("") if fold.lower() == "q": self.u.custom_quit() f_path = self.u.check_path(fold) if not f_path: print("That path does not exist!\n") self.u.grab("Press [enter] to return...") continue # Let's check if it's a folder. If not, make the next directory up the target if not os.path.isdir(f_path): f_path = os.path.dirname(os.path.realpath(f_path)) # Walk the contents of f_path and ensure we have all the needed files lower_contents = [y.lower() for y in os.listdir(f_path)] # Check if we got an InstallAssistant.pkg - and if so, just open that if "installassistant.pkg" in lower_contents: self.u.head("InstallAssistant.pkg Found") print("") print("Located InstallAssistant.pkg in the passed folder.\n") print("As of macOS Big Sur (11.x), Apple changed how they distribute the OS files in") print("the software update catalog.\n") print("Double clicking the InstallAssistant.pkg will open it in Installer, which will") print("copy the Install macOS [version].app to your /Applications folder.\n") print("Opening InstallAssistant.pkg...") self.r.run({"args":["open",os.path.join(f_path,"InstallAssistant.pkg")]}) print("") self.u.grab("Press [enter] to return...") continue missing_list = [x for x in self.target_files if not x.lower() in lower_contents] if len(missing_list): self.u.head("Missing Required Files") print("") print("That folder is missing the following required files:") print(", ".join(missing_list)) print("") self.u.grab("Press [enter] to return...") # Time to build the installer! cwd = os.getcwd() os.chdir(f_path) base_mounts = [] try: self.u.head("Building Installer") print("") print("Taking ownership of downloaded files...") for x in self.target_files: print(" - {}...".format(x)) self.r.run({"args":["chmod","a+x",x]}) print("Mounting BaseSystem.dmg...") base_mounts = self.mount_dmg("BaseSystem.dmg") if not len(base_mounts): raise Exception("Mount Failed!", "No mount points were returned from BaseSystem.dmg") base_mount = base_mounts[0] # Let's assume the first print("Locating Installer app...") install_app = next((x for x in os.listdir(base_mount) if os.path.isdir(os.path.join(base_mount,x)) and x.lower().endswith(".app") and not x.startswith(".")),None) if not install_app: raise Exception("Installer app not located in {}".format(base_mount)) print(" - Found {}".format(install_app)) # Copy the .app over out = self.r.run({"args":["cp","-R",os.path.join(base_mount,install_app),os.path.join(f_path,install_app)]}) if out[2] != 0: raise Exception("Copy Failed!", out[1]) print("Unmounting BaseSystem.dmg...") for x in base_mounts: self.unmount_dmg(x) base_mounts = [] shared_support = os.path.join(f_path,install_app,"Contents","SharedSupport") if not os.path.exists(shared_support): print("Creating SharedSupport directory...") os.makedirs(shared_support) print("Copying files to SharedSupport...") for x in self.target_files: y = "InstallESD.dmg" if x.lower() == "installesddmg.pkg" else x # InstallESDDmg.pkg gets renamed to InstallESD.dmg - all others stay the same print(" - {}{}".format(x, " --> {}".format(y) if y != x else "")) out = self.r.run({"args":["cp","-R",os.path.join(f_path,x),os.path.join(shared_support,y)]}) if out[2] != 0: raise Exception("Copy Failed!", out[1]) print("Patching InstallInfo.plist...") with open(os.path.join(shared_support,"InstallInfo.plist"),"rb") as f: p = plist.load(f) if "Payload Image Info" in p: pii = p["Payload Image Info"] if "URL" in pii: pii["URL"] = pii["URL"].replace("InstallESDDmg.pkg","InstallESD.dmg") if "id" in pii: pii["id"] = pii["id"].replace("com.apple.pkg.InstallESDDmg","com.apple.dmg.InstallESD") pii.pop("chunklistURL",None) pii.pop("chunklistid",None) with open(os.path.join(shared_support,"InstallInfo.plist"),"wb") as f: plist.dump(p,f) print("") print("Created: {}".format(install_app)) print("Saved to: {}".format(os.path.join(f_path,install_app))) print("") self.u.grab("Press [enter] to return...") except Exception as e: print("An error occurred:") print(" - {}".format(e)) print("") if len(base_mounts): for x in base_mounts: print(" - Unmounting {}...".format(x)) self.unmount_dmg(x) print("") self.u.grab("Press [enter] to return...") if __name__ == '__main__': b = buildMacOSInstallApp() b.main() ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2018 CorpNewt Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: MakeInstall.bat ================================================ @echo off REM Get our local path and args before delayed expansion - allows % and ! set "thisDir=%~dp0" set "args=%*" setlocal enableDelayedExpansion REM Setup initial vars set "script_name=" set /a tried=0 set "toask=yes" set "pause_on_error=yes" set "py2v=" set "py2path=" set "py3v=" set "py3path=" set "pypath=" set "targetpy=3" REM use_py3: REM TRUE = Use if found, use py2 otherwise REM FALSE = Use py2 REM FORCE = Use py3 set "use_py3=TRUE" REM We'll parse if the first argument passed is REM --install-python and if so, we'll just install REM Can optionally take a version number as the REM second arg - i.e. --install-python 3.13.1 set "just_installing=FALSE" set "user_provided=" REM Get the system32 (or equivalent) path call :getsyspath "syspath" REM Make sure the syspath exists if "!syspath!" == "" ( if exist "%SYSTEMROOT%\system32\cmd.exe" ( if exist "%SYSTEMROOT%\system32\reg.exe" ( if exist "%SYSTEMROOT%\system32\where.exe" ( REM Fall back on the default path if it exists set "ComSpec=%SYSTEMROOT%\system32\cmd.exe" set "syspath=%SYSTEMROOT%\system32\" ) ) ) if "!syspath!" == "" ( cls echo ### ### echo # Missing Required Files # echo ### ### echo. echo Could not locate cmd.exe, reg.exe, or where.exe echo. echo Please ensure your ComSpec environment variable is properly configured and echo points directly to cmd.exe, then try again. echo. echo Current CompSpec Value: "%ComSpec%" echo. echo Press [enter] to quit. pause > nul exit /b 1 ) ) if "%~1" == "--install-python" ( set "just_installing=TRUE" set "user_provided=%~2" goto installpy ) goto checkscript :checkscript REM Check for our script first set "looking_for=!script_name!" if "!script_name!" == "" ( set "looking_for=%~n0.py or %~n0.command" set "script_name=%~n0.py" if not exist "!thisDir!\!script_name!" ( set "script_name=%~n0.command" ) ) if not exist "!thisDir!\!script_name!" ( cls echo ### ### echo # Target Not Found # echo ### ### echo. echo Could not find !looking_for!. echo Please make sure to run this script from the same directory echo as !looking_for!. echo. echo Press [enter] to quit. pause > nul exit /b 1 ) goto checkpy :checkpy call :updatepath for /f "USEBACKQ tokens=*" %%x in (`!syspath!where.exe python 2^> nul`) do ( call :checkpyversion "%%x" "py2v" "py2path" "py3v" "py3path" ) for /f "USEBACKQ tokens=*" %%x in (`!syspath!where.exe python3 2^> nul`) do ( call :checkpyversion "%%x" "py2v" "py2path" "py3v" "py3path" ) for /f "USEBACKQ tokens=*" %%x in (`!syspath!where.exe py 2^> nul`) do ( call :checkpylauncher "%%x" "py2v" "py2path" "py3v" "py3path" ) REM Walk our returns to see if we need to install if /i "!use_py3!" == "FALSE" ( set "targetpy=2" set "pypath=!py2path!" ) else if /i "!use_py3!" == "FORCE" ( set "pypath=!py3path!" ) else if /i "!use_py3!" == "TRUE" ( set "pypath=!py3path!" if "!pypath!" == "" set "pypath=!py2path!" ) if not "!pypath!" == "" ( goto runscript ) if !tried! lss 1 ( if /i "!toask!"=="yes" ( REM Better ask permission first goto askinstall ) else ( goto installpy ) ) else ( cls echo ### ### echo # Python Not Found # echo ### ### echo. REM Couldn't install for whatever reason - give the error message echo Python is not installed or not found in your PATH var. echo Please go to https://www.python.org/downloads/windows/ to echo download and install the latest version, then try again. echo. echo Make sure you check the box labeled: echo. echo "Add Python X.X to PATH" echo. echo Where X.X is the py version you're installing. echo. echo Press [enter] to quit. pause > nul exit /b 1 ) goto runscript :checkpylauncher REM Attempt to check the latest python 2 and 3 versions via the py launcher for /f "USEBACKQ tokens=*" %%x in (`%~1 -2 -c "import sys; print(sys.executable)" 2^> nul`) do ( call :checkpyversion "%%x" "%~2" "%~3" "%~4" "%~5" ) for /f "USEBACKQ tokens=*" %%x in (`%~1 -3 -c "import sys; print(sys.executable)" 2^> nul`) do ( call :checkpyversion "%%x" "%~2" "%~3" "%~4" "%~5" ) goto :EOF :checkpyversion set "version="&for /f "tokens=2* USEBACKQ delims= " %%a in (`"%~1" -V 2^>^&1`) do ( REM Ensure we have a version number call :isnumber "%%a" if not "!errorlevel!" == "0" goto :EOF set "version=%%a" ) if not defined version goto :EOF if "!version:~0,1!" == "2" ( REM Python 2 call :comparepyversion "!version!" "!%~2!" if "!errorlevel!" == "1" ( set "%~2=!version!" set "%~3=%~1" ) ) else ( REM Python 3 call :comparepyversion "!version!" "!%~4!" if "!errorlevel!" == "1" ( set "%~4=!version!" set "%~5=%~1" ) ) goto :EOF :isnumber set "var="&for /f "delims=0123456789." %%i in ("%~1") do set var=%%i if defined var (exit /b 1) exit /b 0 :comparepyversion REM Exits with status 0 if equal, 1 if v1 gtr v2, 2 if v1 lss v2 for /f "tokens=1,2,3 delims=." %%a in ("%~1") do ( set a1=%%a set a2=%%b set a3=%%c ) for /f "tokens=1,2,3 delims=." %%a in ("%~2") do ( set b1=%%a set b2=%%b set b3=%%c ) if not defined a1 set a1=0 if not defined a2 set a2=0 if not defined a3 set a3=0 if not defined b1 set b1=0 if not defined b2 set b2=0 if not defined b3 set b3=0 if %a1% gtr %b1% exit /b 1 if %a1% lss %b1% exit /b 2 if %a2% gtr %b2% exit /b 1 if %a2% lss %b2% exit /b 2 if %a3% gtr %b3% exit /b 1 if %a3% lss %b3% exit /b 2 exit /b 0 :askinstall cls echo ### ### echo # Python Not Found # echo ### ### echo. echo Python !targetpy! was not found on the system or in the PATH var. echo. set /p "menu=Would you like to install it now? [y/n]: " if /i "!menu!"=="y" ( REM We got the OK - install it goto installpy ) else if "!menu!"=="n" ( REM No OK here... set /a tried=!tried!+1 goto checkpy ) REM Incorrect answer - go back goto askinstall :installpy REM This will attempt to download and install python set /a tried=!tried!+1 cls echo ### ### echo # Downloading Python # echo ### ### echo. set "release=!user_provided!" if "!release!" == "" ( REM No explicit release set - get the latest from python.org echo Gathering latest version... powershell -command "[Net.ServicePointManager]::SecurityProtocol=[Net.SecurityProtocolType]::Tls12;(new-object System.Net.WebClient).DownloadFile('https://www.python.org/downloads/windows/','%TEMP%\pyurl.txt')" REM Extract it if it's gzip compressed powershell -command "$infile='%TEMP%\pyurl.txt';$outfile='%TEMP%\pyurl.temp';try{$input=New-Object System.IO.FileStream $infile,([IO.FileMode]::Open),([IO.FileAccess]::Read),([IO.FileShare]::Read);$output=New-Object System.IO.FileStream $outfile,([IO.FileMode]::Create),([IO.FileAccess]::Write),([IO.FileShare]::None);$gzipStream=New-Object System.IO.Compression.GzipStream $input,([IO.Compression.CompressionMode]::Decompress);$buffer=New-Object byte[](1024);while($true){$read=$gzipstream.Read($buffer,0,1024);if($read -le 0){break};$output.Write($buffer,0,$read)};$gzipStream.Close();$output.Close();$input.Close();Move-Item -Path $outfile -Destination $infile -Force}catch{}" if not exist "%TEMP%\pyurl.txt" ( if /i "!just_installing!" == "TRUE" ( echo - Failed to get info exit /b 1 ) else ( goto checkpy ) ) pushd "%TEMP%" :: Version detection code slimmed by LussacZheng (https://github.com/corpnewt/gibMacOS/issues/20) for /f "tokens=9 delims=< " %%x in ('findstr /i /c:"Latest Python !targetpy! Release" pyurl.txt') do ( set "release=%%x" ) popd REM Let's delete our txt file now - we no longer need it del "%TEMP%\pyurl.txt" if "!release!" == "" ( if /i "!just_installing!" == "TRUE" ( echo - Failed to get python version exit /b 1 ) else ( goto checkpy ) ) echo Located Version: !release! ) else ( echo User-Provided Version: !release! REM Update our targetpy to reflect the first number of REM our release for /f "tokens=1 delims=." %%a in ("!release!") do ( call :isnumber "%%a" if "!errorlevel!" == "0" ( set "targetpy=%%a" ) ) ) echo Building download url... REM At this point - we should have the version number. REM We can build the url like so: "https://www.python.org/ftp/python/[version]/python-[version]-amd64.exe" set "url=https://www.python.org/ftp/python/!release!/python-!release!-amd64.exe" set "pytype=exe" if "!targetpy!" == "2" ( set "url=https://www.python.org/ftp/python/!release!/python-!release!.amd64.msi" set "pytype=msi" ) echo - !url! echo Downloading... REM Now we download it with our slick powershell command powershell -command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; (new-object System.Net.WebClient).DownloadFile('!url!','%TEMP%\pyinstall.!pytype!')" REM If it doesn't exist - we bail if not exist "%TEMP%\pyinstall.!pytype!" ( if /i "!just_installing!" == "TRUE" ( echo - Failed to download python installer exit /b 1 ) else ( goto checkpy ) ) REM It should exist at this point - let's run it to install silently echo Running python !pytype! installer... pushd "%TEMP%" if /i "!pytype!" == "exe" ( echo - pyinstall.exe /quiet PrependPath=1 Include_test=0 Shortcuts=0 Include_launcher=0 pyinstall.exe /quiet PrependPath=1 Include_test=0 Shortcuts=0 Include_launcher=0 ) else ( set "foldername=!release:.=!" echo - msiexec /i pyinstall.msi /qb ADDLOCAL=ALL TARGETDIR="%LocalAppData%\Programs\Python\Python!foldername:~0,2!" msiexec /i pyinstall.msi /qb ADDLOCAL=ALL TARGETDIR="%LocalAppData%\Programs\Python\Python!foldername:~0,2!" ) popd set "py_error=!errorlevel!" echo Installer finished with status: !py_error! echo Cleaning up... REM Now we should be able to delete the installer and check for py again del "%TEMP%\pyinstall.!pytype!" REM If it worked, then we should have python in our PATH REM this does not get updated right away though - let's try REM manually updating the local PATH var call :updatepath if /i "!just_installing!" == "TRUE" ( echo. echo Done. ) else ( goto checkpy ) exit /b :runscript REM Python found cls REM Checks the args gathered at the beginning of the script. REM Make sure we're not just forwarding empty quotes. set "arg_test=!args:"=!" if "!arg_test!"=="" ( "!pypath!" "!thisDir!!script_name!" ) else ( "!pypath!" "!thisDir!!script_name!" !args! ) if /i "!pause_on_error!" == "yes" ( if not "%ERRORLEVEL%" == "0" ( echo. echo Script exited with error code: %ERRORLEVEL% echo. echo Press [enter] to exit... pause > nul ) ) goto :EOF :undouble REM Helper function to strip doubles of a single character out of a string recursively set "string_value=%~2" :undouble_continue set "check=!string_value:%~3%~3=%~3!" if not "!check!" == "!string_value!" ( set "string_value=!check!" goto :undouble_continue ) set "%~1=!check!" goto :EOF :updatepath set "spath=" set "upath=" for /f "USEBACKQ tokens=2* delims= " %%i in (`!syspath!reg.exe query "HKCU\Environment" /v "Path" 2^> nul`) do ( if not "%%j" == "" set "upath=%%j" ) for /f "USEBACKQ tokens=2* delims= " %%i in (`!syspath!reg.exe query "HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment" /v "Path" 2^> nul`) do ( if not "%%j" == "" set "spath=%%j" ) if not "%spath%" == "" ( REM We got something in the system path set "PATH=%spath%" if not "%upath%" == "" ( REM We also have something in the user path set "PATH=%PATH%;%upath%" ) ) else if not "%upath%" == "" ( set "PATH=%upath%" ) REM Remove double semicolons from the adjusted PATH call :undouble "PATH" "%PATH%" ";" goto :EOF :getsyspath REM Helper method to return a valid path to cmd.exe, reg.exe, and where.exe by REM walking the ComSpec var - will also repair it in memory if need be REM Strip double semi-colons call :undouble "temppath" "%ComSpec%" ";" REM Dirty hack to leverage the "line feed" approach - there are some odd side REM effects with this. Do not use this variable name in comments near this REM line - as it seems to behave erradically. (set LF=^ %=this line is empty=% ) REM Replace instances of semi-colons with a line feed and wrap REM in parenthesis to work around some strange batch behavior set "testpath=%temppath:;=!LF!%" REM Let's walk each path and test if cmd.exe, reg.exe, and where.exe exist there set /a found=0 for /f "tokens=* delims=" %%i in ("!testpath!") do ( REM Only continue if we haven't found it yet if not "%%i" == "" ( if !found! lss 1 ( set "checkpath=%%i" REM Remove "cmd.exe" from the end if it exists if /i "!checkpath:~-7!" == "cmd.exe" ( set "checkpath=!checkpath:~0,-7!" ) REM Pad the end with a backslash if needed if not "!checkpath:~-1!" == "\" ( set "checkpath=!checkpath!\" ) REM Let's see if cmd, reg, and where exist there - and set it if so if EXIST "!checkpath!cmd.exe" ( if EXIST "!checkpath!reg.exe" ( if EXIST "!checkpath!where.exe" ( set /a found=1 set "ComSpec=!checkpath!cmd.exe" set "%~1=!checkpath!" ) ) ) ) ) ) goto :EOF ================================================ FILE: MakeInstall.py ================================================ from Scripts import utils, diskwin, downloader, run import os, sys, tempfile, shutil, zipfile, platform, json, time class WinUSB: def __init__(self): self.u = utils.Utils("MakeInstall") if not self.u.check_admin(): # Try to self-elevate self.u.elevate(os.path.realpath(__file__)) exit() self.min_plat = 9600 # Make sure we're on windows self.verify_os() # Setup initial vars self.d = diskwin.Disk() self.dl = downloader.Downloader() self.r = run.Run() self.scripts = "Scripts" self.s_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), self.scripts) # self.dd_url = "http://www.chrysocome.net/downloads/ddrelease64.exe" self.dd_url = "https://github.com/corpnewt/gibMacOS/files/4573241/ddrelease64.exe.zip" # Rehost due to download issues self.dd_name = ".".join(os.path.basename(self.dd_url).split(".")[:-1]) # Get the name without the last extension self.z_json = "https://sourceforge.net/projects/sevenzip/best_release.json" self.z_url2 = "https://www.7-zip.org/a/7z1806-x64.msi" self.z_url = "https://www.7-zip.org/a/7z[[vers]]-x64.msi" self.z_name = "7z.exe" self.bi_url = "https://raw.githubusercontent.com/corpnewt/gibMacOS/master/Scripts/BOOTICEx64.exe" self.bi_name = "BOOTICEx64.exe" self.clover_url = "https://api.github.com/repos/CloverHackyColor/CloverBootloader/releases" self.dids_url = "https://api.github.com/repos/dids/clover-builder/releases" self.oc_url = "https://api.github.com/repos/acidanthera/OpenCorePkg/releases" self.oc_boot = "boot" self.oc_boot_alt = "bootX64" self.oc_boot0 = "boot0" self.oc_boot1 = "boot1f32" # self.oc_boot_url = "https://github.com/acidanthera/OpenCorePkg/raw/master/Utilities/LegacyBoot/" self.oc_boot_url = "https://github.com/acidanthera/OpenCorePkg/raw/870017d0e5d53abeaf0347997da912c3e382a04a/Utilities/LegacyBoot/" self.diskpart = os.path.join(os.environ['SYSTEMDRIVE'] + "\\", "Windows", "System32", "diskpart.exe") # From Tim Sutton's brigadier: https://github.com/timsutton/brigadier/blob/master/brigadier self.z_path = None self.z_path64 = os.path.join(os.environ['SYSTEMDRIVE'] + "\\", "Program Files", "7-Zip", "7z.exe") self.z_path32 = os.path.join(os.environ['SYSTEMDRIVE'] + "\\", "Program Files (x86)", "7-Zip", "7z.exe") self.recovery_suffixes = ( "recoveryhdupdate.pkg", "recoveryhdmetadmg.pkg", "basesystem.dmg", "recoveryimage.dmg" ) self.dd_bootsector = True self.boot0 = "boot0af" self.boot1 = "boot1f32alt" self.boot = "boot6" self.efi_id = "c12a7328-f81f-11d2-ba4b-00a0c93ec93b" # EFI self.bas_id = "ebd0a0a2-b9e5-4433-87c0-68b6b72699c7" # Microsoft Basic Data self.hfs_id = "48465300-0000-11AA-AA11-00306543ECAC" # HFS+ self.rec_id = "426F6F74-0000-11AA-AA11-00306543ECAC" # Apple Boot partition (Recovery HD) self.show_all_disks = False def verify_os(self): self.u.head("Verifying OS") print("") print("Verifying OS name...") if not os.name=="nt": print("") print("This script is only for Windows!") print("") self.u.grab("Press [enter] to exit...") exit(1) print(" - Name = NT") print("Verifying OS version...") # Verify we're at version 9600 or greater try: # Set plat to the last item of the output split by . - looks like: # Windows-8.1-6.3.9600 # or this: # Windows-10-10.0.17134-SP0 plat = int(platform.platform().split(".")[-1].split("-")[0]) except: plat = 0 if plat < self.min_plat: print("") print("Currently running {}, this script requires version {} or newer.".format(platform.platform(), self.min_plat)) print("") self.u.grab("Press [enter] to exit...") exit(1) print(" - Version = {}".format(plat)) print("") print("{} >= {}, continuing...".format(plat, self.min_plat)) def get_disks_of_type(self, disk_list, disk_type=(0,2)): disks = {} for disk in disk_list: if disk_list[disk].get("type",0) in disk_type: disks[disk] = disk_list[disk] return disks def check_dd(self): # Checks if ddrelease64.exe exists in our Scripts dir # and if not - downloads it # # Returns True if exists/downloaded successfully # or False if issues. # Check for dd.exe in the current dir if os.path.exists(os.path.join(self.s_path, self.dd_name)): # print("Located {}!".format(self.dd_name)) # Got it return True print("Couldn't locate {} - downloading...".format(self.dd_name)) temp = tempfile.mkdtemp() z_file = os.path.basename(self.dd_url) # Now we need to download self.dl.stream_to_file(self.dd_url, os.path.join(temp,z_file)) print(" - Extracting...") # Extract with built-in tools \o/ cwd = os.getcwd() os.chdir(temp) with zipfile.ZipFile(os.path.join(temp,z_file)) as z: z.extractall(temp) for x in os.listdir(temp): if self.dd_name.lower() == x.lower(): # Found it print(" - Found {}".format(x)) print(" - Copying to {} directory...".format(self.scripts)) shutil.copy(os.path.join(temp,x), os.path.join(self.s_path,x)) # Return to prior cwd os.chdir(cwd) # Remove the temp folder shutil.rmtree(temp,ignore_errors=True) print("") return os.path.exists(os.path.join(self.s_path, self.dd_name)) def check_7z(self): # Check the PATH var first z_path = self.r.run({"args":["where.exe","7z.exe"]})[0].split("\n")[0].rstrip("\r") self.z_path = next((x for x in (z_path,self.z_path64,self.z_path32) if x and os.path.isfile(x)),None) if self.z_path: return True print("Didn't locate {} - downloading...".format(self.z_name)) # Didn't find it - let's do some stupid stuff # First we get our json response - or rather, try to, then parse it # looking for the current version dl_url = None try: json_data = json.loads(self.dl.get_string(self.z_json)) v_num = json_data.get("release",{}).get("filename","").split("/")[-1].lower().split("-")[0].replace("7z","").replace(".exe","") if len(v_num): dl_url = self.z_url.replace("[[vers]]",v_num) except: pass if not dl_url: dl_url = self.z_url2 temp = tempfile.mkdtemp() dl_file = self.dl.stream_to_file(dl_url, os.path.join(temp, self.z_name)) if not dl_file: # Didn't download right shutil.rmtree(temp,ignore_errors=True) return False print("") print("Installing 7zip...") # From Tim Sutton's brigadier: https://github.com/timsutton/brigadier/blob/master/brigadier out = self.r.run({"args":["msiexec", "/qn", "/i", os.path.join(temp, self.z_name)],"stream":True}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print("Error ({})".format(out[2])) print("") self.u.grab("Press [enter] to exit...") exit(1) print("") self.z_path = self.z_path64 if os.path.exists(self.z_path64) else self.z_path32 if os.path.exists(self.z_path32) else None return self.z_path and os.path.exists(self.z_path) def check_bi(self): # Checks for BOOTICEx64.exe in our scripts dir # and downloads it if need be if os.path.exists(os.path.join(self.s_path, self.bi_name)): # print("Located {}!".format(self.bi_name)) # Got it return True print("Couldn't locate {} - downloading...".format(self.bi_name)) self.dl.stream_to_file(self.bi_url, os.path.join(self.s_path, self.bi_name)) print("") return os.path.exists(os.path.join(self.s_path,self.bi_name)) def get_dl_url_from_json(self,json_data,suffix=(".lzma",".iso.7z")): try: j_list = json.loads(json_data) except: return None j_list = j_list if isinstance(j_list,list) else [j_list] for j in j_list: dl_link = next((x.get("browser_download_url", None) for x in j.get("assets", []) if x.get("browser_download_url", "").endswith(suffix)), None) if dl_link: break if not dl_link: return None return { "url" : dl_link, "name" : os.path.basename(dl_link), "info" : j.get("body", None) } def get_dl_info(self,clover_version=None): # Returns the latest download package and info in a # dictionary: { "url" : dl_url, "name" : name, "info" : update_info } # Attempt Dids' repo first - falling back on Clover's official repo as needed clover_urls = (self.clover_url,self.dids_url) try: assert int(clover_version) <= 5122 # Check if we're trying to get r5122 or prior # If we didn't throw an exception, we can reverse the order of the URLs to check # Dids' builder first clover_urls = (self.dids_url,self.clover_url) except: pass # Wasn't a proper int, or was above 5122 for url in clover_urls: # Tag is e.g. 5098 on Slice's repo, and e.g. v2.5k_r5098 or v5.0_r5xxx on Dids' # accommodate as needed if not clover_version: search_url = url elif url == self.clover_url: # Using CloverHackyColor's repo - set the tag to the version search_url = "{}/tags/{}".format(url,clover_version) else: # Using Dids' clover builder - figure out the prefix based on the version search_url = "{}/tags/v{}_r{}".format(url,"5.0" if clover_version >= "5118" else "2.5k",clover_version) print(" - Checking {}".format(search_url)) json_data = self.dl.get_string(search_url, False) if not json_data: print(" --> Not found!") else: return self.get_dl_url_from_json(json_data) return None def get_oc_dl_info(self): json_data = self.dl.get_string(self.oc_url, False) if not json_data: print(" --> Not found!") else: return self.get_dl_url_from_json(json_data,suffix="-RELEASE.zip") def diskpart_flag(self, disk, as_efi=False): # Sets and unsets the GUID needed for a GPT EFI partition ID self.u.head("Changing ID With DiskPart") print("") print("Setting type as {}...".format("EFI" if as_efi else "Basic Data")) print("") # - EFI system partition: c12a7328-f81f-11d2-ba4b-00a0c93ec93b # - Basic data partition: ebd0a0a2-b9e5-4433-87c0-68b6b72699c7 dp_script = "\n".join([ "select disk {}".format(disk.get("index",-1)), "sel part 1", "set id={}".format(self.efi_id if as_efi else self.bas_id) ]) temp = tempfile.mkdtemp() script = os.path.join(temp, "diskpart.txt") try: with open(script,"w") as f: f.write(dp_script) except: shutil.rmtree(temp) print("Error creating script!") print("") self.u.grab("Press [enter] to return...") return # Let's try to run it! out = self.r.run({"args":[self.diskpart,"/s",script],"stream":True}) # Ditch our script regardless of whether diskpart worked or not shutil.rmtree(temp) print("") if out[2] != 0: # Error city! print("DiskPart exited with non-zero status ({}). Aborting.".format(out[2])) else: print("Done - You may need to replug your drive for the") print("changes to take effect.") print("") self.u.grab("Press [enter] to return...") def diskpart_erase(self, disk, gpt=False, clover_version = None, local_file = None): # Generate a script that we can pipe to diskpart to erase our disk self.u.head("Erasing With DiskPart") print("") # Then we'll re-gather our disk info on success and move forward # Using MBR to effectively set the individual partition types # Keeps us from having issues mounting the EFI on Windows - # and also lets us explicitly set the partition id for the main # data partition. if not gpt: print("Using MBR...") dp_script = "\n".join([ "select disk {}".format(disk.get("index",-1)), "clean", "convert mbr", "create partition primary size=200", "format quick fs=fat32 label='BOOT'", "active", "create partition primary", "select part 2", "set id=AB", # AF = HFS, AB = Recovery "select part 1", "assign" ]) else: print("Using GPT...") dp_script = "\n".join([ "select disk {}".format(disk.get("index",-1)), "clean", "convert gpt", "create partition primary size=200", "format quick fs=fat32 label='BOOT'", "create partition primary id={}".format(self.hfs_id) ]) temp = tempfile.mkdtemp() script = os.path.join(temp, "diskpart.txt") try: with open(script,"w") as f: f.write(dp_script) except: shutil.rmtree(temp) print("Error creating script!") print("") self.u.grab("Press [enter] to return...") return # Let's try to run it! out = self.r.run({"args":[self.diskpart,"/s",script],"stream":True}) # Ditch our script regardless of whether diskpart worked or not shutil.rmtree(temp) if out[2] != 0: # Error city! print("") print("DiskPart exited with non-zero status ({}). Aborting.".format(out[2])) print("") self.u.grab("Press [enter] to return...") return # We should now have a fresh drive to work with # Let's write an image or something self.u.head("Updating Disk Information") print("") print("Re-populating list...") self.d.update() print("Relocating disk {}".format(disk["index"])) disk = self.d.disks[str(disk["index"])] self.select_package(disk, clover_version, local_file=local_file) def select_package(self, disk, clover_version = None, local_file = None): self.u.head("Select Recovery Package") print("") print("{}. {} - {} ({})".format( disk.get("index",-1), disk.get("model","Unknown"), self.dl.get_size(disk.get("size",-1),strip_zeroes=True), ["Unknown","No Root Dir","Removable","Local","Network","Disc","RAM Disk"][disk.get("type",0)] )) print("") print("M. Main Menu") print("Q. Quit") print("") print("(To copy a file's path, shift + right-click in Explorer and select 'Copy as path')\n") menu = self.u.grab("Please paste the recovery update pkg/dmg path to extract: ") if menu.lower() == "q": self.u.custom_quit() if menu.lower() == "m": return path = self.u.check_path(menu) if not path: self.select_package(disk, clover_version, local_file=local_file) return # Got the package - let's make sure it's named right - just in case if os.path.basename(path).lower().endswith(".hfs"): # We have an hfs image already - bypass extraction self.dd_image(disk, path, clover_version, local_file=local_file) return # If it's a directory, find the first recovery hit if os.path.isdir(path): for f in os.listdir(path): if f.lower().endswith(self.recovery_suffixes): path = os.path.join(path, f) break # Make sure it's named right for recovery stuffs if not path.lower().endswith(self.recovery_suffixes): self.u.head("Invalid Package") print("") print("{} is not in the available recovery package names:\n{}".format(os.path.basename(path), ", ".join(self.recovery_suffixes))) print("") print("Ensure you're passing a proper recovery package.") print("") self.u.grab("Press [enter] to return to package selection...") self.select_package(disk, clover_version, local_file=local_file) return self.u.head("Extracting Package") print("") temp = tempfile.mkdtemp() cwd = os.getcwd() os.chdir(temp) print("Located {}...".format(os.path.basename(path))) if not path.lower().endswith(".dmg"): # Extract in sections and remove any files we run into print("Extracting Recovery dmg...") out = self.r.run({"args":[self.z_path, "e", "-txar", path, "*.dmg"]}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print("An error occurred extracting: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return print("Extracting BaseSystem.dmg...") # No files to delete here - let's extract the next part out = self.r.run({"args":[self.z_path, "e", "*.dmg", "*/Base*.dmg"]}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print("An error occurred extracting: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return # If we got here - we should delete everything in the temp folder except # for a .dmg that starts with Base del_list = [x for x in os.listdir(temp) if not (x.lower().startswith("base") and x.lower().endswith(".dmg"))] for d in del_list: os.remove(os.path.join(temp, d)) # Onto the last command print("Extracting hfs...") out = self.r.run({"args":[self.z_path, "e", "-tdmg", path if path.lower().endswith(".dmg") else "Base*.dmg", "*.hfs"]}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print("An error occurred extracting: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return # If we got here - we should delete everything in the temp folder except # for a .dmg that starts with Base del_list = [x for x in os.listdir(temp) if not x.lower().endswith(".hfs")] for d in del_list: os.remove(os.path.join(temp, d)) print("Extracted successfully!") hfs = next((x for x in os.listdir(temp) if x.lower().endswith(".hfs")),None) # Now to dd our image - if it exists if not hfs: print("Missing the .hfs file! Aborting.") print("") self.u.grab("Press [enter] to return...") else: self.dd_image(disk, os.path.join(temp, hfs), clover_version, local_file=local_file) shutil.rmtree(temp,ignore_errors=True) def dd_image(self, disk, image, clover_version = None, local_file = None): # Let's dd the shit out of our disk self.u.head("Copying Image To Drive") print("") print("Image: {}".format(image)) print("") print("Disk {}. {} - {} ({})".format( disk.get("index",-1), disk.get("model","Unknown"), self.dl.get_size(disk.get("size",-1),strip_zeroes=True), ["Unknown","No Root Dir","Removable","Local","Network","Disc","RAM Disk"][disk.get("type",0)] )) print("") args = [ os.path.join(self.s_path, self.dd_name), "if={}".format(image), "of=\\\\?\\Device\\Harddisk{}\\Partition2".format(disk.get("index",-1)), "bs=8M", "--progress" ] print(" ".join(args)) print("") print("This may take some time!") print("") out = self.r.run({"args":args}) if len(out[1].split("Error")) > 1: # We had some error text - dd, even when failing likes to give us a 0 # status code. It also sends a ton of text through stderr - so we comb # that for "Error" then split by that to skip the extra fluff and show only # the error. print("An error occurred:\n\n{}".format("Error"+out[1].split("Error")[1])) print("") self.u.grab("Press [enter] to return to the main menu...") return # Install Clover/OC to the target drive if clover_version == "OpenCore": self.install_oc(disk, local_file=local_file) else: self.install_clover(disk, clover_version, local_file=local_file) def install_oc(self, disk, local_file = None): self.u.head("Installing OpenCore") print("") print("Gathering info...") if not local_file: o = self.get_oc_dl_info() if o is None: print(" - Error communicating with github!") print("") self.u.grab("Press [enter] to return...") return print(" - Got {}".format(o.get("name","Unknown Version"))) print("Downloading...") temp = tempfile.mkdtemp() os.chdir(temp) self.dl.stream_to_file(o["url"], os.path.join(temp, o["name"])) else: print("Using local file: {}".format(local_file)) temp = tempfile.mkdtemp() os.chdir(temp) o = {"name":os.path.basename(local_file)} # Copy to the temp folder shutil.copy(local_file,os.path.join(temp,o["name"])) print("") # Empty space to clear the download progress if not os.path.exists(os.path.join(temp, o["name"])): shutil.rmtree(temp,ignore_errors=True) print(" - Download failed. Aborting...") print("") self.u.grab("Press [enter] to return...") return oc_zip = o["name"] # Got a valid file in our temp dir print("Extracting {}...".format(oc_zip)) out = self.r.run({"args":[self.z_path, "x", os.path.join(temp,oc_zip)]}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print(" - An error occurred extracting: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return # We need to also gather our boot, boot0af, and boot1f32 files print("Gathering DUET boot files...") uefi_only = False duet_loc = os.path.join(temp,"Utilities","LegacyBoot") for x in (self.oc_boot,self.oc_boot_alt,self.oc_boot0,self.oc_boot1): # Check the local dir first if os.path.exists(os.path.join(duet_loc,x)): print(" - {}".format(x)) # Copy it over target_name = self.oc_boot if x == self.oc_boot_alt else x shutil.copy(os.path.join(duet_loc,x), os.path.join(temp,target_name)) missing_list = [x for x in (self.oc_boot,self.oc_boot0,self.oc_boot1) if not os.path.exists(os.path.join(temp,x))] if missing_list: print(" - Missing: {}".format(", ".join(missing_list))) print("Attempting to download...") for x in missing_list: print(" - {}".format(x)) self.dl.stream_to_file(self.oc_boot_url + x, os.path.join(temp,x),False) if not all((os.path.exists(os.path.join(temp,x)) for x in missing_list)): print("Could not located all required DUET files - USB will be UEFI ONLY") uefi_only = True # At this point, we should have a boot0xx file and an EFI folder in the temp dir # We need to udpate the disk list though - to reflect the current file system on part 1 # of our current disk self.d.update() # assumes our disk number stays the same # Some users are having issues with the "partitions" key not populating - possibly a 3rd party disk management soft? # Possibly a bad USB? # We'll see if the key exists - if not, we'll throw an error. if self.d.disks[str(disk["index"])].get("partitions",None) is None: # No partitions found. shutil.rmtree(temp,ignore_errors=True) print("No partitions located on disk!") print("") self.u.grab("Press [enter] to return...") return part = self.d.disks[str(disk["index"])]["partitions"].get("0",{}).get("letter",None) # get the first partition's letter if part is None: shutil.rmtree(temp,ignore_errors=True) print("Lost original disk - or formatting failed!") print("") self.u.grab("Press [enter] to return...") return # Here we have our disk and partitions and such - the BOOT partition # will be the first partition # Let's copy over the EFI folder and then dd the boot0xx file print("Copying EFI folder to {}/EFI...".format(part)) source_efi = None if os.path.exists(os.path.join(temp,"EFI")): source_efi = os.path.join(temp,"EFI") elif os.path.exists(os.path.join(temp,"X64","EFI")): source_efi = os.path.join(temp,"X64","EFI") if not source_efi: print(" - Source EFI not found!") print("") self.u.grab("Press [enter] to return...") return if os.path.exists("{}/EFI".format(part)): print(" - EFI exists - removing...") shutil.rmtree("{}/EFI".format(part),ignore_errors=True) time.sleep(1) # Added because windows is dumb shutil.copytree(source_efi, "{}/EFI".format(part)) if not uefi_only: # Copy boot over to the root of the EFI volume print("Copying {} to {}/boot...".format(self.oc_boot,part)) shutil.copy(os.path.join(temp,self.oc_boot),"{}/boot".format(part)) # Use bootice to update the MBR and PBR - always on the first # partition (which is 0 in bootice) print("Updating the MBR with {}...".format(self.oc_boot0)) args = [ os.path.join(self.s_path,self.bi_name), "/device={}".format(disk.get("index",-1)), "/mbr", "/restore", "/file={}".format(os.path.join(temp,self.oc_boot0)), "/keep_dpt", "/quiet" ] out = self.r.run({"args":args}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print(" - An error occurred updating the MBR: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return print("Updating the PBR with {}...".format(self.oc_boot1)) args = [ os.path.join(self.s_path,self.bi_name), "/device={}:0".format(disk.get("index",-1)), "/pbr", "/restore", "/file={}".format(os.path.join(temp,self.oc_boot1)), "/keep_bpb", "/quiet" ] out = self.r.run({"args":args}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print(" - An error occurred updating the PBR: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return print("Cleaning up...") shutil.rmtree(temp,ignore_errors=True) print("") print("Done.") print("") self.u.grab("Press [enter] to return to the main menu...") def install_clover(self, disk, clover_version = None, local_file = None): self.u.head("Installing Clover - {}".format("Latest" if not clover_version else "r"+clover_version)) print("") print("Gathering info...") if not local_file: c = self.get_dl_info(clover_version) if c is None: if clover_version is None: print(" - Error communicating with github!") else: print(" - Error gathering info for Clover r{}".format(clover_version)) print("") self.u.grab("Press [enter] to return...") return print(" - Got {}".format(c.get("name","Unknown Version"))) print("Downloading...") temp = tempfile.mkdtemp() os.chdir(temp) self.dl.stream_to_file(c["url"], os.path.join(temp, c["name"])) else: print("Using local file: {}".format(local_file)) temp = tempfile.mkdtemp() os.chdir(temp) c = {"name":os.path.basename(local_file)} # Copy to the temp folder shutil.copy(local_file,os.path.join(temp,c["name"])) print("") # Empty space to clear the download progress if not os.path.exists(os.path.join(temp, c["name"])): shutil.rmtree(temp,ignore_errors=True) print(" - Download failed. Aborting...") print("") self.u.grab("Press [enter] to return...") return clover_archive = c["name"] # Got a valid file in our temp dir print("Extracting {}...".format(clover_archive)) out = self.r.run({"args":[self.z_path, "e", os.path.join(temp,clover_archive)]}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print(" - An error occurred extracting: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return # Should result in a .tar file clover_tar = next((x for x in os.listdir(temp) if x.lower().endswith(".tar")),None) if clover_tar: # Got a .tar archive - get the .iso print("Extracting {}...".format(clover_tar)) out = self.r.run({"args":[self.z_path, "e", os.path.join(temp,clover_tar)]}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print(" - An error occurred extracting: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return # Should result in a .iso file clover_iso = next((x for x in os.listdir(temp) if x.lower().endswith(".iso")),None) if not clover_iso: shutil.rmtree(temp,ignore_errors=True) print(" - No .iso found - aborting...") print("") self.u.grab("Press [enter] to return...") return # Got the .iso - let's extract the needed parts print("Extracting EFI from {}...".format(clover_iso)) out = self.r.run({"args":[self.z_path, "x", os.path.join(temp,clover_iso), "EFI*"]}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print(" - An error occurred extracting: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return print("Extracting {} from {}...".format(self.boot0,clover_iso)) out = self.r.run({"args":[self.z_path, "e", os.path.join(temp,clover_iso), self.boot0, "-r"]}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print(" - An error occurred extracting: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return print("Extracting {} from {}...".format(self.boot1,clover_iso)) out = self.r.run({"args":[self.z_path, "e", os.path.join(temp,clover_iso), self.boot1, "-r"]}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print(" - An error occurred extracting: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return print("Extracting {} from {}...".format(self.boot,clover_iso)) out = self.r.run({"args":[self.z_path, "e", os.path.join(temp,clover_iso), self.boot, "-r"]}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print(" - An error occurred extracting: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return # At this point, we should have a boot0xx file and an EFI folder in the temp dir # We need to udpate the disk list though - to reflect the current file system on part 1 # of our current disk self.d.update() # assumes our disk number stays the same # Some users are having issues with the "partitions" key not populating - possibly a 3rd party disk management soft? # Possibly a bad USB? # We'll see if the key exists - if not, we'll throw an error. if self.d.disks[str(disk["index"])].get("partitions",None) is None: # No partitions found. shutil.rmtree(temp,ignore_errors=True) print("No partitions located on disk!") print("") self.u.grab("Press [enter] to return...") return part = self.d.disks[str(disk["index"])]["partitions"].get("0",{}).get("letter",None) # get the first partition's letter if part is None: shutil.rmtree(temp,ignore_errors=True) print("Lost original disk - or formatting failed!") print("") self.u.grab("Press [enter] to return...") return # Here we have our disk and partitions and such - the CLOVER partition # will be the first partition # Let's copy over the EFI folder and then dd the boot0xx file print("Copying EFI folder to {}/EFI...".format(part)) if os.path.exists("{}/EFI".format(part)): print(" - EFI exists - removing...") shutil.rmtree("{}/EFI".format(part),ignore_errors=True) time.sleep(1) # Added because windows is dumb shutil.copytree(os.path.join(temp,"EFI"), "{}/EFI".format(part)) # Copy boot6 over to the root of the EFI volume - and rename it to boot print("Copying {} to {}/boot...".format(self.boot,part)) shutil.copy(os.path.join(temp,self.boot),"{}/boot".format(part)) # Use bootice to update the MBR and PBR - always on the first # partition (which is 0 in bootice) print("Updating the MBR with {}...".format(self.boot0)) args = [ os.path.join(self.s_path,self.bi_name), "/device={}".format(disk.get("index",-1)), "/mbr", "/restore", "/file={}".format(os.path.join(temp,self.boot0)), "/keep_dpt", "/quiet" ] out = self.r.run({"args":args}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print(" - An error occurred updating the MBR: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return print("Updating the PBR with {}...".format(self.boot1)) args = [ os.path.join(self.s_path,self.bi_name), "/device={}:0".format(disk.get("index",-1)), "/pbr", "/restore", "/file={}".format(os.path.join(temp,self.boot1)), "/keep_bpb", "/quiet" ] out = self.r.run({"args":args}) if out[2] != 0: shutil.rmtree(temp,ignore_errors=True) print(" - An error occurred updating the PBR: {}".format(out[2])) print("") self.u.grab("Press [enter] to return...") return print("Cleaning up...") shutil.rmtree(temp,ignore_errors=True) print("") print("Done.") print("") self.u.grab("Press [enter] to return to the main menu...") def main(self): # Start out with our cd in the right spot. os.chdir(os.path.dirname(os.path.realpath(__file__))) # Let's make sure we have the required files needed self.u.head("Checking Required Tools") print("") if not self.check_dd(): print("Couldn't find or install {} - aborting!\n".format(self.dd_name)) self.u.grab("Press [enter] to exit...") exit(1) if not self.check_7z(): print("Couldn't find or install {} - aborting!\n".format(self.z_name)) self.u.grab("Press [enter] to exit...") exit(1) if not self.check_bi(): print("Couldn't find or install {} - aborting!\n".format(self.bi_name)) self.u.grab("Press [enter] to exit...") exit(1) # Let's just setup a real simple interface and try to write some data self.u.head("Gathering Disk Info") print("") print("Populating list...") self.d.update() print("") print("Done!") # Let's serve up a list of *only* removable media self.u.head("Potential Removable Media") print("") rem_disks = self.get_disks_of_type(self.d.disks) if not self.show_all_disks else self.d.disks # Types: 0 = Unknown, 1 = No Root Dir, 2 = Removable, 3 = Local, 4 = Network, 5 = Disc, 6 = RAM disk if self.show_all_disks: print("!WARNING! This list includes ALL disk types.") print("!WARNING! Be ABSOLUTELY sure before selecting") print("!WARNING! a disk!") else: print("!WARNING! This list includes both Removable AND") print("!WARNING! Unknown disk types. Be ABSOLUTELY sure") print("!WARNING! before selecting a disk!") print("") for disk in sorted(rem_disks,key=lambda x:int(x)): print("{}. {} - {} ({})".format( disk, rem_disks[disk].get("model","Unknown"), self.dl.get_size(rem_disks[disk].get("size",-1),strip_zeroes=True), ["Unknown","No Root Dir","Removable","Local","Network","Disc","RAM Disk"][rem_disks[disk].get("type",0)] )) if not len(rem_disks[disk].get("partitions",{})): print(" No Mounted Partitions") else: parts = rem_disks[disk]["partitions"] for p in sorted(parts,key=lambda x:int(x)): print(" {}. {} ({}) {} - {}".format( p, parts[p].get("letter","No Letter"), "No Name" if not parts[p].get("name",None) else parts[p].get("name","No Name"), parts[p].get("file system","Unknown FS"), self.dl.get_size(parts[p].get("size",-1),strip_zeroes=True) )) print("") print("Q. Quit") print("") print("Usage: [drive number][options] r[Clover revision (optional), requires C]\n (eg. 1B C r5092)") print(" Options are as follows with precedence B > F > E > U > G:") print(" B = Only install the boot manager to the drive's first partition.") print(" F = Skip formatting the disk - will install the boot manager to the first") print(" partition, and dd the recovery image to the second.") print(" E = Sets the type of the drive's first partition to EFI.") print(" U = Similar to E, but sets the type to Basic Data (useful for editing).") print(" G = Format as GPT (default is MBR).") print(" C = Use Clover instead of OpenCore.") print(" L = Provide a local archive for the boot manager - must still use C if Clover.") print(" D = Used without a drive number, toggles showing all disks (currently {}).".format("ENABLED" if self.show_all_disks else "DISABLED")) print("") menu = self.u.grab("Please select a disk or press [enter] with no options to refresh: ") if not len(menu): self.main() return if menu.lower() == "q": self.u.custom_quit() if menu.lower() == "d": self.show_all_disks ^= True self.main() return only_boot = set_efi = unset_efi = use_gpt = user_provided = no_format = False local_file = None use_oc = True if "b" in menu.lower(): only_boot = True menu = menu.lower().replace("b","") if "c" in menu.lower(): use_oc = False menu = menu.lower().replace("c","") if "o" in menu.lower(): # Remove legacy "o" value menu = menu.lower().replace("o","") if "e" in menu.lower(): set_efi = True menu = menu.lower().replace("e","") if "u" in menu.lower(): unset_efi = True menu = menu.lower().replace("u","") if "g" in menu.lower(): use_gpt = True menu = menu.lower().replace("g","") if "l" in menu.lower(): user_provided = True menu = menu.lower().replace("l","") if "f" in menu.lower(): no_format = True menu = menu.lower().replace("f","") # Extract Clover version from args if found clover_list = [x for x in menu.split() if x.lower().startswith("r") and all(y in "0123456789" for y in x[1:])] menu = " ".join([x for x in menu.split() if not x in clover_list]) clover_version = None if not len(clover_list) else clover_list[0][1:] # Skip the "r" prefix # Prepare for OC if need be if use_oc: clover_version = "OpenCore" selected_disk = rem_disks.get(menu.strip(),None) if not selected_disk: self.u.head("Invalid Choice") print("") print("Disk {} is not an option.".format(menu)) print("") self.u.grab("Returning in 5 seconds...", timeout=5) self.main() return # Got a disk! if user_provided: # Prompt the user for the target archive while True: self.u.head("Local Archive") print("") if use_oc: print("NOTE: OpenCore archives are expected to be .zip!") else: print("NOTE: Clover archives are expected to be an ISO packed in either .tar.lzma or .7z!") print("") print("M. Return to the menu") print("Q. Quit") print("") print("(To copy a file's path, shift + right-click in Explorer and select 'Copy as path')\n") l = self.u.grab("Please {} archive path here: ".format("OpenCore" if use_oc else "Clover")) if not len(l): continue if l.lower() == "m": break elif l.lower() == "q": self.u.custom_quit() l_check = self.u.check_path(l) if not l_check or not l_check.lower().endswith(".zip" if use_oc else (".tar.lzma",".7z")): continue # Got a valid path that ends with the proper extension local_file = l_check break # Check if we got something if not local_file: self.main() return if only_boot: if use_oc: self.install_oc(selected_disk, local_file=local_file) else: self.install_clover(selected_disk, clover_version, local_file=local_file) elif no_format: # Make sure we warn the user that the second partition **NEEDS** to be a RAW # partition for dd to properly work while True: self.u.head("WARNING") print("") print("{}. {} - {} ({})".format( selected_disk.get("index",-1), selected_disk.get("model","Unknown"), self.dl.get_size(selected_disk.get("size",-1),strip_zeroes=True), ["Unknown","No Root Dir","Removable","Local","Network","Disc","RAM Disk"][selected_disk.get("type",0)] )) print("") print("In order to continue without formatting, the selected disk's first") print("partition MUST be FAT32, and the second MUST be RAW. If that is not") print("the case, the operation WILL fail.") print("") yn = self.u.grab("Continue? (y/n): ") if yn.lower() == "n": self.main() return if yn.lower() == "y": break self.select_package(selected_disk, clover_version, local_file=local_file) elif set_efi: self.diskpart_flag(selected_disk, True) elif unset_efi: self.diskpart_flag(selected_disk, False) else: # Check erase while True: self.u.head("Erase {}".format(selected_disk.get("model","Unknown"))) print("") print("{}. {} - {} ({})".format( selected_disk.get("index",-1), selected_disk.get("model","Unknown"), self.dl.get_size(selected_disk.get("size",-1),strip_zeroes=True), ["Unknown","No Root Dir","Removable","Local","Network","Disc","RAM Disk"][selected_disk.get("type",0)] )) print("") print("If you continue - THIS DISK WILL BE ERASED") print("ALL DATA WILL BE LOST AND ALL PARTITIONS WILL") print("BE REMOVED!!!!!!!") print("") yn = self.u.grab("Continue? (y/n): ") if yn.lower() == "n": self.main() return if yn.lower() == "y": break # Got the OK to erase! Let's format a diskpart script! self.diskpart_erase(selected_disk, use_gpt, clover_version, local_file=local_file) self.main() if __name__ == '__main__': w = WinUSB() w.main() ================================================ FILE: Readme.md ================================================ Py2/py3 script that can download macOS components direct from Apple Can also now build Internet Recovery USB installers from Windows using [dd](http://www.chrysocome.net/dd) and [7zip](https://www.7-zip.org/download.html). **NOTE:** As of macOS 11 (Big Sur), Apple has changed the way they distribute macOS, and internet recovery USBs can no longer be built via MakeInstall on Windows. macOS versions through Catalina will still work though. **NOTE 2:** As of macOS 11 (Big Sur), Apple distributes the OS via an InstallAssistant.pkg file. `BuildmacOSInstallApp.command` is not needed to create the install application when in macOS in this case - and you can simply run `InstallAssistant.pkg`, which will place the install app in your /Applications folder on macOS. Thanks to: * FoxletFox for [FetchMacOS](http://www.insanelymac.com/forum/topic/326366-fetchmacos-a-tool-to-download-macos-on-non-mac-platforms/) and outlining the URL setup * munki for his [macadmin-scripts](https://github.com/munki/macadmin-scripts) * timsutton for [brigadier](https://github.com/timsutton/brigadier) * wolfmannight for [manOSDownloader_rc](https://www.insanelymac.com/forum/topic/338810-create-legit-copy-of-macos-from-apple-catalog/) off which BuildmacOSInstallApp.command is based ================================================ FILE: Scripts/__init__.py ================================================ from os.path import dirname, basename, isfile import glob modules = glob.glob(dirname(__file__)+"/*.py") __all__ = [ basename(f)[:-3] for f in modules if isfile(f) and not f.endswith('__init__.py')] ================================================ FILE: Scripts/disk.py ================================================ import subprocess, plistlib, sys, os, time, json sys.path.append(os.path.abspath(os.path.dirname(os.path.realpath(__file__)))) import run if sys.version_info < (3,0): # Force use of StringIO instead of cStringIO as the latter # has issues with Unicode strings from StringIO import StringIO class Disk: def __init__(self): self.r = run.Run() self.diskutil = self.get_diskutil() self.os_version = ".".join( self.r.run({"args":["sw_vers", "-productVersion"]})[0].split(".")[:2] ) self.full_os_version = self.r.run({"args":["sw_vers", "-productVersion"]})[0] if len(self.full_os_version.split(".")) < 3: # Add .0 in case of 10.14 self.full_os_version += ".0" self.sudo_mount_version = "10.13.6" self.sudo_mount_types = ["efi"] self.apfs = {} self._update_disks() def _get_str(self, val): # Helper method to return a string value based on input type if (sys.version_info < (3,0) and isinstance(val, unicode)) or (sys.version_info >= (3,0) and isinstance(val, bytes)): return val.encode("utf-8") return str(val) def _get_plist(self, s): p = {} try: if sys.version_info >= (3, 0): p = plistlib.loads(s.encode("utf-8")) else: # p = plistlib.readPlistFromString(s) # We avoid using readPlistFromString() as that uses # cStringIO and fails when Unicode strings are detected # Don't subclass - keep the parser local from xml.parsers.expat import ParserCreate # Create a new PlistParser object - then we need to set up # the values and parse. pa = plistlib.PlistParser() # We also monkey patch this to encode unicode as utf-8 def end_string(): d = pa.getData() if isinstance(d,unicode): d = d.encode("utf-8") pa.addObject(d) pa.end_string = end_string parser = ParserCreate() parser.StartElementHandler = pa.handleBeginElement parser.EndElementHandler = pa.handleEndElement parser.CharacterDataHandler = pa.handleData if isinstance(s, unicode): # Encode unicode -> string; use utf-8 for safety s = s.encode("utf-8") # Parse the string parser.Parse(s, 1) p = pa.root except Exception as e: print(e) pass return p def _compare_versions(self, vers1, vers2, pad = -1): # Helper method to compare ##.## strings # # vers1 < vers2 = True # vers1 = vers2 = None # vers1 > vers2 = False # # Must be separated with a period # Sanitize the pads pad = -1 if not type(pad) is int else pad # Cast as strings vers1 = str(vers1) vers2 = str(vers2) # Split to lists v1_parts = vers1.split(".") v2_parts = vers2.split(".") # Equalize lengths if len(v1_parts) < len(v2_parts): v1_parts.extend([str(pad) for x in range(len(v2_parts) - len(v1_parts))]) elif len(v2_parts) < len(v1_parts): v2_parts.extend([str(pad) for x in range(len(v1_parts) - len(v2_parts))]) # Iterate and compare for i in range(len(v1_parts)): # Remove non-numeric v1 = ''.join(c for c in v1_parts[i] if c.isdigit()) v2 = ''.join(c for c in v2_parts[i] if c.isdigit()) # If empty - make it a pad var v1 = pad if not len(v1) else v1 v2 = pad if not len(v2) else v2 # Compare if int(v1) < int(v2): return True elif int(v1) > int(v2): return False # Never differed - return None, must be equal return None def update(self): self._update_disks() def _update_disks(self): self.disks = self.get_disks() self.disk_text = self.get_disk_text() if self._compare_versions("10.12", self.os_version): self.apfs = self.get_apfs() else: self.apfs = {} def get_diskutil(self): # Returns the path to the diskutil binary return self.r.run({"args":["which", "diskutil"]})[0].split("\n")[0].split("\r")[0] def get_disks(self): # Returns a dictionary object of connected disks disk_list = self.r.run({"args":[self.diskutil, "list", "-plist"]})[0] return self._get_plist(disk_list) def get_disk_text(self): # Returns plain text listing connected disks return self.r.run({"args":[self.diskutil, "list"]})[0] def get_disk_info(self, disk): disk_id = self.get_identifier(disk) if not disk_id: return None disk_list = self.r.run({"args":[self.diskutil, "info", "-plist", disk_id]})[0] return self._get_plist(disk_list) def get_disk_fs(self, disk): disk_id = self.get_identifier(disk) if not disk_id: return None return self.get_disk_info(disk_id).get("FilesystemName", None) def get_disk_fs_type(self, disk): disk_id = self.get_identifier(disk) if not disk_id: return None return self.get_disk_info(disk_id).get("FilesystemType", None) def get_apfs(self): # Returns a dictionary object of apfs disks output = self.r.run({"args":"echo y | " + self.diskutil + " apfs list -plist", "shell" : True}) if not output[2] == 0: # Error getting apfs info - return an empty dict return {} disk_list = output[0] p_list = disk_list.split(" 1: # We had text before the start - get only the plist info disk_list = "= 1), "B") # Determine our rounding approach - first make sure it's an int; default to 2 on error try:round_to=int(round_to) except:round_to=2 round_to = 0 if round_to < 0 else 15 if round_to > 15 else round_to # Ensure it's between 0 and 15 bval = round(s_dict[biggest], round_to) # Split our number based on decimal points a,b = str(bval).split(".") # Check if we need to strip or pad zeroes b = b.rstrip("0") if strip_zeroes else b.ljust(round_to,"0") if round_to > 0 else "" return "{:,}{} {}".format(int(a),"" if not b else "."+b,biggest) def _process_hook(queue, total_size, bytes_so_far=0, update_interval=1.0, max_packets=0): packets = [] speed = remaining = "" last_update = time.time() while True: # Write our info first so we have *some* status while # waiting for packets if total_size > 0: percent = float(bytes_so_far) / total_size percent = round(percent*100, 2) t_s = get_size(total_size) try: b_s = get_size(bytes_so_far, t_s.split(" ")[1]) except: b_s = get_size(bytes_so_far) perc_str = " {:.2f}%".format(percent) bar_width = (TERMINAL_WIDTH // 3)-len(perc_str) progress = "=" * int(bar_width * (percent/100)) sys.stdout.write("\r\033[K{}/{} | {}{}{}{}{}".format( b_s, t_s, progress, " " * (bar_width-len(progress)), perc_str, speed, remaining )) else: b_s = get_size(bytes_so_far) sys.stdout.write("\r\033[K{}{}".format(b_s, speed)) sys.stdout.flush() # Now we gather the next packet try: packet = queue.get(timeout=update_interval) # Packets should be formatted as a tuple of # (timestamp, len(bytes_downloaded)) # If "DONE" is passed, we assume the download # finished - and bail if packet == "DONE": print("") # Jump to the next line return # Append our packet to the list and ensure we're not # beyond our max. # Only check max if it's > 0 packets.append(packet) if max_packets > 0: packets = packets[-max_packets:] # Increment our bytes so far as well bytes_so_far += packet[1] except q.Empty: # Didn't get anything - reset the speed # and packets packets = [] speed = " | 0 B/s" remaining = " | ?? left" if total_size > 0 else "" except KeyboardInterrupt: print("") # Jump to the next line return # If we have packets and it's time for an update, process # the info. update_check = time.time() if packets and update_check - last_update >= update_interval: last_update = update_check # Refresh our update timestamp speed = " | ?? B/s" if len(packets) > 1: # Let's calculate the amount downloaded over how long try: first,last = packets[0][0],packets[-1][0] chunks = sum([float(x[1]) for x in packets]) t = last-first assert t >= 0 bytes_speed = 1. / t * chunks speed = " | {}/s".format(get_size(bytes_speed,round_to=1)) # Get our remaining time if total_size > 0: seconds_left = (total_size-bytes_so_far) / bytes_speed days = seconds_left // 86400 hours = (seconds_left - (days*86400)) // 3600 mins = (seconds_left - (days*86400) - (hours*3600)) // 60 secs = seconds_left - (days*86400) - (hours*3600) - (mins*60) if days > 99 or bytes_speed == 0: remaining = " | ?? left" else: remaining = " | {}{:02d}:{:02d}:{:02d} left".format( "{}:".format(int(days)) if days else "", int(hours), int(mins), int(round(secs)) ) except: pass # Clear the packets so we don't reuse the same ones packets = [] class Downloader: def __init__(self,**kwargs): self.ua = kwargs.get("useragent",{"User-Agent":"Mozilla"}) self.chunk = None # Auto-assign if None, otherwise explicit self.min_chunk = 1024 # 1 KiB min chunk size self.max_chunk = 1024 * 1024 * 4 # 4 MiB max chunk size self.chunk_rate = 0.1 # Update every 0.1 seconds self.chunk_growth = 1.5 # Max multiplier for chunk growth if os.name=="nt": os.system("color") # Initialize cmd for ANSI escapes # Provide reasonable default logic to workaround macOS CA file handling cafile = ssl.get_default_verify_paths().openssl_cafile try: # If default OpenSSL CA file does not exist, use that from certifi if not os.path.exists(cafile): import certifi cafile = certifi.where() self.ssl_context = ssl.create_default_context(cafile=cafile) except: # None of the above worked, disable certificate verification for now self.ssl_context = ssl._create_unverified_context() return def _decode(self, value, encoding="utf-8", errors="ignore"): # Helper method to only decode if bytes type if sys.version_info >= (3,0) and isinstance(value, bytes): return value.decode(encoding,errors) return value def _update_main_name(self): # Windows running python 2 seems to have issues with multiprocessing # if the case of the main script's name is incorrect: # e.g. Downloader.py vs downloader.py # # To work around this, we try to scrape for the correct case if # possible. try: path = os.path.abspath(sys.modules["__main__"].__file__) except AttributeError as e: # This likely means we're running from the interpreter # directly return None if not os.path.isfile(path): return None # Get the file name and folder path name = os.path.basename(path).lower() fldr = os.path.dirname(path) # Walk the files in the folder until we find our # name - then steal its case and update that path for f in os.listdir(fldr): if f.lower() == name: # Got it new_path = os.path.join(fldr,f) sys.modules["__main__"].__file__ = new_path return new_path # If we got here, it wasn't found return None def _get_headers(self, headers = None): # Fall back on the default ua if none provided target = headers if isinstance(headers,dict) else self.ua new_headers = {} # Shallow copy to prevent changes to the headers # overriding the original for k in target: new_headers[k] = target[k] return new_headers def open_url(self, url, headers = None): headers = self._get_headers(headers) # Wrap up the try/except block so we don't have to do this for each function try: response = urlopen(Request(url, headers=headers), context=self.ssl_context) except Exception as e: # No fixing this - bail return None return response def get_size(self, *args, **kwargs): return get_size(*args,**kwargs) def get_string(self, url, progress = True, headers = None, expand_gzip = True): response = self.get_bytes(url,progress,headers,expand_gzip) if response is None: return None return self._decode(response) def get_bytes(self, url, progress = True, headers = None, expand_gzip = True): response = self.open_url(url, headers) if response is None: return None try: total_size = int(response.headers['Content-Length']) except: total_size = -1 chunk_so_far = b"" packets = queue = process = None if progress: # Make sure our vars are initialized packets = [] if progress else None queue = multiprocessing.Queue() # Create the multiprocess and start it process = multiprocessing.Process( target=_process_hook, args=(queue,total_size) ) process.daemon = True # Filthy hack for earlier python versions on Windows if os.name == "nt" and hasattr(multiprocessing,"forking"): self._update_main_name() process.start() try: chunk_size = self.chunk or 1024 auto_chunk_size = not self.chunk while True: t = time.perf_counter() chunk = response.read(chunk_size) chunk_time = time.perf_counter()-t if progress: # Add our items to the queue queue.put((time.time(),len(chunk))) if not chunk: break chunk_so_far += chunk if auto_chunk_size: # Adjust our chunk size based on the internet speed at our defined rate chunk_rate = int(len(chunk) / chunk_time * self.chunk_rate) chunk_change_max = round(chunk_size * self.chunk_growth) chunk_rate_clamped = min(max(self.min_chunk, chunk_rate), chunk_change_max) chunk_size = min(chunk_rate_clamped, self.max_chunk) finally: # Close the response whenever we're done response.close() if expand_gzip and response.headers.get("Content-Encoding","unknown").lower() == "gzip": fileobj = BytesIO(chunk_so_far) gfile = gzip.GzipFile(fileobj=fileobj) return gfile.read() if progress: # Finalize the queue and wait queue.put("DONE") process.join() return chunk_so_far def stream_to_file(self, url, file_path, progress = True, headers = None, ensure_size_if_present = True, allow_resume = False): response = self.open_url(url, headers) if response is None: return None bytes_so_far = 0 try: total_size = int(response.headers['Content-Length']) except: total_size = -1 packets = queue = process = None mode = "wb" if allow_resume and os.path.isfile(file_path) and total_size != -1: # File exists, we're resuming and have a target size. Check the # local file size. current_size = os.stat(file_path).st_size if current_size == total_size: # File is already complete - return the path return file_path elif current_size < total_size: response.close() # File is not complete - seek to our current size bytes_so_far = current_size mode = "ab" # Append # We also need to try creating a new request # in order to pass our range header new_headers = self._get_headers(headers) # Get the start byte, 0-indexed byte_string = "bytes={}-".format(current_size) new_headers["Range"] = byte_string response = self.open_url(url, new_headers) if response is None: return None if progress: # Make sure our vars are initialized packets = [] if progress else None queue = multiprocessing.Queue() # Create the multiprocess and start it process = multiprocessing.Process( target=_process_hook, args=(queue,total_size,bytes_so_far) ) process.daemon = True # Filthy hack for earlier python versions on Windows if os.name == "nt" and hasattr(multiprocessing,"forking"): self._update_main_name() process.start() with open(file_path,mode) as f: chunk_size = self.chunk or 1024 auto_chunk_size = not self.chunk try: while True: t = time.perf_counter() chunk = response.read(chunk_size) chunk_time = time.perf_counter()-t bytes_so_far += len(chunk) if progress: # Add our items to the queue queue.put((time.time(),len(chunk))) if not chunk: break f.write(chunk) if auto_chunk_size: # Adjust our chunk size based on the internet speed at our defined rate chunk_rate = int(len(chunk) / chunk_time * self.chunk_rate) chunk_change_max = round(chunk_size * self.chunk_growth) chunk_rate_clamped = min(max(self.min_chunk, chunk_rate), chunk_change_max) chunk_size = min(chunk_rate_clamped, self.max_chunk) finally: # Close the response whenever we're done response.close() if progress: # Finalize the queue and wait queue.put("DONE") process.join() if ensure_size_if_present and total_size != -1: # We're verifying size - make sure we got what we asked for if bytes_so_far != total_size: return None # We didn't - imply it failed return file_path if os.path.exists(file_path) else None ================================================ FILE: Scripts/plist.py ================================================ ### ### # Imports # ### ### import datetime, os, plistlib, struct, sys, itertools, binascii from io import BytesIO if sys.version_info < (3,0): # Force use of StringIO instead of cStringIO as the latter # has issues with Unicode strings from StringIO import StringIO else: from io import StringIO try: basestring # Python 2 unicode except NameError: basestring = str # Python 3 unicode = str try: FMT_XML = plistlib.FMT_XML FMT_BINARY = plistlib.FMT_BINARY except AttributeError: FMT_XML = "FMT_XML" FMT_BINARY = "FMT_BINARY" ### ### # Helper Methods # ### ### def wrap_data(value): if not _check_py3(): return plistlib.Data(value) return value def extract_data(value): if not _check_py3() and isinstance(value,plistlib.Data): return value.data return value def _check_py3(): return sys.version_info >= (3, 0) def _is_binary(fp): if isinstance(fp, basestring): return fp.startswith(b"bplist00") header = fp.read(32) fp.seek(0) return header[:8] == b'bplist00' def _seek_past_whitespace(fp): offset = 0 while True: byte = fp.read(1) if not byte: # End of file, reset offset and bail offset = 0 break if not byte.isspace(): # Found our first non-whitespace character break offset += 1 # Seek to the first non-whitespace char fp.seek(offset) return offset ### ### # Deprecated Functions - Remapped # ### ### def readPlist(pathOrFile): if not isinstance(pathOrFile, basestring): return load(pathOrFile) with open(pathOrFile, "rb") as f: return load(f) def writePlist(value, pathOrFile): if not isinstance(pathOrFile, basestring): return dump(value, pathOrFile, fmt=FMT_XML, sort_keys=True, skipkeys=False) with open(pathOrFile, "wb") as f: return dump(value, f, fmt=FMT_XML, sort_keys=True, skipkeys=False) ### ### # Remapped Functions # ### ### def load(fp, fmt=None, use_builtin_types=None, dict_type=dict): if _is_binary(fp): use_builtin_types = False if use_builtin_types is None else use_builtin_types try: p = _BinaryPlistParser(use_builtin_types=use_builtin_types, dict_type=dict_type) except: # Python 3.9 removed use_builtin_types p = _BinaryPlistParser(dict_type=dict_type) return p.parse(fp) elif _check_py3(): offset = _seek_past_whitespace(fp) use_builtin_types = True if use_builtin_types is None else use_builtin_types # We need to monkey patch this to allow for hex integers - code taken/modified from # https://github.com/python/cpython/blob/3.8/Lib/plistlib.py if fmt is None: header = fp.read(32) fp.seek(offset) for info in plistlib._FORMATS.values(): if info['detect'](header): P = info['parser'] break else: raise plistlib.InvalidFileException() else: P = plistlib._FORMATS[fmt]['parser'] try: p = P(use_builtin_types=use_builtin_types, dict_type=dict_type) except: # Python 3.9 removed use_builtin_types p = P(dict_type=dict_type) if isinstance(p,plistlib._PlistParser): # Monkey patch! def end_integer(): d = p.get_data() value = int(d,16) if d.lower().startswith("0x") else int(d) if -1 << 63 <= value < 1 << 64: p.add_object(value) else: raise OverflowError("Integer overflow at line {}".format(p.parser.CurrentLineNumber)) def end_data(): try: p.add_object(plistlib._decode_base64(p.get_data())) except Exception as e: raise Exception("Data error at line {}: {}".format(p.parser.CurrentLineNumber,e)) p.end_integer = end_integer p.end_data = end_data return p.parse(fp) else: offset = _seek_past_whitespace(fp) # Is not binary - assume a string - and try to load # We avoid using readPlistFromString() as that uses # cStringIO and fails when Unicode strings are detected # Don't subclass - keep the parser local from xml.parsers.expat import ParserCreate # Create a new PlistParser object - then we need to set up # the values and parse. p = plistlib.PlistParser() parser = ParserCreate() parser.StartElementHandler = p.handleBeginElement parser.EndElementHandler = p.handleEndElement parser.CharacterDataHandler = p.handleData # We also need to monkey patch this to allow for other dict_types, hex int support # proper line output for data errors, and for unicode string decoding def begin_dict(attrs): d = dict_type() p.addObject(d) p.stack.append(d) def end_integer(): d = p.getData() value = int(d,16) if d.lower().startswith("0x") else int(d) if -1 << 63 <= value < 1 << 64: p.addObject(value) else: raise OverflowError("Integer overflow at line {}".format(parser.CurrentLineNumber)) def end_data(): try: p.addObject(plistlib.Data.fromBase64(p.getData())) except Exception as e: raise Exception("Data error at line {}: {}".format(parser.CurrentLineNumber,e)) def end_string(): d = p.getData() if isinstance(d,unicode): d = d.encode("utf-8") p.addObject(d) p.begin_dict = begin_dict p.end_integer = end_integer p.end_data = end_data p.end_string = end_string if isinstance(fp, unicode): # Encode unicode -> string; use utf-8 for safety fp = fp.encode("utf-8") if isinstance(fp, basestring): # It's a string - let's wrap it up fp = StringIO(fp) # Parse it parser.ParseFile(fp) return p.root def loads(value, fmt=None, use_builtin_types=None, dict_type=dict): if _check_py3() and isinstance(value, basestring): # If it's a string - encode it value = value.encode() try: return load(BytesIO(value),fmt=fmt,use_builtin_types=use_builtin_types,dict_type=dict_type) except: # Python 3.9 removed use_builtin_types return load(BytesIO(value),fmt=fmt,dict_type=dict_type) def dump(value, fp, fmt=FMT_XML, sort_keys=True, skipkeys=False): if fmt == FMT_BINARY: # Assume binary at this point writer = _BinaryPlistWriter(fp, sort_keys=sort_keys, skipkeys=skipkeys) writer.write(value) elif fmt == FMT_XML: if _check_py3(): plistlib.dump(value, fp, fmt=fmt, sort_keys=sort_keys, skipkeys=skipkeys) else: # We need to monkey patch a bunch here too in order to avoid auto-sorting # of keys writer = plistlib.PlistWriter(fp) def writeDict(d): if d: writer.beginElement("dict") items = sorted(d.items()) if sort_keys else d.items() for key, value in items: if not isinstance(key, basestring): if skipkeys: continue raise TypeError("keys must be strings") writer.simpleElement("key", key) writer.writeValue(value) writer.endElement("dict") else: writer.simpleElement("dict") writer.writeDict = writeDict writer.writeln("") writer.writeValue(value) writer.writeln("") else: # Not a proper format raise ValueError("Unsupported format: {}".format(fmt)) def dumps(value, fmt=FMT_XML, skipkeys=False, sort_keys=True): # We avoid using writePlistToString() as that uses # cStringIO and fails when Unicode strings are detected f = BytesIO() if _check_py3() else StringIO() dump(value, f, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys) value = f.getvalue() if _check_py3(): value = value.decode("utf-8") return value ### ### # Binary Plist Stuff For Py2 # ### ### # From the python 3 plistlib.py source: https://github.com/python/cpython/blob/3.11/Lib/plistlib.py # Tweaked to function on both Python 2 and 3 class UID: def __init__(self, data): if not isinstance(data, int): raise TypeError("data must be an int") # It seems Apple only uses 32-bit unsigned ints for UIDs. Although the comment in # CoreFoundation's CFBinaryPList.c detailing the binary plist format theoretically # allows for 64-bit UIDs, most functions in the same file use 32-bit unsigned ints, # with the sole function hinting at 64-bits appearing to be a leftover from copying # and pasting integer handling code internally, and this code has not changed since # it was added. (In addition, code in CFPropertyList.c to handle CF$UID also uses a # 32-bit unsigned int.) # # if data >= 1 << 64: # raise ValueError("UIDs cannot be >= 2**64") if data >= 1 << 32: raise ValueError("UIDs cannot be >= 2**32 (4294967296)") if data < 0: raise ValueError("UIDs must be positive") self.data = data def __index__(self): return self.data def __repr__(self): return "%s(%s)" % (self.__class__.__name__, repr(self.data)) def __reduce__(self): return self.__class__, (self.data,) def __eq__(self, other): if not isinstance(other, UID): return NotImplemented return self.data == other.data def __hash__(self): return hash(self.data) class InvalidFileException (ValueError): def __init__(self, message="Invalid file"): ValueError.__init__(self, message) _BINARY_FORMAT = {1: 'B', 2: 'H', 4: 'L', 8: 'Q'} _undefined = object() class _BinaryPlistParser: """ Read or write a binary plist file, following the description of the binary format. Raise InvalidFileException in case of error, otherwise return the root object. see also: http://opensource.apple.com/source/CF/CF-744.18/CFBinaryPList.c """ def __init__(self, use_builtin_types, dict_type): self._use_builtin_types = use_builtin_types self._dict_type = dict_type def parse(self, fp): try: # The basic file format: # HEADER # object... # refid->offset... # TRAILER self._fp = fp self._fp.seek(-32, os.SEEK_END) trailer = self._fp.read(32) if len(trailer) != 32: raise InvalidFileException() ( offset_size, self._ref_size, num_objects, top_object, offset_table_offset ) = struct.unpack('>6xBBQQQ', trailer) self._fp.seek(offset_table_offset) self._object_offsets = self._read_ints(num_objects, offset_size) self._objects = [_undefined] * num_objects return self._read_object(top_object) except (OSError, IndexError, struct.error, OverflowError, UnicodeDecodeError): raise InvalidFileException() def _get_size(self, tokenL): """ return the size of the next object.""" if tokenL == 0xF: m = self._fp.read(1)[0] if not _check_py3(): m = ord(m) m = m & 0x3 s = 1 << m f = '>' + _BINARY_FORMAT[s] return struct.unpack(f, self._fp.read(s))[0] return tokenL def _read_ints(self, n, size): data = self._fp.read(size * n) if size in _BINARY_FORMAT: return struct.unpack('>' + _BINARY_FORMAT[size] * n, data) else: if not size or len(data) != size * n: raise InvalidFileException() return tuple(int(binascii.hexlify(data[i: i + size]),16) for i in range(0, size * n, size)) '''return tuple(int.from_bytes(data[i: i + size], 'big') for i in range(0, size * n, size))''' def _read_refs(self, n): return self._read_ints(n, self._ref_size) def _read_object(self, ref): """ read the object by reference. May recursively read sub-objects (content of an array/dict/set) """ result = self._objects[ref] if result is not _undefined: return result offset = self._object_offsets[ref] self._fp.seek(offset) token = self._fp.read(1)[0] if not _check_py3(): token = ord(token) tokenH, tokenL = token & 0xF0, token & 0x0F if token == 0x00: # \x00 or 0x00 result = None elif token == 0x08: # \x08 or 0x08 result = False elif token == 0x09: # \x09 or 0x09 result = True # The referenced source code also mentions URL (0x0c, 0x0d) and # UUID (0x0e), but neither can be generated using the Cocoa libraries. elif token == 0x0f: # \x0f or 0x0f result = b'' elif tokenH == 0x10: # int result = int(binascii.hexlify(self._fp.read(1 << tokenL)),16) if tokenL >= 3: # Signed - adjust result = result-(result & 1 << 2**tokenL*8-1)*2 elif token == 0x22: # real result = struct.unpack('>f', self._fp.read(4))[0] elif token == 0x23: # real result = struct.unpack('>d', self._fp.read(8))[0] elif token == 0x33: # date f = struct.unpack('>d', self._fp.read(8))[0] # timestamp 0 of binary plists corresponds to 1/1/2001 # (year of Mac OS X 10.0), instead of 1/1/1970. result = (datetime.datetime(2001, 1, 1) + datetime.timedelta(seconds=f)) elif tokenH == 0x40: # data s = self._get_size(tokenL) if self._use_builtin_types or not hasattr(plistlib, "Data"): result = self._fp.read(s) else: result = plistlib.Data(self._fp.read(s)) elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) result = self._fp.read(s).decode('ascii') result = result elif tokenH == 0x60: # unicode string s = self._get_size(tokenL) result = self._fp.read(s * 2).decode('utf-16be') elif tokenH == 0x80: # UID # used by Key-Archiver plist files result = UID(int(binascii.hexlify(self._fp.read(1 + tokenL)),16)) elif tokenH == 0xA0: # array s = self._get_size(tokenL) obj_refs = self._read_refs(s) result = [] self._objects[ref] = result result.extend(self._read_object(x) for x in obj_refs) # tokenH == 0xB0 is documented as 'ordset', but is not actually # implemented in the Apple reference code. # tokenH == 0xC0 is documented as 'set', but sets cannot be used in # plists. elif tokenH == 0xD0: # dict s = self._get_size(tokenL) key_refs = self._read_refs(s) obj_refs = self._read_refs(s) result = self._dict_type() self._objects[ref] = result for k, o in zip(key_refs, obj_refs): key = self._read_object(k) if hasattr(plistlib, "Data") and isinstance(key, plistlib.Data): key = key.data result[key] = self._read_object(o) else: raise InvalidFileException() self._objects[ref] = result return result def _count_to_size(count): if count < 1 << 8: return 1 elif count < 1 << 16: return 2 elif count < 1 << 32: return 4 else: return 8 _scalars = (str, int, float, datetime.datetime, bytes) class _BinaryPlistWriter (object): def __init__(self, fp, sort_keys, skipkeys): self._fp = fp self._sort_keys = sort_keys self._skipkeys = skipkeys def write(self, value): # Flattened object list: self._objlist = [] # Mappings from object->objectid # First dict has (type(object), object) as the key, # second dict is used when object is not hashable and # has id(object) as the key. self._objtable = {} self._objidtable = {} # Create list of all objects in the plist self._flatten(value) # Size of object references in serialized containers # depends on the number of objects in the plist. num_objects = len(self._objlist) self._object_offsets = [0]*num_objects self._ref_size = _count_to_size(num_objects) self._ref_format = _BINARY_FORMAT[self._ref_size] # Write file header self._fp.write(b'bplist00') # Write object list for obj in self._objlist: self._write_object(obj) # Write refnum->object offset table top_object = self._getrefnum(value) offset_table_offset = self._fp.tell() offset_size = _count_to_size(offset_table_offset) offset_format = '>' + _BINARY_FORMAT[offset_size] * num_objects self._fp.write(struct.pack(offset_format, *self._object_offsets)) # Write trailer sort_version = 0 trailer = ( sort_version, offset_size, self._ref_size, num_objects, top_object, offset_table_offset ) self._fp.write(struct.pack('>5xBBBQQQ', *trailer)) def _flatten(self, value): # First check if the object is in the object table, not used for # containers to ensure that two subcontainers with the same contents # will be serialized as distinct values. if isinstance(value, _scalars): if (type(value), value) in self._objtable: return elif hasattr(plistlib, "Data") and isinstance(value, plistlib.Data): if (type(value.data), value.data) in self._objtable: return elif id(value) in self._objidtable: return # Add to objectreference map refnum = len(self._objlist) self._objlist.append(value) if isinstance(value, _scalars): self._objtable[(type(value), value)] = refnum elif hasattr(plistlib, "Data") and isinstance(value, plistlib.Data): self._objtable[(type(value.data), value.data)] = refnum else: self._objidtable[id(value)] = refnum # And finally recurse into containers if isinstance(value, dict): keys = [] values = [] items = value.items() if self._sort_keys: items = sorted(items) for k, v in items: if not isinstance(k, basestring): if self._skipkeys: continue raise TypeError("keys must be strings") keys.append(k) values.append(v) for o in itertools.chain(keys, values): self._flatten(o) elif isinstance(value, (list, tuple)): for o in value: self._flatten(o) def _getrefnum(self, value): if isinstance(value, _scalars): return self._objtable[(type(value), value)] elif hasattr(plistlib, "Data") and isinstance(value, plistlib.Data): return self._objtable[(type(value.data), value.data)] else: return self._objidtable[id(value)] def _write_size(self, token, size): if size < 15: self._fp.write(struct.pack('>B', token | size)) elif size < 1 << 8: self._fp.write(struct.pack('>BBB', token | 0xF, 0x10, size)) elif size < 1 << 16: self._fp.write(struct.pack('>BBH', token | 0xF, 0x11, size)) elif size < 1 << 32: self._fp.write(struct.pack('>BBL', token | 0xF, 0x12, size)) else: self._fp.write(struct.pack('>BBQ', token | 0xF, 0x13, size)) def _write_object(self, value): ref = self._getrefnum(value) self._object_offsets[ref] = self._fp.tell() if value is None: self._fp.write(b'\x00') elif value is False: self._fp.write(b'\x08') elif value is True: self._fp.write(b'\x09') elif isinstance(value, int): if value < 0: try: self._fp.write(struct.pack('>Bq', 0x13, value)) except struct.error: raise OverflowError(value) # from None elif value < 1 << 8: self._fp.write(struct.pack('>BB', 0x10, value)) elif value < 1 << 16: self._fp.write(struct.pack('>BH', 0x11, value)) elif value < 1 << 32: self._fp.write(struct.pack('>BL', 0x12, value)) elif value < 1 << 63: self._fp.write(struct.pack('>BQ', 0x13, value)) elif value < 1 << 64: self._fp.write(binascii.unhexlify("14"+hex(value)[2:].rstrip("L").rjust(32,"0"))) else: raise OverflowError(value) elif isinstance(value, float): self._fp.write(struct.pack('>Bd', 0x23, value)) elif isinstance(value, datetime.datetime): f = (value - datetime.datetime(2001, 1, 1)).total_seconds() self._fp.write(struct.pack('>Bd', 0x33, f)) elif (_check_py3() and isinstance(value, (bytes, bytearray))) or (hasattr(plistlib, "Data") and isinstance(value, plistlib.Data)): if not isinstance(value, (bytes, bytearray)): value = value.data # Unpack it self._write_size(0x40, len(value)) self._fp.write(value) elif isinstance(value, basestring): try: t = value.encode('ascii') self._write_size(0x50, len(value)) except UnicodeEncodeError: t = value.encode('utf-16be') self._write_size(0x60, len(t) // 2) self._fp.write(t) elif isinstance(value, UID) or (hasattr(plistlib,"UID") and isinstance(value, plistlib.UID)): if value.data < 0: raise ValueError("UIDs must be positive") elif value.data < 1 << 8: self._fp.write(struct.pack('>BB', 0x80, value)) elif value.data < 1 << 16: self._fp.write(struct.pack('>BH', 0x81, value)) elif value.data < 1 << 32: self._fp.write(struct.pack('>BL', 0x83, value)) # elif value.data < 1 << 64: # self._fp.write(struct.pack('>BQ', 0x87, value)) else: raise OverflowError(value) elif isinstance(value, (list, tuple)): refs = [self._getrefnum(o) for o in value] s = len(refs) self._write_size(0xA0, s) self._fp.write(struct.pack('>' + self._ref_format * s, *refs)) elif isinstance(value, dict): keyRefs, valRefs = [], [] if self._sort_keys: rootItems = sorted(value.items()) else: rootItems = value.items() for k, v in rootItems: if not isinstance(k, basestring): if self._skipkeys: continue raise TypeError("keys must be strings") keyRefs.append(self._getrefnum(k)) valRefs.append(self._getrefnum(v)) s = len(keyRefs) self._write_size(0xD0, s) self._fp.write(struct.pack('>' + self._ref_format * s, *keyRefs)) self._fp.write(struct.pack('>' + self._ref_format * s, *valRefs)) else: raise TypeError(value) ================================================ FILE: Scripts/run.py ================================================ import sys, subprocess, time, threading, shlex try: from Queue import Queue, Empty except: from queue import Queue, Empty ON_POSIX = 'posix' in sys.builtin_module_names class Run: def __init__(self): return def _read_output(self, pipe, q): try: for line in iter(lambda: pipe.read(1), b''): q.put(line) except ValueError: pass pipe.close() def _create_thread(self, output): # Creates a new queue and thread object to watch based on the output pipe sent q = Queue() t = threading.Thread(target=self._read_output, args=(output, q)) t.daemon = True return (q,t) def _stream_output(self, comm, shell = False): output = error = "" p = None try: if shell and type(comm) is list: comm = " ".join(shlex.quote(x) for x in comm) if not shell and type(comm) is str: comm = shlex.split(comm) p = subprocess.Popen(comm, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0, universal_newlines=True, close_fds=ON_POSIX) # Setup the stdout thread/queue q,t = self._create_thread(p.stdout) qe,te = self._create_thread(p.stderr) # Start both threads t.start() te.start() while True: c = z = "" try: c = q.get_nowait() except Empty: pass else: sys.stdout.write(c) output += c sys.stdout.flush() try: z = qe.get_nowait() except Empty: pass else: sys.stderr.write(z) error += z sys.stderr.flush() if not c==z=="": continue # Keep going until empty # No output - see if still running p.poll() if p.returncode != None: # Subprocess ended break # No output, but subprocess still running - stall for 20ms time.sleep(0.02) o, e = p.communicate() return (output+o, error+e, p.returncode) except: if p: try: o, e = p.communicate() except: o = e = "" return (output+o, error+e, p.returncode) return ("", "Command not found!", 1) def _decode(self, value, encoding="utf-8", errors="ignore"): # Helper method to only decode if bytes type if sys.version_info >= (3,0) and isinstance(value, bytes): return value.decode(encoding,errors) return value def _run_command(self, comm, shell = False): c = None try: if shell and type(comm) is list: comm = " ".join(shlex.quote(x) for x in comm) if not shell and type(comm) is str: comm = shlex.split(comm) p = subprocess.Popen(comm, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE) c = p.communicate() except: if c == None: return ("", "Command not found!", 1) return (self._decode(c[0]), self._decode(c[1]), p.returncode) def run(self, command_list, leave_on_fail = False): # Command list should be an array of dicts if type(command_list) is dict: # We only have one command command_list = [command_list] output_list = [] for comm in command_list: args = comm.get("args", []) shell = comm.get("shell", False) stream = comm.get("stream", False) sudo = comm.get("sudo", False) stdout = comm.get("stdout", False) stderr = comm.get("stderr", False) mess = comm.get("message", None) show = comm.get("show", False) if not mess == None: print(mess) if not len(args): # nothing to process continue if sudo: # Check if we have sudo out = self._run_command(["which", "sudo"]) if "sudo" in out[0]: # Can sudo if type(args) is list: args.insert(0, out[0].replace("\n", "")) # add to start of list elif type(args) is str: args = out[0].replace("\n", "") + " " + args # add to start of string if show: print(" ".join(args)) if stream: # Stream it! out = self._stream_output(args, shell) else: # Just run and gather output out = self._run_command(args, shell) if stdout and len(out[0]): print(out[0]) if stderr and len(out[1]): print(out[1]) # Append output output_list.append(out) # Check for errors if leave_on_fail and out[2] != 0: # Got an error - leave break if len(output_list) == 1: # We only ran one command - just return that output return output_list[0] return output_list ================================================ FILE: Scripts/utils.py ================================================ import sys, os, time, re, json, datetime, ctypes, subprocess if os.name == "nt": # Windows import msvcrt else: # Not Windows \o/ import select class Utils: def __init__(self, name = "Python Script", interactive = True): self.name = name self.interactive = interactive # Init our colors before we need to print anything cwd = os.getcwd() os.chdir(os.path.dirname(os.path.realpath(__file__))) if os.path.exists("colors.json"): self.colors_dict = json.load(open("colors.json")) else: self.colors_dict = {} os.chdir(cwd) def check_admin(self): # Returns whether or not we're admin try: is_admin = os.getuid() == 0 except AttributeError: is_admin = ctypes.windll.shell32.IsUserAnAdmin() != 0 return is_admin def elevate(self, file): # Runs the passed file as admin if self.check_admin(): return if os.name == "nt": ctypes.windll.shell32.ShellExecuteW(None, "runas", '"{}"'.format(sys.executable), '"{}"'.format(file), None, 1) else: try: p = subprocess.Popen(["which", "sudo"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) c = p.communicate()[0].decode("utf-8", "ignore").replace("\n", "") os.execv(c, [ sys.executable, 'python'] + sys.argv) except: exit(1) def compare_versions(self, vers1, vers2, **kwargs): # Helper method to compare ##.## strings # # vers1 < vers2 = True # vers1 = vers2 = None # vers1 > vers2 = False # Sanitize the pads pad = str(kwargs.get("pad", "")) sep = str(kwargs.get("separator", ".")) ignore_case = kwargs.get("ignore_case", True) # Cast as strings vers1 = str(vers1) vers2 = str(vers2) if ignore_case: vers1 = vers1.lower() vers2 = vers2.lower() # Split and pad lists v1_parts, v2_parts = self.pad_length(vers1.split(sep), vers2.split(sep)) # Iterate and compare for i in range(len(v1_parts)): # Remove non-numeric v1 = ''.join(c.lower() for c in v1_parts[i] if c.isalnum()) v2 = ''.join(c.lower() for c in v2_parts[i] if c.isalnum()) # Equalize the lengths v1, v2 = self.pad_length(v1, v2) # Compare if str(v1) < str(v2): return True elif str(v1) > str(v2): return False # Never differed - return None, must be equal return None def pad_length(self, var1, var2, pad = "0"): # Pads the vars on the left side to make them equal length pad = "0" if len(str(pad)) < 1 else str(pad)[0] if not type(var1) == type(var2): # Type mismatch! Just return what we got return (var1, var2) if len(var1) < len(var2): if type(var1) is list: var1.extend([str(pad) for x in range(len(var2) - len(var1))]) else: var1 = "{}{}".format((pad*(len(var2)-len(var1))), var1) elif len(var2) < len(var1): if type(var2) is list: var2.extend([str(pad) for x in range(len(var1) - len(var2))]) else: var2 = "{}{}".format((pad*(len(var1)-len(var2))), var2) return (var1, var2) def check_path(self, path): # Let's loop until we either get a working path, or no changes test_path = path last_path = None while True: # Bail if we've looped at least once and the path didn't change if last_path != None and last_path == test_path: return None last_path = test_path # Check if we stripped everything out if not len(test_path): return None # Check if we have a valid path if os.path.exists(test_path): return os.path.abspath(test_path) # Check for quotes if test_path[0] == test_path[-1] and test_path[0] in ('"',"'"): test_path = test_path[1:-1] continue # Check for a tilde and expand if needed if test_path[0] == "~": tilde_expanded = os.path.expanduser(test_path) if tilde_expanded != test_path: # Got a change test_path = tilde_expanded continue # Let's check for spaces - strip from the left first, then the right if test_path[0] in (" ","\t"): test_path = test_path[1:] continue if test_path[-1] in (" ","\t"): test_path = test_path[:-1] continue # Maybe we have escapes to handle? test_path = "\\".join([x.replace("\\", "") for x in test_path.split("\\\\")]) def grab(self, prompt, **kwargs): # Takes a prompt, a default, and a timeout and shows it with that timeout # returning the result timeout = kwargs.get("timeout",0) default = kwargs.get("default","") if not self.interactive: return default # If we don't have a timeout - then skip the timed sections if timeout <= 0: try: if sys.version_info >= (3, 0): return input(prompt) else: return str(raw_input(prompt)) except EOFError: return default # Write our prompt sys.stdout.write(prompt) sys.stdout.flush() if os.name == "nt": start_time = time.time() i = '' while True: if msvcrt.kbhit(): c = msvcrt.getche() if ord(c) == 13: # enter_key break elif ord(c) >= 32: # space_char i += c.decode() if sys.version_info >= (3,0) and isinstance(c,bytes) else c else: time.sleep(0.02) # Delay for 20ms to prevent CPU workload if len(i) == 0 and (time.time() - start_time) > timeout: break else: i, o, e = select.select( [sys.stdin], [], [], timeout ) if i: i = sys.stdin.readline().strip() print('') # needed to move to next line if len(i) > 0: return i else: return default def cls(self): if not self.interactive: return if os.name == "nt": os.system("cls") elif os.environ.get("TERM"): os.system("clear") def cprint(self, message, **kwargs): strip_colors = kwargs.get("strip_colors", False) if os.name == "nt" or not self.interactive: strip_colors = True reset = u"\u001b[0m" # Requires sys import for c in self.colors: if strip_colors: message = message.replace(c["find"], "") else: message = message.replace(c["find"], c["replace"]) if strip_colors: return message sys.stdout.write(message) print(reset) # Needs work to resize the string if color chars exist '''# Header drawing method def head(self, text = None, width = 55): if text == None: text = self.name self.cls() print(" {}".format("#"*width)) len_text = self.cprint(text, strip_colors=True) mid_len = int(round(width/2-len(len_text)/2)-2) middle = " #{}{}{}#".format(" "*mid_len, len_text, " "*((width - mid_len - len(len_text))-2)) if len(middle) > width+1: # Get the difference di = len(middle) - width # Add the padding for the ...# di += 3 # Trim the string middle = middle[:-di] newlen = len(middle) middle += "...#" find_list = [ c["find"] for c in self.colors ] # Translate colored string to len middle = middle.replace(len_text, text + self.rt_color) # always reset just in case self.cprint(middle) print("#"*width)''' # Header drawing method def head(self, text = None, width = 55): if not self.interactive: sys.stderr.write(str(text)+"\n") sys.stderr.flush() return if text is None: text = self.name self.cls() print(" {}".format("#"*width)) mid_len = int(round(width/2-len(text)/2)-2) middle = " #{}{}{}#".format(" "*mid_len, text, " "*((width - mid_len - len(text))-2)) if len(middle) > width+1: # Get the difference di = len(middle) - width # Add the padding for the ...# di += 3 # Trim the string middle = middle[:-di] + "...#" print(middle) print("#"*width) print("") def info(self, text): if self.interactive: print(text) else: sys.stderr.write(str(text)+"\n") sys.stderr.flush() def resize(self, width, height): print('\033[8;{};{}t'.format(height, width)) def custom_quit(self): self.head() print("by CorpNewt\n") print("Thanks for testing it out, for bugs/comments/complaints") print("send me a message on Reddit, or check out my GitHub:\n") print("www.reddit.com/u/corpnewt") print("www.github.com/corpnewt\n") # Get the time and wish them a good morning, afternoon, evening, and night hr = datetime.datetime.now().time().hour if hr > 3 and hr < 12: print("Have a nice morning!\n\n") elif hr >= 12 and hr < 17: print("Have a nice afternoon!\n\n") elif hr >= 17 and hr < 21: print("Have a nice evening!\n\n") else: print("Have a nice night!\n\n") exit(0) ================================================ FILE: gibMacOS.bat ================================================ @echo off REM Get our local path and args before delayed expansion - allows % and ! set "thisDir=%~dp0" set "args=%*" setlocal enableDelayedExpansion REM Setup initial vars set "script_name=" set /a tried=0 set "toask=yes" set "pause_on_error=yes" set "py2v=" set "py2path=" set "py3v=" set "py3path=" set "pypath=" set "targetpy=3" REM use_py3: REM TRUE = Use if found, use py2 otherwise REM FALSE = Use py2 REM FORCE = Use py3 set "use_py3=TRUE" REM We'll parse if the first argument passed is REM --install-python and if so, we'll just install REM Can optionally take a version number as the REM second arg - i.e. --install-python 3.13.1 set "just_installing=FALSE" set "user_provided=" REM Get the system32 (or equivalent) path call :getsyspath "syspath" REM Make sure the syspath exists if "!syspath!" == "" ( if exist "%SYSTEMROOT%\system32\cmd.exe" ( if exist "%SYSTEMROOT%\system32\reg.exe" ( if exist "%SYSTEMROOT%\system32\where.exe" ( REM Fall back on the default path if it exists set "ComSpec=%SYSTEMROOT%\system32\cmd.exe" set "syspath=%SYSTEMROOT%\system32\" ) ) ) if "!syspath!" == "" ( cls echo ### ### echo # Missing Required Files # echo ### ### echo. echo Could not locate cmd.exe, reg.exe, or where.exe echo. echo Please ensure your ComSpec environment variable is properly configured and echo points directly to cmd.exe, then try again. echo. echo Current CompSpec Value: "%ComSpec%" echo. echo Press [enter] to quit. pause > nul exit /b 1 ) ) if "%~1" == "--install-python" ( set "just_installing=TRUE" set "user_provided=%~2" goto installpy ) goto checkscript :checkscript REM Check for our script first set "looking_for=!script_name!" if "!script_name!" == "" ( set "looking_for=%~n0.py or %~n0.command" set "script_name=%~n0.py" if not exist "!thisDir!\!script_name!" ( set "script_name=%~n0.command" ) ) if not exist "!thisDir!\!script_name!" ( cls echo ### ### echo # Target Not Found # echo ### ### echo. echo Could not find !looking_for!. echo Please make sure to run this script from the same directory echo as !looking_for!. echo. echo Press [enter] to quit. pause > nul exit /b 1 ) goto checkpy :checkpy call :updatepath for /f "USEBACKQ tokens=*" %%x in (`!syspath!where.exe python 2^> nul`) do ( call :checkpyversion "%%x" "py2v" "py2path" "py3v" "py3path" ) for /f "USEBACKQ tokens=*" %%x in (`!syspath!where.exe python3 2^> nul`) do ( call :checkpyversion "%%x" "py2v" "py2path" "py3v" "py3path" ) for /f "USEBACKQ tokens=*" %%x in (`!syspath!where.exe py 2^> nul`) do ( call :checkpylauncher "%%x" "py2v" "py2path" "py3v" "py3path" ) REM Walk our returns to see if we need to install if /i "!use_py3!" == "FALSE" ( set "targetpy=2" set "pypath=!py2path!" ) else if /i "!use_py3!" == "FORCE" ( set "pypath=!py3path!" ) else if /i "!use_py3!" == "TRUE" ( set "pypath=!py3path!" if "!pypath!" == "" set "pypath=!py2path!" ) if not "!pypath!" == "" ( goto runscript ) if !tried! lss 1 ( if /i "!toask!"=="yes" ( REM Better ask permission first goto askinstall ) else ( goto installpy ) ) else ( cls echo ### ### echo # Python Not Found # echo ### ### echo. REM Couldn't install for whatever reason - give the error message echo Python is not installed or not found in your PATH var. echo Please go to https://www.python.org/downloads/windows/ to echo download and install the latest version, then try again. echo. echo Make sure you check the box labeled: echo. echo "Add Python X.X to PATH" echo. echo Where X.X is the py version you're installing. echo. echo Press [enter] to quit. pause > nul exit /b 1 ) goto runscript :checkpylauncher REM Attempt to check the latest python 2 and 3 versions via the py launcher for /f "USEBACKQ tokens=*" %%x in (`%~1 -2 -c "import sys; print(sys.executable)" 2^> nul`) do ( call :checkpyversion "%%x" "%~2" "%~3" "%~4" "%~5" ) for /f "USEBACKQ tokens=*" %%x in (`%~1 -3 -c "import sys; print(sys.executable)" 2^> nul`) do ( call :checkpyversion "%%x" "%~2" "%~3" "%~4" "%~5" ) goto :EOF :checkpyversion set "version="&for /f "tokens=2* USEBACKQ delims= " %%a in (`"%~1" -V 2^>^&1`) do ( REM Ensure we have a version number call :isnumber "%%a" if not "!errorlevel!" == "0" goto :EOF set "version=%%a" ) if not defined version goto :EOF if "!version:~0,1!" == "2" ( REM Python 2 call :comparepyversion "!version!" "!%~2!" if "!errorlevel!" == "1" ( set "%~2=!version!" set "%~3=%~1" ) ) else ( REM Python 3 call :comparepyversion "!version!" "!%~4!" if "!errorlevel!" == "1" ( set "%~4=!version!" set "%~5=%~1" ) ) goto :EOF :isnumber set "var="&for /f "delims=0123456789." %%i in ("%~1") do set var=%%i if defined var (exit /b 1) exit /b 0 :comparepyversion REM Exits with status 0 if equal, 1 if v1 gtr v2, 2 if v1 lss v2 for /f "tokens=1,2,3 delims=." %%a in ("%~1") do ( set a1=%%a set a2=%%b set a3=%%c ) for /f "tokens=1,2,3 delims=." %%a in ("%~2") do ( set b1=%%a set b2=%%b set b3=%%c ) if not defined a1 set a1=0 if not defined a2 set a2=0 if not defined a3 set a3=0 if not defined b1 set b1=0 if not defined b2 set b2=0 if not defined b3 set b3=0 if %a1% gtr %b1% exit /b 1 if %a1% lss %b1% exit /b 2 if %a2% gtr %b2% exit /b 1 if %a2% lss %b2% exit /b 2 if %a3% gtr %b3% exit /b 1 if %a3% lss %b3% exit /b 2 exit /b 0 :askinstall cls echo ### ### echo # Python Not Found # echo ### ### echo. echo Python !targetpy! was not found on the system or in the PATH var. echo. set /p "menu=Would you like to install it now? [y/n]: " if /i "!menu!"=="y" ( REM We got the OK - install it goto installpy ) else if "!menu!"=="n" ( REM No OK here... set /a tried=!tried!+1 goto checkpy ) REM Incorrect answer - go back goto askinstall :installpy REM This will attempt to download and install python set /a tried=!tried!+1 cls echo ### ### echo # Downloading Python # echo ### ### echo. set "release=!user_provided!" if "!release!" == "" ( REM No explicit release set - get the latest from python.org echo Gathering latest version... powershell -command "[Net.ServicePointManager]::SecurityProtocol=[Net.SecurityProtocolType]::Tls12;(new-object System.Net.WebClient).DownloadFile('https://www.python.org/downloads/windows/','%TEMP%\pyurl.txt')" REM Extract it if it's gzip compressed powershell -command "$infile='%TEMP%\pyurl.txt';$outfile='%TEMP%\pyurl.temp';try{$input=New-Object System.IO.FileStream $infile,([IO.FileMode]::Open),([IO.FileAccess]::Read),([IO.FileShare]::Read);$output=New-Object System.IO.FileStream $outfile,([IO.FileMode]::Create),([IO.FileAccess]::Write),([IO.FileShare]::None);$gzipStream=New-Object System.IO.Compression.GzipStream $input,([IO.Compression.CompressionMode]::Decompress);$buffer=New-Object byte[](1024);while($true){$read=$gzipstream.Read($buffer,0,1024);if($read -le 0){break};$output.Write($buffer,0,$read)};$gzipStream.Close();$output.Close();$input.Close();Move-Item -Path $outfile -Destination $infile -Force}catch{}" if not exist "%TEMP%\pyurl.txt" ( if /i "!just_installing!" == "TRUE" ( echo - Failed to get info exit /b 1 ) else ( goto checkpy ) ) pushd "%TEMP%" :: Version detection code slimmed by LussacZheng (https://github.com/corpnewt/gibMacOS/issues/20) for /f "tokens=9 delims=< " %%x in ('findstr /i /c:"Latest Python !targetpy! Release" pyurl.txt') do ( set "release=%%x" ) popd REM Let's delete our txt file now - we no longer need it del "%TEMP%\pyurl.txt" if "!release!" == "" ( if /i "!just_installing!" == "TRUE" ( echo - Failed to get python version exit /b 1 ) else ( goto checkpy ) ) echo Located Version: !release! ) else ( echo User-Provided Version: !release! REM Update our targetpy to reflect the first number of REM our release for /f "tokens=1 delims=." %%a in ("!release!") do ( call :isnumber "%%a" if "!errorlevel!" == "0" ( set "targetpy=%%a" ) ) ) echo Building download url... REM At this point - we should have the version number. REM We can build the url like so: "https://www.python.org/ftp/python/[version]/python-[version]-amd64.exe" set "url=https://www.python.org/ftp/python/!release!/python-!release!-amd64.exe" set "pytype=exe" if "!targetpy!" == "2" ( set "url=https://www.python.org/ftp/python/!release!/python-!release!.amd64.msi" set "pytype=msi" ) echo - !url! echo Downloading... REM Now we download it with our slick powershell command powershell -command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; (new-object System.Net.WebClient).DownloadFile('!url!','%TEMP%\pyinstall.!pytype!')" REM If it doesn't exist - we bail if not exist "%TEMP%\pyinstall.!pytype!" ( if /i "!just_installing!" == "TRUE" ( echo - Failed to download python installer exit /b 1 ) else ( goto checkpy ) ) REM It should exist at this point - let's run it to install silently echo Running python !pytype! installer... pushd "%TEMP%" if /i "!pytype!" == "exe" ( echo - pyinstall.exe /quiet PrependPath=1 Include_test=0 Shortcuts=0 Include_launcher=0 pyinstall.exe /quiet PrependPath=1 Include_test=0 Shortcuts=0 Include_launcher=0 ) else ( set "foldername=!release:.=!" echo - msiexec /i pyinstall.msi /qb ADDLOCAL=ALL TARGETDIR="%LocalAppData%\Programs\Python\Python!foldername:~0,2!" msiexec /i pyinstall.msi /qb ADDLOCAL=ALL TARGETDIR="%LocalAppData%\Programs\Python\Python!foldername:~0,2!" ) popd set "py_error=!errorlevel!" echo Installer finished with status: !py_error! echo Cleaning up... REM Now we should be able to delete the installer and check for py again del "%TEMP%\pyinstall.!pytype!" REM If it worked, then we should have python in our PATH REM this does not get updated right away though - let's try REM manually updating the local PATH var call :updatepath if /i "!just_installing!" == "TRUE" ( echo. echo Done. ) else ( goto checkpy ) exit /b :runscript REM Python found cls REM Checks the args gathered at the beginning of the script. REM Make sure we're not just forwarding empty quotes. set "arg_test=!args:"=!" if "!arg_test!"=="" ( "!pypath!" "!thisDir!!script_name!" ) else ( "!pypath!" "!thisDir!!script_name!" !args! ) if /i "!pause_on_error!" == "yes" ( if not "%ERRORLEVEL%" == "0" ( echo. echo Script exited with error code: %ERRORLEVEL% echo. echo Press [enter] to exit... pause > nul ) ) goto :EOF :undouble REM Helper function to strip doubles of a single character out of a string recursively set "string_value=%~2" :undouble_continue set "check=!string_value:%~3%~3=%~3!" if not "!check!" == "!string_value!" ( set "string_value=!check!" goto :undouble_continue ) set "%~1=!check!" goto :EOF :updatepath set "spath=" set "upath=" for /f "USEBACKQ tokens=2* delims= " %%i in (`!syspath!reg.exe query "HKCU\Environment" /v "Path" 2^> nul`) do ( if not "%%j" == "" set "upath=%%j" ) for /f "USEBACKQ tokens=2* delims= " %%i in (`!syspath!reg.exe query "HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment" /v "Path" 2^> nul`) do ( if not "%%j" == "" set "spath=%%j" ) if not "%spath%" == "" ( REM We got something in the system path set "PATH=%spath%" if not "%upath%" == "" ( REM We also have something in the user path set "PATH=%PATH%;%upath%" ) ) else if not "%upath%" == "" ( set "PATH=%upath%" ) REM Remove double semicolons from the adjusted PATH call :undouble "PATH" "%PATH%" ";" goto :EOF :getsyspath REM Helper method to return a valid path to cmd.exe, reg.exe, and where.exe by REM walking the ComSpec var - will also repair it in memory if need be REM Strip double semi-colons call :undouble "temppath" "%ComSpec%" ";" REM Dirty hack to leverage the "line feed" approach - there are some odd side REM effects with this. Do not use this variable name in comments near this REM line - as it seems to behave erradically. (set LF=^ %=this line is empty=% ) REM Replace instances of semi-colons with a line feed and wrap REM in parenthesis to work around some strange batch behavior set "testpath=%temppath:;=!LF!%" REM Let's walk each path and test if cmd.exe, reg.exe, and where.exe exist there set /a found=0 for /f "tokens=* delims=" %%i in ("!testpath!") do ( REM Only continue if we haven't found it yet if not "%%i" == "" ( if !found! lss 1 ( set "checkpath=%%i" REM Remove "cmd.exe" from the end if it exists if /i "!checkpath:~-7!" == "cmd.exe" ( set "checkpath=!checkpath:~0,-7!" ) REM Pad the end with a backslash if needed if not "!checkpath:~-1!" == "\" ( set "checkpath=!checkpath!\" ) REM Let's see if cmd, reg, and where exist there - and set it if so if EXIST "!checkpath!cmd.exe" ( if EXIST "!checkpath!reg.exe" ( if EXIST "!checkpath!where.exe" ( set /a found=1 set "ComSpec=!checkpath!cmd.exe" set "%~1=!checkpath!" ) ) ) ) ) ) goto :EOF ================================================ FILE: gibMacOS.command ================================================ #!/usr/bin/env bash # Get the curent directory, the script name # and the script name with "py" substituted for the extension. args=( "$@" ) dir="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" script="${0##*/}" target="${script%.*}.py" # use_py3: # TRUE = Use if found, use py2 otherwise # FALSE = Use py2 # FORCE = Use py3 use_py3="TRUE" # We'll parse if the first argument passed is # --install-python and if so, we'll just install # Can optionally take a version number as the # second arg - i.e. --install-python 3.13.1 just_installing="FALSE" tempdir="" compare_to_version () { # Compares our OS version to the passed OS version, and # return a 1 if we match the passed compare type, or a 0 if we don't. # $1 = 0 (equal), 1 (greater), 2 (less), 3 (gequal), 4 (lequal) # $2 = OS version to compare ours to if [ -z "$1" ] || [ -z "$2" ]; then # Missing info - bail. return fi local current_os= comp= current_os="$(sw_vers -productVersion 2>/dev/null)" comp="$(vercomp "$current_os" "$2")" # Check gequal and lequal first if [[ "$1" == "3" && ("$comp" == "1" || "$comp" == "0") ]] || [[ "$1" == "4" && ("$comp" == "2" || "$comp" == "0") ]] || [[ "$comp" == "$1" ]]; then # Matched echo "1" else # No match echo "0" fi } set_use_py3_if () { # Auto sets the "use_py3" variable based on # conditions passed # $1 = 0 (equal), 1 (greater), 2 (less), 3 (gequal), 4 (lequal) # $2 = OS version to compare # $3 = TRUE/FALSE/FORCE in case of match if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then # Missing vars - bail with no changes. return fi if [ "$(compare_to_version "$1" "$2")" == "1" ]; then use_py3="$3" fi } get_remote_py_version () { local pyurl= py_html= py_vers= py_num="3" pyurl="https://www.python.org/downloads/macos/" py_html="$(curl -L $pyurl --compressed 2>&1)" if [ -z "$use_py3" ]; then use_py3="TRUE" fi if [ "$use_py3" == "FALSE" ]; then py_num="2" fi py_vers="$(echo "$py_html" | grep -i "Latest Python $py_num Release" | awk '{print $8}' | cut -d'<' -f1)" echo "$py_vers" } download_py () { local vers="$1" url= clear echo " ### ###" echo " # Downloading Python #" echo "### ###" echo if [ -z "$vers" ]; then echo "Gathering latest version..." vers="$(get_remote_py_version)" if [ -z "$vers" ]; then if [ "$just_installing" == "TRUE" ]; then echo " - Failed to get info!" exit 1 else # Didn't get it still - bail print_error fi fi echo "Located Version: $vers" else # Got a version passed echo "User-Provided Version: $vers" fi echo "Building download url..." url="$(\ curl -L https://www.python.org/downloads/release/python-${vers//./}/ --compressed 2>&1 | \ grep -iE "python-$vers-macos.*.pkg\"" | \ grep -iE "a href=" | \ awk -F'"' '{ print $2 }' | \ head -n 1\ )" if [ -z "$url" ]; then if [ "$just_installing" == "TRUE" ]; then echo " - Failed to build download url!" exit 1 else # Couldn't get the URL - bail print_error fi fi echo " - $url" echo "Downloading..." # Create a temp dir and download to it tempdir="$(mktemp -d 2>/dev/null || mktemp -d -t 'tempdir')" curl "$url" -o "$tempdir/python.pkg" if [ "$?" != "0" ]; then echo " - Failed to download python installer!" exit $? fi echo echo "Running python install package..." echo sudo installer -pkg "$tempdir/python.pkg" -target / if [ "$?" != "0" ]; then echo " - Failed to install python!" exit $? fi echo # Now we expand the package and look for a shell update script pkgutil --expand "$tempdir/python.pkg" "$tempdir/python" if [ -e "$tempdir/python/Python_Shell_Profile_Updater.pkg/Scripts/postinstall" ]; then # Run the script echo "Updating PATH..." echo "$tempdir/python/Python_Shell_Profile_Updater.pkg/Scripts/postinstall" echo fi vers_folder="Python $(echo "$vers" | cut -d'.' -f1 -f2)" if [ -f "/Applications/$vers_folder/Install Certificates.command" ]; then # Certs script exists - let's execute that to make sure our certificates are updated echo "Updating Certificates..." echo "/Applications/$vers_folder/Install Certificates.command" echo fi echo "Cleaning up..." cleanup if [ "$just_installing" == "TRUE" ]; then echo echo "Done." else # Now we check for py again downloaded="TRUE" clear main fi } cleanup () { if [ -d "$tempdir" ]; then rm -Rf "$tempdir" fi } print_error() { clear cleanup echo " ### ###" echo " # Python Not Found #" echo "### ###" echo echo "Python is not installed or not found in your PATH var." echo if [ "$kernel" == "Darwin" ]; then echo "Please go to https://www.python.org/downloads/macos/ to" echo "download and install the latest version, then try again." else echo "Please install python through your package manager and" echo "try again." fi echo exit 1 } print_target_missing() { clear cleanup echo " ### ###" echo " # Target Not Found #" echo "### ###" echo echo "Could not locate $target!" echo exit 1 } format_version () { local vers="$1" echo "$(echo "$1" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }')" } vercomp () { # Modified from: https://apple.stackexchange.com/a/123408/11374 local ver1="$(format_version "$1")" ver2="$(format_version "$2")" if [ $ver1 -gt $ver2 ]; then echo "1" elif [ $ver1 -lt $ver2 ]; then echo "2" else echo "0" fi } get_local_python_version() { # $1 = Python bin name (defaults to python3) # Echoes the path to the highest version of the passed python bin if any local py_name="$1" max_version= python= python_version= python_path= if [ -z "$py_name" ]; then py_name="python3" fi py_list="$(which -a "$py_name" 2>/dev/null)" # Walk that newline separated list while read python; do if [ -z "$python" ]; then # Got a blank line - skip continue fi if [ "$check_py3_stub" == "1" ] && [ "$python" == "/usr/bin/python3" ]; then # See if we have a valid developer path xcode-select -p > /dev/null 2>&1 if [ "$?" != "0" ]; then # /usr/bin/python3 path - but no valid developer dir continue fi fi python_version="$(get_python_version $python)" if [ -z "$python_version" ]; then # Didn't find a py version - skip continue fi # Got the py version - compare to our max if [ -z "$max_version" ] || [ "$(vercomp "$python_version" "$max_version")" == "1" ]; then # Max not set, or less than the current - update it max_version="$python_version" python_path="$python" fi done <<< "$py_list" echo "$python_path" } get_python_version() { local py_path="$1" py_version= # Get the python version by piping stderr into stdout (for py2), then grepping the output for # the word "python", getting the second element, and grepping for an alphanumeric version number py_version="$($py_path -V 2>&1 | grep -i python | cut -d' ' -f2 | grep -E "[A-Za-z\d\.]+")" if [ ! -z "$py_version" ]; then echo "$py_version" fi } prompt_and_download() { if [ "$downloaded" != "FALSE" ] || [ "$kernel" != "Darwin" ]; then # We already tried to download, or we're not on macOS - just bail print_error fi clear echo " ### ###" echo " # Python Not Found #" echo "### ###" echo target_py="Python 3" printed_py="Python 2 or 3" if [ "$use_py3" == "FORCE" ]; then printed_py="Python 3" elif [ "$use_py3" == "FALSE" ]; then target_py="Python 2" printed_py="Python 2" fi echo "Could not locate $printed_py!" echo echo "This script requires $printed_py to run." echo while true; do read -p "Would you like to install the latest $target_py now? (y/n): " yn case $yn in [Yy]* ) download_py;break;; [Nn]* ) print_error;; esac done } main() { local python= version= # Verify our target exists if [ ! -f "$dir/$target" ]; then # Doesn't exist print_target_missing fi if [ -z "$use_py3" ]; then use_py3="TRUE" fi if [ "$use_py3" != "FALSE" ]; then # Check for py3 first python="$(get_local_python_version python3)" fi if [ "$use_py3" != "FORCE" ] && [ -z "$python" ]; then # We aren't using py3 explicitly, and we don't already have a path python="$(get_local_python_version python2)" if [ -z "$python" ]; then # Try just looking for "python" python="$(get_local_python_version python)" fi fi if [ -z "$python" ]; then # Didn't ever find it - prompt prompt_and_download return 1 fi # Found it - start our script and pass all args "$python" "$dir/$target" "${args[@]}" } # Keep track of whether or not we're on macOS to determine if # we can download and install python for the user as needed. kernel="$(uname -s)" # Check to see if we need to force based on # macOS version. 10.15 has a dummy python3 version # that can trip up some py3 detection in other scripts. # set_use_py3_if "3" "10.15" "FORCE" downloaded="FALSE" # Check for the aforementioned /usr/bin/python3 stub if # our OS version is 10.15 or greater. check_py3_stub="$(compare_to_version "3" "10.15")" trap cleanup EXIT if [ "$1" == "--install-python" ] && [ "$kernel" == "Darwin" ]; then just_installing="TRUE" download_py "$2" else main fi ================================================ FILE: gibMacOS.py ================================================ #!/usr/bin/env python3 from Scripts import downloader,utils,run,plist import os, shutil, time, sys, argparse, re, json, subprocess class ProgramError(Exception): def __init__(self, message, title = "Error"): super(Exception, self).__init__(message) self.title = title class gibMacOS: def __init__(self, interactive = True, download_dir = None): self.interactive = interactive self.download_dir = download_dir self.d = downloader.Downloader() self.u = utils.Utils("gibMacOS", interactive=interactive) self.r = run.Run() self.min_w = 80 self.min_h = 24 if os.name == "nt": self.min_w = 120 self.min_h = 30 self.resize() self.catalog_suffix = { "public" : "beta", "publicrelease" : "", "customer" : "customerseed", "developer" : "seed" } # Load settings.json if it exists in the Scripts folder self.settings_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"Scripts","settings.json") self.settings = {} if os.path.exists(self.settings_path): try: self.settings = json.load(open(self.settings_path)) except: pass # Load prod_cache.json if it exists in the Scripts folder self.prod_cache_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"Scripts","prod_cache.plist") self.prod_cache = {} if os.path.exists(self.prod_cache_path): try: with open(self.prod_cache_path,"rb") as f: self.prod_cache = plist.load(f) assert isinstance(self.prod_cache,dict) except: self.prod_cache = {} # If > 16, assume X-5, else 10.X # e.g. 17 = Monterey, 18 = Ventura, 19 = Sonoma, 20 = Sequoia self.current_macos = self.settings.get("current_macos",20) self.min_macos = 5 self.print_urls = self.settings.get("print_urls",False) self.print_json = False self.hide_pid = self.settings.get("hide_pid",False) self.mac_os_names_url = { "8" : "mountainlion", "7" : "lion", "6" : "snowleopard", "5" : "leopard" } self.version_names = { "tiger" : "10.4", "leopard" : "10.5", "snow leopard" : "10.6", "lion" : "10.7", "mountain lion" : "10.8", "mavericks" : "10.9", "yosemite" : "10.10", "el capitan" : "10.11", "sierra" : "10.12", "high sierra" : "10.13", "mojave" : "10.14", "catalina" : "10.15", "big sur" : "11", "monterey" : "12", "ventura" : "13", "sonoma" : "14", "sequoia" : "15", "tahoe" : "26" } self.current_catalog = self.settings.get("current_catalog","publicrelease") self.catalog_data = None self.scripts = "Scripts" self.local_catalog = os.path.join(os.path.dirname(os.path.realpath(__file__)),self.scripts,"sucatalog.plist") self.caffeinate_downloads = self.settings.get("caffeinate_downloads",True) self.caffeinate_process = None self.save_local = False self.force_local = False self.find_recovery = self.settings.get("find_recovery",False) self.recovery_suffixes = ( "RecoveryHDUpdate.pkg", "RecoveryHDMetaDmg.pkg" ) self.settings_to_save = ( "current_macos", "current_catalog", "print_urls", "find_recovery", "hide_pid", "caffeinate_downloads" ) self.mac_prods = [] def resize(self, width=0, height=0): if not self.interactive: return width = width if width > self.min_w else self.min_w height = height if height > self.min_h else self.min_h self.u.resize(width, height) def save_settings(self): # Ensure we're using the latest values for setting in self.settings_to_save: self.settings[setting] = getattr(self,setting,None) try: json.dump(self.settings,open(self.settings_path,"w"),indent=2) except Exception as e: raise ProgramError( "Failed to save settings to:\n\n{}\n\nWith error:\n\n - {}\n".format(self.settings_path,repr(e)), title="Error Saving Settings") def save_prod_cache(self): try: with open(self.prod_cache_path,"wb") as f: plist.dump(self.prod_cache,f) except Exception as e: raise ProgramError( "Failed to save product cache to:\n\n{}\n\nWith error:\n\n - {}\n".format(self.prod_cache_path,repr(e)), title="Error Saving Product Cache") def set_prods(self): self.resize() if not self.get_catalog_data(self.save_local): message = "The currently selected catalog ({}) was not reachable\n".format(self.current_catalog) if self.save_local: message += "and I was unable to locate a valid catalog file at:\n - {}\n".format( self.local_catalog ) message += "Please ensure you have a working internet connection." if self.interactive: print(message) self.u.grab("\nPress [enter] to continue...") return raise ProgramError(message, title="Catalog Data Error") self.u.head("Parsing Data") self.u.info("Scanning products after catalog download...\n") self.mac_prods = self.get_dict_for_prods(self.get_installers()) def set_catalog(self, catalog): self.current_catalog = catalog.lower() if catalog.lower() in self.catalog_suffix else "publicrelease" def num_to_macos(self,macos_num,for_url=True): if for_url: # Resolve 8-5 to their names and show Big Sur as 10.16 return self.mac_os_names_url.get(str(macos_num),"10.{}".format(macos_num)) if macos_num <= 16 else str(macos_num-5) # Return 10.xx for anything Catalina and lower, otherwise 11+ return "10.{}".format(macos_num) if macos_num <= 15 else str(macos_num-5) def macos_to_num(self,macos): try: macos_parts = [int(x) for x in macos.split(".")][:2 if macos.startswith("10.") else 1] if macos_parts[0] == 11: macos_parts = [10,16] # Big sur except: return None if len(macos_parts) > 1: return macos_parts[1] return 5+macos_parts[0] def get_macos_versions(self,minos=None,maxos=None,catalog=""): if minos is None: minos = self.min_macos if maxos is None: maxos = self.current_macos if minos > maxos: minos,maxos = maxos,minos # Ensure min is less than or equal os_versions = [self.num_to_macos(x,for_url=True) for x in range(minos,min(maxos+1,21))] # until sequoia if maxos > 30: # since tahoe os_versions.extend([self.num_to_macos(x,for_url=True) for x in range(31,maxos+1)]) if catalog: # We have a custom catalog - prepend the first entry + catalog to the list custom_cat_entry = os_versions[-1]+catalog os_versions.append(custom_cat_entry) return os_versions def build_url(self, **kwargs): catalog = kwargs.get("catalog", self.current_catalog).lower() catalog = catalog if catalog.lower() in self.catalog_suffix else "publicrelease" version = int(kwargs.get("version", self.current_macos)) return "https://swscan.apple.com/content/catalogs/others/index-{}.merged-1.sucatalog".format( "-".join(reversed(self.get_macos_versions(self.min_macos,version,catalog=self.catalog_suffix.get(catalog,"")))) ) def get_catalog_data(self, local = False): # Gets the data based on our current_catalog url = self.build_url(catalog=self.current_catalog, version=self.current_macos) self.u.head("Downloading Catalog") if local: self.u.info("Checking for:\n - {}".format( self.local_catalog )) if os.path.exists(self.local_catalog): self.u.info(" - Found - loading...") try: with open(self.local_catalog, "rb") as f: self.catalog_data = plist.load(f) assert isinstance(self.catalog_data,dict) return True except Exception as e: self.u.info(" - Error loading: {}".format(e)) self.u.info(" - Downloading instead...\n") else: self.u.info(" - Not found - downloading instead...\n") self.u.info("Currently downloading {} catalog from:\n\n{}\n".format(self.current_catalog, url)) try: b = self.d.get_bytes(url, self.interactive) self.u.info("") self.catalog_data = plist.loads(b) except: self.u.info("Error downloading!") return False try: # Assume it's valid data - dump it to a local file if local or self.force_local: self.u.info(" - Saving to:\n - {}".format( self.local_catalog )) with open(self.local_catalog, "wb") as f: plist.dump(self.catalog_data, f) except Exception as e: self.u.info(" - Error saving: {}".format(e)) return False return True def get_installers(self, plist_dict = None): if not plist_dict: plist_dict = self.catalog_data if not plist_dict: return [] mac_prods = [] for p in plist_dict.get("Products", {}): if not self.find_recovery: val = plist_dict.get("Products",{}).get(p,{}).get("ExtendedMetaInfo",{}).get("InstallAssistantPackageIdentifiers",{}) if val.get("OSInstall",{}) == "com.apple.mpkg.OSInstall" or val.get("SharedSupport","").startswith("com.apple.pkg.InstallAssistant"): mac_prods.append(p) else: # Find out if we have any of the recovery_suffixes if any(x for x in plist_dict.get("Products",{}).get(p,{}).get("Packages",[]) if x["URL"].endswith(self.recovery_suffixes)): mac_prods.append(p) return mac_prods def get_build_version(self, dist_dict): build = version = name = "Unknown" try: dist_url = dist_dict.get("English",dist_dict.get("en","")) dist_file = self.d.get_string(dist_url,False) assert isinstance(dist_file,str) except: dist_file = "" build_search = "macOSProductBuildVersion" if "macOSProductBuildVersion" in dist_file else "BUILD" vers_search = "macOSProductVersion" if "macOSProductVersion" in dist_file else "VERSION" try: build = dist_file.split("{}".format(build_search))[1].split("")[1].split("")[0] except: pass try: version = dist_file.split("{}".format(vers_search))[1].split("")[1].split("")[0] except: pass try: name = re.search(r"(.+?)",dist_file).group(1) except: pass try: # XXX: This is parsing a JavaScript array from the script part of the dist file. device_ids = re.search(r"var supportedDeviceIDs\s*=\s*\[([^]]+)\];", dist_file)[1] device_ids = list(set(i.lower() for i in re.findall(r"'([^',]+)'", device_ids))) except: device_ids = [] return (build,version,name,device_ids) def get_dict_for_prods(self, prods, plist_dict = None): plist_dict = plist_dict or self.catalog_data or {} prod_list = [] # Keys required to consider a cached element valid prod_keys = ( "build", "date", "description", "device_ids", "installer", "product", "time", "title", "version", ) def get_packages_and_size(plist_dict,prod,recovery): # Iterate the available packages and save their urls and sizes packages = [] size = -1 if recovery: # Only get the recovery packages packages = [x for x in plist_dict.get("Products",{}).get(prod,{}).get("Packages",[]) if x["URL"].endswith(self.recovery_suffixes)] else: # Add them all! packages = plist_dict.get("Products",{}).get(prod,{}).get("Packages",[]) # Get size size = self.d.get_size(sum([i["Size"] for i in packages])) return (packages,size) def print_prod(prod,prod_list): self.u.info(" -->{}. {} ({}){}".format( str(len(prod_list)+1).rjust(3), prod["title"], prod["build"], " - FULL Install" if self.find_recovery and prod["installer"] else "" )) def prod_valid(prod,prod_list,prod_keys): # Check if the prod has all prod keys, and # none are "Unknown" if not isinstance(prod_list,dict) or not prod in prod_list or \ not all(x in prod_list[prod] for x in prod_keys): # Wrong type, missing the prod, or prod_list keys return False # Let's make sure none of the keys return Unknown if any(prod_list[prod].get(x,"Unknown")=="Unknown" for x in prod_keys): return False return True # Boolean to keep track of cache updates prod_changed = False for prod in prods: if prod_valid(prod,self.prod_cache,prod_keys): # Already have it - and it's valid. # Create a shallow copy prodd = {} for key in self.prod_cache[prod]: prodd[key] = self.prod_cache[prod][key] # Update the packages and size lists prodd["packages"],prodd["size"] = get_packages_and_size(plist_dict,prod,self.find_recovery) # Add to our list and continue on prod_list.append(prodd) # Log the product print_prod(prodd,prod_list) continue # Grab the ServerMetadataURL for the passed product key if it exists prodd = {"product":prod} try: url = plist_dict.get("Products",{}).get(prod,{}).get("ServerMetadataURL","") assert url b = self.d.get_bytes(url,False) smd = plist.loads(b) except: smd = {} # Populate some info! prodd["date"] = plist_dict.get("Products",{}).get(prod,{}).get("PostDate","") prodd["installer"] = plist_dict.get("Products",{}).get(prod,{}).get("ExtendedMetaInfo",{}).get("InstallAssistantPackageIdentifiers",{}).get("OSInstall",{}) == "com.apple.mpkg.OSInstall" prodd["time"] = time.mktime(prodd["date"].timetuple()) + prodd["date"].microsecond / 1E6 prodd["version"] = smd.get("CFBundleShortVersionString","Unknown").strip() # Try to get the description too try: desc = smd.get("localization",{}).get("English",{}).get("description","").decode("utf-8") desctext = desc.split('"p1">')[1].split("")[0] except: desctext = "" prodd["description"] = desctext prodd["packages"],prodd["size"] = get_packages_and_size(plist_dict,prod,self.find_recovery) # Get size prodd["size"] = self.d.get_size(sum([i["Size"] for i in prodd["packages"]])) # Attempt to get the build/version/name/device-ids info from the dist prodd["build"],v,n,prodd["device_ids"] = self.get_build_version(plist_dict.get("Products",{}).get(prod,{}).get("Distributions",{})) prodd["title"] = smd.get("localization",{}).get("English",{}).get("title",n) if v.lower() != "unknown": prodd["version"] = v prod_list.append(prodd) # If we were able to resolve the SMD URL - or it didn't exist, save it to the cache if smd or not plist_dict.get("Products",{}).get(prod,{}).get("ServerMetadataURL",""): prod_changed = True # Create a temp prod dict so we can save all but the packages and # size keys - as those are determined based on self.find_recovery temp_prod = {} for key in prodd: if key in ("packages","size"): continue if prodd[key] == "Unknown": # Don't cache Unknown values temp_prod = None break temp_prod[key] = prodd[key] if temp_prod: # Only update the cache if it changed self.prod_cache[prod] = temp_prod # Log the product print_prod(prodd,prod_list) # Try saving the cache for later if prod_changed and self.prod_cache: try: self.save_prod_cache() except: pass # Sort by newest prod_list = sorted(prod_list, key=lambda x:x["time"], reverse=True) return prod_list def start_caffeinate(self): # Check if we need to caffeinate if sys.platform.lower() == "darwin" \ and self.caffeinate_downloads \ and os.path.isfile("/usr/bin/caffeinate"): # Terminate any existing caffeinate process self.term_caffeinate_proc() # Create a new caffeinate process self.caffeinate_process = subprocess.Popen( ["/usr/bin/caffeinate"], stderr=getattr(subprocess,"DEVNULL",open(os.devnull,"w")), stdout=getattr(subprocess,"DEVNULL",open(os.devnull,"w")), stdin=getattr(subprocess,"DEVNULL",open(os.devnull,"w")) ) return self.caffeinate_process def term_caffeinate_proc(self): if self.caffeinate_process is None: return True try: if self.caffeinate_process.poll() is None: # Save the time we started waiting start = time.time() while self.caffeinate_process.poll() is None: # Make sure we haven't waited too long if time.time() - start > 10: print(" - Timed out trying to terminate caffeinate process with PID {}!".format( self.caffeinate_process.pid )) return False # It's alive - terminate it self.caffeinate_process.terminate() # Sleep to let things settle time.sleep(0.02) except: pass return True # Couldn't poll - or we termed it def download_prod(self, prod, dmg = False): # Takes a dictonary of details and downloads it self.resize() name = "{} - {} {} ({})".format(prod["product"], prod["version"], prod["title"], prod["build"]).replace(":","").strip() download_dir = self.download_dir or os.path.join(os.path.dirname(os.path.realpath(__file__)), "macOS Downloads", self.current_catalog, name) dl_list = [] for x in prod["packages"]: if not x.get("URL",None): continue if dmg and not x.get("URL","").lower().endswith(".dmg"): continue # add it to the list dl_list.append(x) if not len(dl_list): raise ProgramError("There were no files to download") done = [] if self.print_json: print(self.product_to_json(prod)) if self.interactive: print("") self.u.grab("Press [enter] to return...") return elif self.print_urls: self.u.head("Download Links") print("{}:\n".format(name)) print("\n".join([" - {} ({}) \n --> {}".format( os.path.basename(x["URL"]), self.d.get_size(x["Size"],strip_zeroes=True) if x.get("Size") is not None else "?? MB", x["URL"] ) for x in dl_list])) if self.interactive: print("") self.u.grab("Press [enter] to return...") return # Only check the dirs if we need to if self.download_dir is None and os.path.exists(download_dir): while True: self.u.head("Already Exists") self.u.info("It looks like you've already downloaded the following package:\n{}\n".format(name)) if not self.interactive: menu = "r" else: print("R. Resume Incomplete Files") print("D. Redownload All Files") print("") print("M. Return") print("Q. Quit") print("") menu = self.u.grab("Please select an option: ") if not len(menu): continue elif menu.lower() == "q": self.u.custom_quit() elif menu.lower() == "m": return elif menu.lower() == "r": break elif menu.lower() == "d": # Remove the old copy, then re-download shutil.rmtree(download_dir) break # Make it anew as needed if not os.path.isdir(download_dir): os.makedirs(download_dir) # Clean up any leftover or missed caffeinate # procs self.term_caffeinate_proc() for c,x in enumerate(dl_list,start=1): url = x["URL"] self.u.head("Downloading File {} of {}".format(c, len(dl_list))) self.u.info("- {} -\n".format(name)) if len(done): self.u.info("\n".join(["{} --> {}".format(y["name"], "Succeeded" if y["status"] else "Failed") for y in done])) self.u.info("") if dmg: self.u.info("NOTE: Only Downloading DMG Files\n") self.u.info("Downloading {}...\n".format(os.path.basename(url))) try: # Caffeinate as needed self.start_caffeinate() result = self.d.stream_to_file(url, os.path.join(download_dir, os.path.basename(url)), allow_resume=True) assert result is not None done.append({"name":os.path.basename(url), "status":True}) except: done.append({"name":os.path.basename(url), "status":False}) # Kill caffeinate if we need to self.term_caffeinate_proc() succeeded = [x for x in done if x["status"]] failed = [x for x in done if not x["status"]] self.u.head("Downloaded {} of {}".format(len(succeeded), len(dl_list))) self.u.info("- {} -\n".format(name)) self.u.info("Succeeded:") if len(succeeded): for x in succeeded: self.u.info(" {}".format(x["name"])) else: self.u.info(" None") self.u.info("\nFailed:") if len(failed): for x in failed: self.u.info(" {}".format(x["name"])) else: self.u.info(" None") self.u.info("\nFiles saved to:\n {}\n".format(download_dir)) if self.interactive: self.u.grab("Press [enter] to return...") elif len(failed): raise ProgramError("{} files failed to download".format(len(failed))) def product_to_json(self, prod): prod_dict = {} for key in ["product", "version", "build", "title", "size", "packages"]: if key in prod: prod_dict[key] = prod[key] prod_dict["date"] = prod["date"].isoformat() prod_dict["deviceIds"] = list(prod["device_ids"]) return json.dumps(prod_dict,indent=2) def show_catalog_url(self): self.resize() self.u.head() print("Current Catalog: {}".format(self.current_catalog)) print("Max macOS Version: {}".format(self.num_to_macos(self.current_macos,for_url=False))) print("") print("{}".format(self.build_url())) if self.interactive: print("") self.u.grab("Press [enter] to return...") def pick_catalog(self): self.resize() self.u.head("Select SU Catalog") count = 0 for x in self.catalog_suffix: count += 1 print("{}. {}".format(count, x)) print("") print("M. Main Menu") print("Q. Quit") print("") menu = self.u.grab("Please select an option: ") if not len(menu): self.pick_catalog() return if menu[0].lower() == "m": return elif menu[0].lower() == "q": self.u.custom_quit() # Should have something to test here try: i = int(menu) self.current_catalog = list(self.catalog_suffix)[i-1] self.save_settings() except: # Incorrect - try again self.pick_catalog() return # If we made it here - then we got something # Reload with the proper catalog self.get_catalog_data() def pick_macos(self): self.resize() self.u.head("Select Max macOS Version") print("Currently set to {}".format(self.num_to_macos(self.current_macos,for_url=False))) print("") print("M. Main Menu") print("Q. Quit") print("") print("Please type the max macOS version for the catalog url") menu = self.u.grab("eg. 10.15 for Catalina, 11 for Big Sur, 12 for Monterey: ") if not len(menu): self.pick_macos() return if menu[0].lower() == "m": return elif menu[0].lower() == "q": self.u.custom_quit() # At this point - we should have something in the proper format version = self.macos_to_num(menu) if not version: return self.current_macos = version self.save_settings() # At this point, we should be good - set teh catalog # data - but if it fails, remove the listed prods if not self.get_catalog_data(): self.catalog_data = None self.u.grab("\nPress [enter] to return...") self.pick_macos() return def main(self, dmg = False): lines = [] lines.append("Available Products:") lines.append(" ") if not len(self.mac_prods): lines.append("No installers in catalog!") lines.append(" ") for num,p in enumerate(self.mac_prods,start=1): var1 = "{}. {} {}".format(str(num).rjust(2), p["title"], p["version"]) var2 = "" if p["build"].lower() != "unknown": var1 += " ({})".format(p["build"]) if not self.hide_pid: var2 = " - {} - Added {} - {}".format(p["product"], p["date"], p["size"]) if self.find_recovery and p["installer"]: # Show that it's a full installer if self.hide_pid: var1 += " - FULL Install" else: var2 += " - FULL Install" lines.append(var1) if not self.hide_pid: lines.append(var2) lines.append(" ") lines.append("M. Change Max-OS Version (Currently {})".format(self.num_to_macos(self.current_macos,for_url=False))) lines.append("C. Change Catalog (Currently {})".format(self.current_catalog)) lines.append("I. Only Print URLs (Currently {})".format("On" if self.print_urls else "Off")) lines.append("H. {} Package IDs and Upload Dates".format("Show" if self.hide_pid else "Hide")) if sys.platform.lower() == "darwin": lines.append("S. Set Current Catalog to SoftwareUpdate Catalog") lines.append("L. Clear SoftwareUpdate Catalog") lines.append("F. Caffeinate Downloads to Prevent Sleep (Currently {})".format("On" if self.caffeinate_downloads else "Off")) lines.append("R. Toggle Recovery-Only (Currently {})".format("On" if self.find_recovery else "Off")) lines.append("U. Show Catalog URL") lines.append("Q. Quit") lines.append(" ") self.resize(len(max(lines)), len(lines)+5) self.u.head() print("\n".join(lines)) menu = self.u.grab("Please select an option: ") if not len(menu): return if menu[0].lower() == "q": self.resize() self.u.custom_quit() elif menu[0].lower() == "u": self.show_catalog_url() elif menu[0].lower() == "m": self.pick_macos() elif menu[0].lower() == "c": self.pick_catalog() elif menu[0].lower() == "i": self.print_urls ^= True self.save_settings() elif menu[0].lower() == "h": self.hide_pid ^= True self.save_settings() elif menu[0].lower() == "s" and sys.platform.lower() == "darwin": # Set the software update catalog to our current catalog url self.u.head("Setting SU CatalogURL") url = self.build_url(catalog=self.current_catalog, version=self.current_macos) print("Setting catalog URL to:\n{}".format(url)) print("") print("sudo softwareupdate --set-catalog {}".format(url)) self.r.run({"args":["softwareupdate","--set-catalog",url],"sudo":True}) print("") self.u.grab("Done",timeout=5) elif menu[0].lower() == "l" and sys.platform.lower() == "darwin": # Clear the software update catalog self.u.head("Clearing SU CatalogURL") print("sudo softwareupdate --clear-catalog") self.r.run({"args":["softwareupdate","--clear-catalog"],"sudo":True}) print("") self.u.grab("Done.", timeout=5) elif menu[0].lower() == "f" and sys.platform.lower() == "darwin": # Toggle our caffeinate downloads value and save settings self.caffeinate_downloads ^= True self.save_settings() elif menu[0].lower() == "r": self.find_recovery ^= True self.save_settings() if menu[0].lower() in ["m","c","r"]: self.resize() self.u.head("Parsing Data") print("Re-scanning products after url preference toggled...\n") self.mac_prods = self.get_dict_for_prods(self.get_installers()) else: # Assume we picked something try: menu = int(menu) except: return if menu < 1 or menu > len(self.mac_prods): return self.download_prod(self.mac_prods[menu-1], dmg) def get_latest(self, device_id = None, dmg = False): self.u.head("Downloading Latest") prods = sorted(self.mac_prods, key=lambda x:x['version'], reverse=True) if device_id: prod = next(p for p in prods if device_id.lower() in p["device_ids"]) if not prod: raise ProgramError("No version found for Device ID '{}'".format(device_id)) else: prod = prods[0] self.download_prod(prod, dmg) def get_for_product(self, prod, dmg = False): self.u.head("Downloading for {}".format(prod)) for p in self.mac_prods: if p["product"] == prod: self.download_prod(p, dmg) return raise ProgramError("{} not found".format(prod)) def get_for_version(self, vers, build = None, device_id = None, dmg = False): self.u.head("Downloading for {} {}".format(vers, build or "")) # Map the versions to their names v = self.version_names.get(vers.lower(),vers.lower()) v_dict = {} for n in self.version_names: v_dict[self.version_names[n]] = n n = v_dict.get(v, v) for p in sorted(self.mac_prods, key=lambda x:x['version'], reverse=True): if build and p["build"] != build: continue if device_id and device_id.lower() not in p["device_ids"]: continue pt = p["title"].lower() pv = p["version"].lower() # Need to compare verisons - n = name, v = version # p["version"] and p["title"] may contain either the version # or name - so check both # We want to make sure, if we match the name to the title, that we only match # once - so Sierra/High Sierra don't cross-match # # First check if p["version"] isn't " " or "1.0" if not pv in [" ","1.0"]: # Have a real version - match this first if pv.startswith(v): self.download_prod(p, dmg) return # Didn't match the version - or version was bad, let's check # the title # Need to make sure n is in the version name, but not equal to it, # and the version name is in p["title"] to disqualify # i.e. - "Sierra" exists in "High Sierra", but does not equal "High Sierra" # and "High Sierra" is in "macOS High Sierra 10.13.6" - This would match name_match = [x for x in self.version_names if n in x and x != n and x in pt] if (n in pt) and not len(name_match): self.download_prod(p, dmg) return raise ProgramError("'{}' '{}' not found".format(vers, build or "")) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("-l", "--latest", help="downloads the version available in the current catalog (overrides --build, --version and --product)", action="store_true") parser.add_argument("-r", "--recovery", help="looks for RecoveryHDUpdate.pkg and RecoveryHDMetaDmg.pkg in lieu of com.apple.mpkg.OSInstall (overrides --dmg)", action="store_true") parser.add_argument("-d", "--dmg", help="downloads only the .dmg files", action="store_true") parser.add_argument("-s", "--savelocal", help="uses a locally saved sucatalog.plist if exists", action="store_true") parser.add_argument("-g", "--local-catalog", help="the path to the sucatalog.plist to use (implies --savelocal)") parser.add_argument("-n", "--newlocal", help="downloads and saves locally, overwriting any prior sucatalog.plist (will use the path from --local-catalog if provided)", action="store_true") parser.add_argument("-c", "--catalog", help="sets the CATALOG to use - publicrelease, public, customer, developer") parser.add_argument("-p", "--product", help="sets the product id to search for (overrides --version)") parser.add_argument("-v", "--version", help="sets the version of macOS to target - eg '-v 10.14' or '-v Yosemite'") parser.add_argument("-b", "--build", help="sets the build of macOS to target - eg '22G120' (must be used together with --version)") parser.add_argument("-m", "--maxos", help="sets the max macOS version to consider when building the url - eg 10.14") parser.add_argument("-D", "--device-id", help="use with --version or --latest to search for versions supporting the specified Device ID - eg VMM-x86_64 for any x86_64") parser.add_argument("-i", "--print-urls", help="only prints the download URLs, does not actually download them", action="store_true") parser.add_argument("-j", "--print-json", help="only prints the product metadata in JSON, does not actually download it", action="store_true") parser.add_argument("--no-interactive", help="run in non-interactive mode (auto-enabled when using --product or --version)", action="store_true") parser.add_argument("-o", "--download-dir", help="overrides directory where the downloaded files are saved") args = parser.parse_args() if args.build and not (args.latest or args.product or args.version): print("The --build option requires a --version") exit(1) interactive = not any((args.no_interactive,args.product,args.version)) g = gibMacOS(interactive=interactive, download_dir=args.download_dir) if args.recovery: args.dmg = False g.find_recovery = args.recovery if args.savelocal: g.save_local = True if args.local_catalog: g.save_local = True g.local_catalog = args.local_catalog if args.newlocal: g.force_local = True if args.print_urls: g.print_urls = True if args.print_json: g.print_json = True if args.maxos: try: version = g.macos_to_num(args.maxos) if version: g.current_macos = version except: pass if args.catalog: # Set the catalog g.set_catalog(args.catalog) try: # Done setting up pre-requisites g.set_prods() if args.latest: g.get_latest(device_id=args.device_id, dmg=args.dmg) elif args.product != None: g.get_for_product(args.product, args.dmg) elif args.version != None: g.get_for_version(args.version, args.build, device_id=args.device_id, dmg=args.dmg) elif g.interactive: while True: try: g.main(args.dmg) except ProgramError as e: g.u.head(e.title) print(str(e)) print("") g.u.grab("Press [enter] to return...") else: raise ProgramError("No command specified") except ProgramError as e: print(str(e)) if g.interactive: print("") g.u.grab("Press [enter] to exit...") else: exit(1) exit(0)