[ { "path": ".github/workflows/config/config.json", "content": "{\n\t\"default\": true,\n\t\"MD048\": { \"style\": \"backtick\" },\n\t\"MD046\": { \"style\": \"fenced\" },\n\t\"MD029\": { \"style\": \"one\" },\n\t\"line-length\": false,\n\t\"no-hard-tabs\": false\n}\n" }, { "path": ".github/workflows/linter.yml", "content": "name: Linter\n\non: [push, pull_request]\n\njobs:\n superlinter:\n name: Super Linter\n runs-on: ubuntu-latest\n\n steps:\n - name: Checkout Code\n uses: actions/checkout@v3\n with:\n # Full git history is needed to get a proper list of changed files within `super-linter`\n fetch-depth: 0\n\n - name: Lint Code Base\n uses: github/super-linter@v4\n env:\n # Don't check already existent files\n VALIDATE_ALL_CODEBASE: false\n VALIDATE_GITHUB_ACTIONS: false\n LINTER_RULES_PATH: /.github/workflows/\n MARKDOWN_CONFIG_FILE: config/config.json\n MARKDOWN_CUSTOM_RULE_GLOBS: rules/rules.js\n DEFAULT_BRANCH: main\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".github/workflows/rules/common/inlineTokenChildren.js", "content": " class InlineTokenChildren {\n constructor(token) {\n if (token.type === \"inline\") {\n this.root = token;\n this.column = -1;\n this.lineNumber = token.map[0];\n } else {\n throw new TypeError(\"wrong argument token type\");\n }\n }\n\n *[Symbol.iterator]() {\n for (let token of this.root.children) {\n let { line, lineNumber } = token;\n if (this.lineNumber !== lineNumber) {\n this.column = -1;\n this.lineNumber = lineNumber;\n }\n this.column = line.indexOf(token.content, this.column + 1);\n yield { token, column: this.column + 1, lineNumber };\n }\n }\n}\n\nmodule.exports = { InlineTokenChildren };\n" }, { "path": ".github/workflows/rules/common/wordPattern.js", "content": "class WordPattern {\n constructor(pattern, parameters) {\n const escapedDots = pattern.replace(/\\\\?\\./g, \"\\\\.\");\n this.pattern = parameters && parameters.hasOwnProperty('noWordBoundary') ? escapedDots : \"\\\\b\" + escapedDots + \"\\\\b\";\n const modifiers = parameters && parameters.hasOwnProperty('caseSensitive') && parameters.caseSensitive ? \"\" : \"i\";\n this.regex = new RegExp(this.pattern, modifiers);\n this.suggestion = parameters && parameters.hasOwnProperty('suggestion') ? parameters.suggestion : pattern;\n this.stringRegex = new RegExp(\"^\" + escapedDots + \"$\", modifiers); // To match \"Category\" column words in changelogs, see case-sensitive.js\n this.skipForUseCases = !!(parameters && parameters.hasOwnProperty('skipForUseCases'));\n }\n\n test(line) {\n return new Match(line.match(this.regex));\n }\n}\n\nclass Match {\n constructor(match) {\n this.match = match;\n }\n\n range() {\n if (this.match) {\n let column = this.match.index + 1;\n let length = this.match[0].length;\n if (this.match[2]) {\n column += this.match[1].length;\n length -= this.match[1].length;\n }\n return [column, length];\n }\n return null;\n }\n\n toString() {\n return this.match ? this.match.toString() : \"null\";\n }\n}\n\nmodule.exports = { WordPattern };\n" }, { "path": ".github/workflows/rules/md101.js", "content": "const { InlineTokenChildren } = require(\"./common/inlineTokenChildren\");\nconst { WordPattern } = require(\"./common/wordPattern\");\n\nconst keywords = [\n new WordPattern(\"iExtractor-manager\"),\n new WordPattern(\"device-info\"),\n new WordPattern(\"device-name\"),\n new WordPattern(\"list_apps\"),\n new WordPattern(\"decrypt_kcache\"),\n new WordPattern(\"decrypt_fs\"),\n new WordPattern(\"curl\"),\n new WordPattern(\"wget\"),\n new WordPattern(\"crontab\"),\n new WordPattern(\"cron\"),\n new WordPattern(\"netcat\"),\n new WordPattern(\"ping\"),\n new WordPattern(\"traceroute\"),\n new WordPattern(\"sudo\"),\n new WordPattern(\"(? {\n var inHeading = false;\n var inLink = false;\n for (let token of params.tokens) {\n switch (token.type) {\n case \"heading_open\":\n inHeading = true; break;\n case \"heading_close\":\n inHeading = false; break;\n case \"inline\":\n let children = new InlineTokenChildren(token);\n for (let { token: child, column, lineNumber } of children) {\n let isText = child.type === \"text\";\n switch (child.type) {\n case \"link_open\":\n inLink = true; break;\n case \"link_close\":\n inLink = false; break;\n }\n for (let k of keywords) {\n let anyCaseMatch = child.content.match(k.regex);\n if (anyCaseMatch != null) {\n let match = anyCaseMatch[0];\n let correct = k.suggestion;\n if ((!inHeading && !inLink && isText) || // Bad not fenced\n (match !== correct)) { // Right fencing, wrong case\n onError({\n lineNumber,\n detail: `Expected \\`${correct}\\`. Actual ${match}.`,\n range: [column + anyCaseMatch.index, match.length]\n })\n }\n }\n }\n }\n }\n }\n }\n};\n" }, { "path": ".github/workflows/rules/md102.js", "content": "const http_keywords = [\n \"GET\",\n \"POST\",\n \"PUT\",\n \"PATCH\",\n \"DELETE\",\n \"Content-Type\",\n \"Content-Encoding\",\n \"User-Agent\",\n \"200 OK\",\n \"401 Unauthorized\",\n \"403 Forbidden\",\n \"API_DATA_READ\",\n \"API_DATA_WRITE\",\n \"API_META_READ\",\n \"API_META_WRITE\",\n \"USER\",\n \"EDITOR\",\n \"ENTITY_GROUP_ADMIN\",\n \"ADMIN\"\n];\nconst keywordsRegex = new RegExp(http_keywords.map(word => \"\\\\b\" + word + \"\\\\b\").join(\"|\"));\n\nconst { InlineTokenChildren } = require(\"./common/inlineTokenChildren\");\n\nmodule.exports = {\n names: [\"MD102\", \"backtick-http\"],\n description: \"HTTP keywords must be fenced.\",\n tags: [\"backtick\", \"HTTP\", \"HTTPS\"],\n \"function\": (params, onError) => {\n var inHeading = false;\n for (let token of params.tokens) {\n switch (token.type) {\n case \"heading_open\":\n inHeading = true; break;\n case \"heading_close\":\n inHeading = false; break;\n case \"inline\":\n if (!inHeading) {\n let children = new InlineTokenChildren(token);\n for (let { token: child, column, lineNumber } of children) {\n if (child.type === \"text\") {\n let exactCaseMatch = child.content.match(keywordsRegex);\n if (exactCaseMatch != null) {\n let match = exactCaseMatch[0];\n onError({\n lineNumber,\n detail: `Expected \\`${match}\\`. Actual ${match}.`,\n range: [column + exactCaseMatch.index, match.length]\n })\n }\n }\n }\n }\n }\n }\n }\n};\n" }, { "path": ".github/workflows/rules/md103.js", "content": "\"use strict\";\n\nmodule.exports = {\n \"names\": [ \"MD103\", \"inline triple backticks\" ],\n \"description\": \"inline triple backticks\",\n \"tags\": [ \"backticks\" ],\n \"function\": function rule(params, onError) {\n for (const inline of params.tokens.filter(function filterToken(token) {\n return token.type === \"inline\";\n })) {\n const index = inline.content.toLowerCase().indexOf(\"```\");\n if (index !== -1) {\n onError({\n \"lineNumber\": inline.lineNumber,\n \"context\": inline.content.substr(index - 1, 4),\n \"detail\": \"Expected `. Actual ```\"\n });\n }\n }\n }\n};\n" }, { "path": ".github/workflows/rules/md104.js", "content": "\"use strict\";\n\nmodule.exports = {\n names: [\"MD104\", \"one line per sentence\"],\n description: \"one line (and only one line) per sentence\",\n tags: [\"sentences\"],\n function: function rule(params, onError) {\n for (const inline of params.tokens.filter(function filterToken(token) {\n return token.type === \"inline\";\n })) {\n var actual_lines = inline.content.split(\"\\n\");\n actual_lines.forEach((line, index, arr) => {\n\t\tlet outside = true;\n\t\tlet count = 0;\n\t\tArray.from(line).forEach((char) => {\n\t\t\tif ((char == \".\" || char == \"?\" || char == \"!\" || char == \";\" || char == \":\") && outside) {\n\t\t\t\tcount++;\n\t\t\t}\n\t\t\tif (char == \"`\") outside = !outside;\n\t\t\tif (char == \"[\") outside = false;\n\t\t\tif (char == \"(\") outside = false;\n\t\t\tif (char == \"]\") outside = true;\n\t\t\tif (char == \")\") outside = true;\n\t\t});\n if (count > 1) {\n onError({\n lineNumber: inline.lineNumber + index,\n detail:\n \"Expected one sentence per line. Multiple end of sentence punctuation signs found on one line!\",\n });\n }\n });\n }\n },\n};\n" }, { "path": ".github/workflows/rules/rules.js", "content": "\"use strict\";\n\nconst rules = [\n\trequire(\"./md101.js\"),\n\trequire(\"./md102.js\"),\n\trequire(\"./md103.js\"),\n\trequire(\"./md104.js\"),\n];\nmodule.exports = rules;\n" }, { "path": ".gitignore", "content": "*~\n*.o\n*.zip\n*.rar\n*.tar\n*gz\n*bz2\n*.obj\n*.a\n*.so\n*.lib\n*.dll\n*.swp\n*.swo\ntags\nTAGS\n*.exe\n*.class\n*.jar\n*.pyc\n*.log\n*.bin\ncore\n.DS_STORE\n" }, { "path": ".gitmodules", "content": "" }, { "path": "LICENSE", "content": "BSD 3-Clause License\n\nCopyright (c) 2016, North Carolina State University and University POLITEHNICA\nof Bucharest.\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "README.md", "content": "# SandBlaster: Reversing the Apple Sandbox\n\nSandBlaster is a tool for reversing (decompiling) binary Apple sandbox profiles. Apple sandbox profiles are written in SBPL (*Sandbox Profile Language*), a Scheme-like language, and are then compiled into an undocumented binary format and shipped. Primarily used on iOS, sandbox profiles are present on macOS as well. SandBlaster is, to our knowledge, the first tool that reverses binary sandbox profiles to their original SBPL format. SandBlaster works on iOS from version 7 onwards including iOS 11.\n\nThe technical report [SandBlaster: Reversing the Apple Sandbox](https://arxiv.org/abs/1608.04303) presents extensive (though a bit outdated) information on SandBlaster internals.\n\nSandBlaster relied on previous work by [Dionysus Blazakis](https://github.com/dionthegod/XNUSandbox) and Stefan Esser's [code](https://github.com/sektioneins/sandbox_toolkit) and [slides](https://www.slideshare.net/i0n1c/ruxcon-2014-stefan-esser-ios8-containers-sandboxes-and-entitlements).\n\nThe reverser (in the `reverse-sandbox/` folder) and the helper tool (in the `helpers/` folder) run on any Python running platform.\n\nSandBlaster may be installed and run standalone, though we recommend installing and running it from within [iExtractor](https://github.com/malus-security/iExtractor). Check the [iExtractor documentation](https://github.com/malus-security/iExtractor/blob/master/README.md) for information.\n\niExtractor is open source software released under the 3-clause BSD license.\n\n## Installation\n\nSandBlaster requires Python2 for the reverser (in `reverse-sandbox/`), Python3 with `lief` library for helper script (in `helpers/`).\n\nAfter cloning the SandBlaster repository, you have to install `lief` for Python3:\n```\npip3 install lief\n```\n\nIf the installation of `lief` fails you need compile to it. More information about how to compile it can be found on the [wiki page](https://lief.quarkslab.com/doc/stable/compilation.html).\n\n## Usage\n\nIn order to use SandBlaster you need access to the binary sandbox profiles and the sandbox operations, a set of strings that define sandbox-specific actions. Sandbox operations and sandbox profiles are extracted using the `helpers/extract_sandbox_data.py` script. Sandbox profiles are extracted from the kernel sandbox extension (as a bundle for iOS 4 and 9-11) or from kernel cache (as a bundle for iOS 12) or from the `sandboxd` file in the iOS filesystem (for iOS 5-8). Sandbox operations are extracted either from kernel extension (for iOS 4-11) or from kernel cache (for iOS 12).\n\nSo, as input data, SandBlaster requires the kernelcache, the kernel sandbox extension and the `sandboxd` file. Information and scripts on extracting them from a publicly available IPSW (*iPhone Software*) file is presented by [iExtractor](https://github.com/malus-security/iExtractor).\n\nBelow are the steps and commands to reverse the sandbox profiles for iOS 8.4.1, assuming the sandbox kernel extension (`com.apple.security.sandbox.kext`) and the `sandboxd` file are available:\n\n```\n# Extract sandbox operations from kernelcache.\ncd helpers/\n./extract_sandbox_data.py -o iPad2,1_8.4.1_12H321.sb_ops iPad2,1_8.4.1_12H321.com.apple.security.sandox.kext 8.4.1\n# Extract binary sandbox profile files from sandboxd.\nmkdir iPad2,1_8.4.1_12H321.sandbox_profiles\n./extract_sandbox_data.py -O iPad2,1_8.4.1_12H321.sandbox_profiles/ iPad2,1_8.4.1_12H321.sandboxd 8.4.1\n# Reverse all binary sandbox profiles.\ncd ../reverse-sandbox/\nmkdir iPad2,1_8.4.1_12H321.reversed_profiles\nfor i in ../helpers/iPad2,1_8.4.1_12H321.sandbox_profiles/*; do python reverse_sandbox.py -r 8.4.1 -o ../helpers/iPad2,1_8.4.1_12H321.sb_ops -d iPad2,1_8.4.1_12H321.reversed_profiles/ \"$i\"; done\n```\n\nBelow are the steps and commands to reverse the sandbox profiles for iOS 9.3, assuming the sandbox kernel extension (`com.apple.security.sandbox.kext`) is available:\n\n```\n# Extract sandbox operations from kernelcache.\ncd helpers/\n./extract_sandbox_data.py -o iPhone5,1_9.3_13E237.sb_ops iPhone5,1_9.3_13E237.com.apple.security.sandox.kext 9.3\n# Extract sandbox profile bundle from kernel sandbox extension.\n./extract_sandbox_data.py -O . iPhone5,1_9.3_13E237.com.apple.security.sandox.kext 9.3\ncd ../reverse-sandbox/\n# Reverse all binary sandbox profiles in sandbox bundle.\nmkdir iPhone5,1_9.3_13E237.reversed_profiles\n# Print all sandbox profiles in bundle.\npython reverse_sandbox.py -r 9.3 -o ../helpers/iPhone5,1_9.3_13E237.sb_ops -d iPhone5,1_9.3_13E237.reversed_profiles/ ../helpers/sandbox_bundle -psb\n# Do actual reversing.\npython reverse_sandbox.py -r 9.3 -o ../helpers/iPhone5,1_9.3_13E237.sb_ops -d iPhone5,1_9.3_13E237.reversed_profiles/ ../helpers/sandbox_bundle\n```\n\nThe extraction of the binary sandbox profiles differs between iOS <= 8 and iOS >= 9. Since iOS >= 9 the binary sandbox profiles are stored in a sandbox bundle in the kernel sandbox extension. The `helpers/extract_sandbox_data.py` script extracts them appropriately depending on the iOS version.\n\nThe `-psb` option for `reverse_sandbox.py` prints out the sandbox profiles part of a sandbox bundle without doing the actual reversing.\n\nThe `reverse_sandbox.py` script needs to be run in its directory (`reverse-sandbox/`) since it needs the other Python modules and the `logger.config` file.\n\n## Internals\n\nThe `helpers/` subfolder contains helper scripts that provide a nicer interface for the external tools.\n\nThe actual reverser is part of the `reverse-sandbox/` folder. Files here can be categorized as follows:\n\n * The main script is `reverse_sandbox.py`. It parses the command line arguments, does basic parsing of the input binary file (extracts sections) and calls the appropriate functions from the other modules.\n * The core of the implementation is `operation_node.py`. It provides functions to build the rules graph corresponding to the sandbox profile and to convert the graph to SBPL. It is called by `reverse_sandbox.py`.\n * Sandbox filters (i.e. match rules inside sandbox profiles) are handled by the implementation in `sandbox_filter.py` and the configuration in `filters.json`, `filter_list.py` and `filters.py`. Filter specific functions are called by `operation_node.py`.\n * Regular expression reversing is handled by `sandbox_regex.py` and `regex_parse.py`. `regex_parse.py` is the back end parser that converts the binary representation to a basic graph. `sandbox_regex.py` converts the graph representation (an automaton) to an actual regular expression (i.e. a string of characters and metacharacters). It is called by `reverse_sandbox.py` for parsing regular expressions, with the resulting regular expression list being passed to the functions exposed by `operation_node.py`; `operation_node.py` passes them on to sandbox filter handling files.\n * The new format for storing strings since iOS 10 is handled by `reverse_string.py`. The primary `SandboxString` class in `reverse_string.py` is used in `sandbox_filter.py`.\n * Logging is configured in the `logger.config` file. By default, `INFO` and higher level messages are printed to the console, while `DEBUG` and higher level messages are printed to the `reverse.log` file.\n\n## Supported iOS Versions\n\nSandBlaster works for iOS version 4 onwards including iOS 12. Apple has been making updates to the binary format of the sandbox profiles: since iOS 9 sandbox profiles are stored in a bundle, since iOS 10 strings are aggregated together in a specialied binary format. iOS 11 didn't bring any change to the format.\n\n## Community\n\nJoin us on [Discord](https://discord.gg/m3gjuyHYw9) for live discussions.\n" }, { "path": "helpers/extract_sandbox_data.py", "content": "#!/usr/bin/env python3\n\nimport sys\nimport argparse\nimport struct\nimport lief\n\nCSTRING_SECTION = '__cstring'\nCONST_SECTION = '__const'\nDATA_SECTION = '__data'\n\n\ndef binary_get_word_size(binary: lief.MachO.Binary):\n \"\"\"Gets the word size of the given binary\n\n The Mach-O binary has 'magic' bytes. These bytes can be used for checking\n whether the binary is 32bit or 64bit.\n Note: iOS 4 and 5 are different to the other sandbox profiles as they have\n no magic values.\n\n Args:\n binary: A sandbox profile in its binary form.\n\n Returns:\n 4: for 32bit MachO binaries\n 8: for 64bit MachO binaries\n \"\"\"\n\n assert (binary.header.magic in\n [lief.MachO.MACHO_TYPES.MAGIC, lief.MachO.MACHO_TYPES.MAGIC_64])\n\n return 4 if binary.header.magic == lief.MachO.MACHO_TYPES.MAGIC else 8\n\n\ndef unpack(bytes_list):\n \"\"\"Unpacks bytes\n\n The information is stored as little endian so '<' is needed.\n For 32bit 'I' is needed and for 64bit 'Q'.\n\n Args:\n bytes_list: A packed list of bytes.\n\n Returns:\n The unpacked 'higher-order' equivalent.\n \"\"\"\n\n if len(bytes_list) == 4:\n return struct.unpack(' 0:\n vaddr_str = str_sect.virtual_address + strs[0]\n xref_vaddrs = get_xref(binary, vaddr_str)\n\n if len(xref_vaddrs) > 0:\n sects = [binary.section_from_virtual_address(x) for x in xref_vaddrs]\n sects = [s for s in sects if 'const' in s.name.lower()]\n assert len(sects) >= 1 and all([sects[0] == s for s in sects])\n return sects[0]\n\n seg = binary.get_segment('__DATA')\n if seg:\n sects = [s for s in seg.sections if s.name == CONST_SECTION]\n assert len(sects) <= 1\n\n if len(sects) == 1:\n return sects[0]\n\n return binary.get_section(CONST_SECTION)\n\n\ndef is_vaddr_in_section(vaddr, section):\n \"\"\"Checks if given virtual address is inside given section.\n\n Args:\n vaddr: A virtual address.\n section: A section of the binary.\n\n Returns:\n True: if the address is inside the section\n False: Otherwise\n \"\"\"\n\n return vaddr >= section.virtual_address \\\n and vaddr < section.virtual_address + section.size\n\n\ndef unpack_pointer(addr_size, binary, vaddr):\n \"\"\"Unpacks a pointer and untags it if it is necessary.\n\n Args:\n binary: A sandbox profile in its binary form.\n vaddr: A virtual address.\n addr_size: The size of an address (4 or 8).\n\n Returns:\n A pointer.\n \"\"\"\n\n ptr = unpack(\n binary.get_content_from_virtual_address(vaddr, addr_size))\n if addr_size == 8:\n ptr = untag_pointer(ptr)\n return ptr\n\n\ndef extract_data_tables_from_section(binary: lief.MachO.Binary, to_data, section):\n \"\"\" Generic implementation of table search. A table is formed of adjacent\n pointers to data.\n\n Args:\n binary: A sandbox profile in its binary form.\n to_data: Function that checks if the data is valid. This function\n returns None for invalid data and anything else otherwise.\n section: A section of the binary.\n\n Returns:\n An array of tables (arrays of data).\n \"\"\"\n\n addr_size = binary_get_word_size(binary)\n start_addr = section.virtual_address\n end_addr = section.virtual_address + section.size\n tables = []\n vaddr = start_addr\n\n while vaddr <= end_addr - addr_size:\n ptr = unpack_pointer(addr_size, binary, vaddr)\n\n data = to_data(binary, ptr)\n if data is None:\n vaddr += addr_size\n continue\n\n table = [data]\n vaddr += addr_size\n\n while vaddr <= end_addr - addr_size:\n ptr = unpack_pointer(addr_size, binary, vaddr)\n\n data = to_data(binary, ptr)\n if data is None:\n break\n\n table.append(data)\n vaddr += addr_size\n\n if table not in tables:\n tables.append(table)\n\n vaddr += addr_size\n\n return tables\n\n\ndef extract_string_tables(binary: lief.MachO.Binary):\n \"\"\"Extracts string tables from the given MachO binary.\n\n Args:\n binary: A sandbox profile in its binary form.\n\n Returns:\n The string tables.\n \"\"\"\n\n return extract_data_tables_from_section(binary,\n binary_get_string_from_address,\n get_tables_section(binary))\n\n\ndef extract_separated_profiles(binary, string_tables):\n \"\"\"Extract separated profiles from given MachO binary. It requires all\n string tables. This function is intended to be used for older version\n of iOS(<=7) because in newer versions the sandbox profiles are bundled.\n\n Args:\n binary: A sandbox profile in its binary form.\n string_tables: The extracted string tables.\n\n Returns:\n A zip object with profiles.\n \"\"\"\n\n def get_profile_names():\n \"\"\"Extracts the profile names.\n\n Returns:\n A list with the names of the sandbox profiles.\n \"\"\"\n\n def transform(arr):\n if len(arr) <= 3:\n return None\n\n ans = []\n tmp = []\n for val in arr:\n if val in ['default', '0123456789abcdef']:\n ans.append(tmp)\n tmp = []\n else:\n tmp.append(val)\n ans.append(tmp)\n return ans\n\n def get_sol(posible):\n ans = [arr for arr in posible\n if 'com.apple.sandboxd' in arr]\n assert len(ans) == 1\n return ans[0]\n\n profile_names_v = [transform(v) for v in string_tables]\n profile_names_v = [v for v in profile_names_v if v is not None]\n profile_names_v = [x for v in profile_names_v for x in v]\n return get_sol(profile_names_v)\n\n def get_profile_contents():\n \"\"\"Extracts the profile names.\n\n Returns:\n The contents of the sandbox profiles.\n \"\"\"\n\n def get_profile_content(binary, vaddr):\n addr_size = binary_get_word_size(binary)\n section = get_section_from_segment(binary, \"__DATA\", DATA_SECTION)\n\n if not is_vaddr_in_section(vaddr, section):\n return None\n\n data = binary.get_content_from_virtual_address(vaddr, 2 * addr_size)\n if len(data) != 2 * addr_size:\n return None\n\n data_vaddr = unpack(data[:addr_size])\n size = unpack(data[addr_size:])\n if not is_vaddr_in_section(vaddr, section):\n return None\n\n data = binary.get_content_from_virtual_address(data_vaddr, size)\n if len(data) != size:\n return None\n return bytes(data)\n\n contents_v = [v for v in\n extract_data_tables_from_section(binary,\n get_profile_content,\n get_tables_section(binary))\n if len(v) > 3]\n\n assert len(contents_v) == 1\n return contents_v[0]\n\n profile_names = get_profile_names()\n profile_contents = get_profile_contents()\n\n assert len(profile_names) == len(profile_contents)\n return zip(profile_names, profile_contents)\n\n\ndef extract_sbops(string_tables):\n \"\"\" Extracts sandbox operations from a given MachO binary.\n If the sandbox profiles are stored either in sandboxd or sandbox kernel\n extension, the operations are stored always in the kernel extension.\n The sandbox operations are stored similar to the separated sandbox profiles\n but this time we have only one table: the name table.\n\n Args:\n string_tables: The binary's string tables.\n\n Returns:\n The sandbox operations.\n \"\"\"\n\n def transform(arr):\n if len(arr) <= 3:\n return None\n\n idxs = []\n for idx, val in enumerate(arr):\n if val == 'default':\n idxs.append(idx)\n\n return [arr[idx:] for idx in idxs]\n\n def get_sol(possible):\n assert len(possible) >= 1\n\n sol = []\n if len(possible) > 1:\n cnt = min(len(arr) for arr in possible)\n for vals in zip(*[val[:cnt] for val in possible]):\n if not all(val == vals[0] for val in vals):\n break\n sol.append(vals[0])\n else:\n sol.append(possible[0][0])\n for pos in possible[0][1:]:\n if pos in ['HOME', 'default']:\n break\n sol.append(pos)\n\n return sol\n\n sbops_v = [transform(v) for v in string_tables]\n sbops_v = [v for v in sbops_v if v is not None and v != []]\n sbops_v = [x for v in sbops_v for x in v]\n\n return get_sol(sbops_v)\n\n\ndef get_ios_major_version(version: str):\n \"\"\"Extracts the major iOS version from a given version.\n\n Args:\n version: A string with the 'full' version.\n Returns:\n An integer with the major iOS version.\n\n \"\"\"\n\n return int(version.split('.')[0])\n\n\ndef findall(searching, pattern):\n \"\"\"Finds all the substring in the given string.\n\n Args:\n searching: A string.\n pattern: A pattern that needs to be searched in the searching string.\n\n Returns:\n The indexes of all substrings equal to pattern inside searching string.\n \"\"\"\n\n i = searching.find(pattern)\n while i != -1:\n yield i\n i = searching.find(pattern, i + 1)\n\n\ndef check_regex(data: bytes, base_index: int, ios_version: int):\n \"\"\" Checks if the regular expression (from sandbox profile) at offset\n base_index from data is valid for newer versions of iOS(>=8).\n\n Args:\n data: An array of bytes.\n base_index: The starting index.\n ios_version: An integer representing the iOS version.\n\n Returns:\n True: if the regular expression is valid for iOS version >= 8.\n False: otherwise.\n \"\"\"\n\n if base_index + 0x10 > len(data):\n return False\n\n if ios_version >= 13:\n size = struct.unpack('I', data[base_index + 0x2: base_index + 0x6])[0]\n else:\n size = struct.unpack('I', data[base_index + 0x4: base_index + 0x8])[0]\n\n if size > 0x1000 or size < 0x8 or base_index + size + 4 > len(data):\n return False\n\n if version != 3:\n return False\n\n if ios_version >= 13:\n sub_size = struct.unpack('= 13).\n\n Args:\n base_index: The starting index.\n count: Bundle size.\n data: An array of bytes.\n Returns:\n The new base index and an offset.\n \"\"\"\n\n re_offset = base_index + 12\n op_nodes_count = struct.unpack('=8).\n\n Args:\n data: An array of bytes.\n base_index: The starting index.\n ios_version: An integer representing the iOS version.\n\n Returns:\n True: if the sandbox profile bundle is valid.\n False: otherwise.\n \"\"\"\n\n if len(data) - base_index < 50:\n return False\n re_offset, aux = struct.unpack('<2H', data[base_index + 2:base_index + 6])\n\n if ios_version >= 13:\n count = struct.unpack('= 12:\n count = (aux - re_offset) * 4\n # bundle should be big\n if count < 0x10:\n return False\n else:\n count = aux\n\n if count > 0x1000 or re_offset < 0x10:\n return False\n\n if ios_version >= 13:\n base_index, re_offset = unpack_for_newer_ios(base_index, count, data)\n\n else:\n re_offset = base_index + re_offset * 8\n if len(data) - re_offset < count * 2:\n return False\n\n for off_index in range(re_offset, re_offset + 2 * count, 2):\n index = struct.unpack('\\n\"\n (name, argument) = self.value.values()\n if argument == None:\n result_str += (level+1)*\"\\t\" + \"\\n\"\n else:\n arg = str(argument).replace('&', '&').replace('\"', '"').replace('\\'', ''').replace('<', '<').replace('>', '>')\n result_str += (level+1)*\"\\t\" + \"\\n\"\n result_str += level*\"\\t\" + \"\\n\"\n else:\n (name, argument) = self.value.values()\n if argument == None:\n result_str += level*\"\\t\" + \"\\n\"\n else:\n arg = str(argument).replace('&', '&').replace('\"', '"').replace('\\'', ''').replace('<', '<').replace('>', '>')\n result_str += level*\"\\t\" + \"\\n\"\n elif self.is_type_require_entitlement():\n if self.is_not:\n result_str += level*\"\\t\" + \"\\n\"\n level += 1\n result_str += level*\"\\t\" + \"', '>')\n result_str += \" value=\\\"\" + _tmp + \"\\\" />\\n\"\n else:\n _tmp = str(n.value)[21:-1].replace('&', '&').replace('\"', '"').replace('\\'', ''').replace('<', '<').replace('>', '>')\n result_str += \" value=\\\"\" + _tmp + \"\\\">\\n\"\n result_str += i.recursive_xml_str(level+1, self.is_not)\n result_str += level*\"\\t\" + \"\\n\"\n if self.is_not:\n level -= 1\n result_str += level*\"\\t\" + \"\\n\"\n else:\n result_str += level*\"\\t\" + \"\\n\"\n for i, v in enumerate(self.value):\n result_str += v.recursive_xml_str(level+1, recursive_is_not)\n result_str += level*\"\\t\" + \"\\n\"\n return result_str\n\n def __str__(self):\n return self.recursive_str(1, False)\n\n def str_debug(self):\n return self.recursive_str_debug(1, False)\n\n def str_simple(self):\n if self.is_type_single():\n return self.value.str_debug()\n elif self.is_type_require_any():\n return \"require-any\"\n elif self.is_type_require_all():\n return \"require-all\"\n elif self.is_type_require_entitlement():\n return self.value.str_debug()[1:-1]\n elif self.is_type_start():\n return \"start\"\n else:\n return \"unknown-type\"\n\n def str_print_debug(self):\n if self.is_type_single():\n return (self.value.str_debug(), None)\n elif self.is_type_require_any():\n return (\"(require-any\", \")\")\n elif self.is_type_require_all():\n return (\"(require-all\", \")\")\n elif self.is_type_require_entitlement():\n return (self.value.str_debug()[:-1], \")\")\n elif self.is_type_start():\n return (None, None)\n else:\n return (\"unknown-type\", None)\n\n def str_print(self):\n if self.is_type_single():\n return (str(self.value), None)\n elif self.is_type_require_any():\n return (\"(require-any\", \")\")\n elif self.is_type_require_all():\n return (\"(require-all\", \")\")\n elif self.is_type_require_entitlement():\n return (str(self.value)[:-1], \")\")\n elif self.is_type_start():\n return (None, None)\n else:\n return (\"unknown-type\", None)\n\n def str_print_not(self):\n result_str = \"\"\n if self.is_type_single():\n if self.is_not:\n value = str(self.value)\n if \"(require-any\" in value:\n result_str = self.value.str_not()\n else:\n result_str += \"(require-not \" + str(self.value) + \")\"\n return result_str\n\n def xml_str(self):\n return self.recursive_xml_str(3, False)\n\n\nclass ReducedEdge():\n start = None\n end = None\n\n def __init__(self, start=None, end=None):\n self.start = start\n self.end = end\n\n def str_debug(self):\n return self.start.str_debug() + \" -> \" + self.end.str_debug()\n\n def str_simple(self):\n #print \"start: %s\" % (self.start.str_simple())\n #print \"end: %s\" % (self.end.str_simple())\n return \"%s -----> %s\" % (self.start.str_simple(), self.end.str_simple())\n\n def __str__(self):\n return str(self.start) + \" -> \" + str(self.end)\n\n\nclass ReducedGraph():\n vertices = []\n edges = []\n final_vertices = []\n reduce_changes_occurred = False\n\n def __init__(self):\n self.vertices = []\n self.edges = []\n self.final_vertices = []\n self.reduce_changes_occurred = False\n\n def add_vertice(self, v):\n self.vertices.append(v)\n\n def add_edge(self, e):\n self.edges.append(e)\n\n def add_edge_by_vertices(self, v_start, v_end):\n e = ReducedEdge(v_start, v_end)\n self.edges.append(e)\n\n def set_final_vertices(self):\n self.final_vertices = []\n for v in self.vertices:\n is_final = True\n for e in self.edges:\n if v == e.start:\n is_final = False\n break\n if is_final:\n self.final_vertices.append(v)\n\n def contains_vertice(self, v):\n return v in self.vertices\n\n def contains_edge(self, e):\n return e in self.edges\n\n def contains_edge_by_vertices(self, v_start, v_end):\n for e in self.edges:\n if e.start == v_start and e.end == v_end:\n return True\n return False\n\n def get_vertice_by_value(self, value):\n for v in self.vertices:\n if v.is_type_single():\n if v.value == value:\n return v\n\n def get_edge_by_vertices(self, v_start, v_end):\n for e in self.edges:\n if e.start == v_start and e.end == v_end:\n return e\n return None\n\n def remove_vertice(self, v):\n edges_copy = list(self.edges)\n for e in edges_copy:\n if e.start == v or e.end == v:\n self.edges.remove(e)\n if v in self.vertices:\n self.vertices.remove(v)\n\n def remove_vertice_update_decision(self, v):\n edges_copy = list(self.edges)\n for e in edges_copy:\n if e.start == v:\n self.edges.remove(e)\n if e.end == v:\n e.start.decision = v.decision\n self.edges.remove(e)\n if v in self.vertices:\n self.vertices.remove(v)\n\n def remove_edge(self, e):\n if e in self.edges:\n self.edges.remove(e)\n\n def remove_edge_by_vertices(self, v_start, v_end):\n e = self.get_edge_by_vertices(v_start, v_end)\n if e:\n self.edges.remove(e)\n\n def replace_vertice_in_edge_start(self, old, new):\n global replace_occurred\n for e in self.edges:\n if e.start == old:\n e.start = new\n replace_occurred = True\n else:\n if isinstance(e.start.value, list):\n e.start.replace_in_list(old, new)\n if replace_occurred:\n e.start.decision = new.decision\n\n def replace_vertice_in_edge_end(self, old, new):\n global replace_occurred\n for e in self.edges:\n if e.end == old:\n e.end = new\n replace_occurred = True\n else:\n if isinstance(e.end.value, list):\n e.end.replace_in_list(old, new)\n if replace_occurred:\n e.end.decision = new.decision\n\n def replace_vertice_in_single_vertices(self, old, new):\n for v in self.vertices:\n if len(self.get_next_vertices(v)) == 0 and len(self.get_prev_vertices(v)) == 0:\n if isinstance(v.value, list):\n v.replace_in_list(old, new)\n\n def replace_vertice_list(self, old, new):\n for v in self.vertices:\n if isinstance(v.value, list):\n v.replace_sublist_in_list(old, new)\n if set(self.get_next_vertices(v)) == set(old):\n for n in old:\n self.remove_edge_by_vertices(v, n)\n self.add_edge_by_vertices(v, new)\n if set(self.get_prev_vertices(v)) == set(old):\n for n in old:\n self.remove_edge_by_vertices(n, v)\n self.add_edge_by_vertices(new, v)\n\n def get_next_vertices(self, v):\n next_vertices = []\n for e in self.edges:\n if e.start == v:\n next_vertices.append(e.end)\n return next_vertices\n\n def get_prev_vertices(self, v):\n prev_vertices = []\n for e in self.edges:\n if e.end == v:\n prev_vertices.append(e.start)\n return prev_vertices\n\n def get_start_vertices(self):\n start_vertices = []\n for v in self.vertices:\n if not self.get_prev_vertices(v):\n start_vertices.append(v)\n return start_vertices\n\n def get_end_vertices(self):\n end_vertices = []\n for v in self.vertices:\n if not self.get_next_vertices(v):\n end_vertices.append(v)\n return end_vertices\n\n def reduce_next_vertices(self, v):\n next_vertices = self.get_next_vertices(v)\n if len(next_vertices) <= 1:\n return\n self.reduce_changes_occurred = True\n new_vertice = ReducedVertice(\"require-any\", next_vertices, next_vertices[0].decision)\n add_to_final = False\n for n in next_vertices:\n self.remove_edge_by_vertices(v, n)\n self.replace_vertice_list(next_vertices, new_vertice)\n for n in next_vertices:\n if n in self.final_vertices:\n self.final_vertices.remove(n)\n add_to_final = True\n # If no more next vertices, remove vertice.\n if not self.get_next_vertices(n):\n if n in self.vertices:\n self.vertices.remove(n)\n self.add_edge_by_vertices(v, new_vertice)\n self.add_vertice(new_vertice)\n if add_to_final:\n self.final_vertices.append(new_vertice)\n\n def reduce_prev_vertices(self, v):\n prev_vertices = self.get_prev_vertices(v)\n if len(prev_vertices) <= 1:\n return\n self.reduce_changes_occurred = True\n new_vertice = ReducedVertice(\"require-any\", prev_vertices, v.decision)\n for p in prev_vertices:\n self.remove_edge_by_vertices(p, v)\n self.replace_vertice_list(prev_vertices, new_vertice)\n for p in prev_vertices:\n # If no more prev vertices, remove vertice.\n if not self.get_prev_vertices(p):\n if p in self.vertices:\n self.vertices.remove(p)\n self.add_vertice(new_vertice)\n self.add_edge_by_vertices(new_vertice, v)\n\n def reduce_vertice_single_prev(self, v):\n global replace_occurred\n prev = self.get_prev_vertices(v)\n if len(prev) != 1:\n logger.debug(\"not a single prev for node\")\n return\n p = prev[0]\n nexts = self.get_next_vertices(p)\n if len(nexts) > 1 or nexts[0] != v:\n logger.debug(\"multiple nexts for prev\")\n return\n require_all_vertices = []\n if p.is_type_require_all():\n require_all_vertices.extend(p.value)\n else:\n require_all_vertices.append(p)\n if v.is_type_require_all():\n require_all_vertices.extend(v.value)\n else:\n require_all_vertices.append(v)\n new_vertice = ReducedVertice(\"require-all\", require_all_vertices, v.decision)\n self.remove_edge_by_vertices(p, v)\n replace_occurred = False\n self.replace_vertice_in_edge_start(v, new_vertice)\n self.replace_vertice_in_edge_end(p, new_vertice)\n self.replace_vertice_in_single_vertices(p, new_vertice)\n self.replace_vertice_in_single_vertices(v, new_vertice)\n self.remove_vertice(p)\n self.remove_vertice(v)\n if not replace_occurred:\n self.add_vertice(new_vertice)\n if v in self.final_vertices:\n self.final_vertices.remove(v)\n self.final_vertices.append(new_vertice)\n\n def reduce_vertice_single_next(self, v):\n global replace_occurred\n next = self.get_next_vertices(v)\n if len(next) != 1:\n return\n n = next[0]\n prevs = self.get_prev_vertices(n)\n if len(prevs) > 1 or prevs[0] != v:\n return\n require_all_vertices = []\n if v.is_type_require_all():\n require_all_vertices.extend(v.value)\n else:\n require_all_vertices.append(v)\n if n.is_type_require_all():\n require_all_vertices.extend(n.value)\n else:\n require_all_vertices.append(n)\n new_vertice = ReducedVertice(\"require-all\", require_all_vertices, n.decision)\n self.remove_edge_by_vertices(v, n)\n replace_occurred = False\n self.replace_vertice_in_edge_start(n, new_vertice)\n self.replace_vertice_in_edge_end(e, new_vertice)\n self.replace_vertice_in_single_vertices(v, new_vertice)\n self.replace_vertice_in_single_vertices(n, new_vertice)\n self.remove_vertice(v)\n self.remove_vertice(n)\n if not replace_occurred:\n self.add_vertice(new_vertice)\n if n in self.final_vertices:\n self.final_vertices.remove(n)\n self.final_vertices.append(new_vertice)\n\n def reduce_graph(self):\n self.set_final_vertices()\n\n logger.debug(\"before everything:\\n\" + self.str_simple())\n # Do until no more changes.\n while True:\n self.reduce_changes_occurred = False\n copy_vertices = list(self.vertices)\n for v in copy_vertices:\n self.reduce_next_vertices(v)\n if self.reduce_changes_occurred == False:\n break\n logger.debug(\"after next:\\n\" + self.str_simple())\n # Do until no more changes.\n while True:\n self.reduce_changes_occurred = False\n copy_vertices = list(self.vertices)\n for v in copy_vertices:\n self.reduce_prev_vertices(v)\n if self.reduce_changes_occurred == False:\n break\n logger.debug(\"after next/prev:\\n\" + self.str_simple())\n\n # Reduce graph starting from final vertices. Keep going until\n # final vertices don't change during an iteration.\n while True:\n copy_final_vertices = list(self.final_vertices)\n for v in copy_final_vertices:\n logger.debug(\"reducing single prev vertex: \" + v.str_debug())\n self.reduce_vertice_single_prev(v)\n logger.debug(\"### new graph is:\")\n logger.debug(self.str_simple())\n if set(copy_final_vertices) == set(self.final_vertices):\n break\n for e in self.edges:\n v = e.end\n logger.debug(\"reducing single prev vertex: \" + v.str_debug())\n self.reduce_vertice_single_prev(v)\n logger.debug(\"after everything:\\n\" + self.str_simple())\n\n def reduce_graph_with_metanodes(self):\n # Add require-any metanode if current node has multiple successors.\n copy_vertices = list(self.vertices)\n for v in copy_vertices:\n nlist = self.get_next_vertices(v)\n if len(nlist) >= 2:\n new_node = ReducedVertice(\"require-any\", None, None)\n self.add_vertice(new_node)\n self.add_edge_by_vertices(v, new_node)\n for n in nlist:\n self.remove_edge_by_vertices(v, n)\n self.add_edge_by_vertices(new_node, n)\n\n start_list = self.get_start_vertices()\n new_node = ReducedVertice(\"start\", None, None)\n self.add_vertice(new_node)\n for s in start_list:\n self.add_edge_by_vertices(new_node, s)\n\n # Add require-all metanode if current node has a require-any as a predecessor and is followed by another node.\n copy_vertices = list(self.vertices)\n for v in copy_vertices:\n prev_vertices = list(self.get_prev_vertices(v))\n next_vertices = list(self.get_next_vertices(v))\n for p in prev_vertices:\n if (p.is_type_require_any() or p.is_type_start()) and next_vertices:\n # Except for when a require-entitlement ending block.\n if v.is_type_require_entitlement():\n has_next_nexts = False\n for n in next_vertices:\n if n.is_type_require_any():\n for n2 in self.get_next_vertices(n):\n if self.get_next_vertices(n2):\n has_next_nexts = True\n break\n else:\n if self.get_next_vertices(n):\n has_next_nexts = True\n break\n if not has_next_nexts:\n continue\n new_node = ReducedVertice(\"require-all\", None, None)\n self.add_vertice(new_node)\n self.remove_edge_by_vertices(p, v)\n self.add_edge_by_vertices(p, new_node)\n self.add_edge_by_vertices(new_node, v)\n\n def str_simple_with_metanodes(self):\n logger.debug(\"==== vertices:\\n\")\n for v in self.vertices:\n logger.debug(v.str_simple())\n logger.debug(\"==== edges:\\n\")\n for e in self.edges:\n logger.debug(e.str_simple())\n\n def str_simple(self):\n message = \"==== vertices:\\n\"\n for v in self.vertices:\n message += \"decision: \" + str(v.decision) + \"\\t\" + v.str_debug() + \"\\n\"\n message += \"==== final vertices:\\n\"\n for v in self.final_vertices:\n message += \"decision: \" + str(v.decision) + \"\\t\" + v.str_debug() + \"\\n\"\n message += \"==== edges:\\n\"\n for e in self.edges:\n message += \"\\t\" + e.str_debug() + \"\\n\"\n return message\n\n def __str__(self):\n result_str = \"\"\n for v in self.vertices:\n result_str += \"(\" + str(v.decision) + \" \"\n if len(self.get_next_vertices(v)) == 0 and len(self.get_next_vertices(v)) == 0:\n if v in self.final_vertices:\n result_str += str(v) + \"\\n\"\n result_str += \")\\n\"\n for e in self.edges:\n result_str += str(e) + \"\\n\"\n result_str += \"\\n\"\n return result_str\n\n def remove_builtin_filters(self):\n copy_vertices = list(self.vertices)\n for v in copy_vertices:\n if re.search(\"###\\$\\$\\$\\*\\*\\*\", str(v)):\n self.remove_vertice_update_decision(v)\n\n def reduce_integrated_vertices(self, integrated_vertices):\n if len(integrated_vertices) == 0:\n return (None, None)\n if len(integrated_vertices) > 1:\n return (ReducedVertice(\"require-any\", integrated_vertices, integrated_vertices[0].decision), integrated_vertices[0].decision)\n require_all_vertices = []\n v = integrated_vertices[0]\n decision = None\n while True:\n if not re.search(\"entitlement-value #t\", str(v)):\n require_all_vertices.append(v)\n next_vertices = self.get_next_vertices(v)\n if decision == None and v.decision != None:\n decision = v.decision\n self.remove_vertice(v)\n if v in self.final_vertices:\n self.final_vertices.remove(v)\n if next_vertices:\n v = next_vertices[0]\n else:\n break\n if len(require_all_vertices) == 0:\n return (None, v.decision)\n if len(require_all_vertices) == 1:\n return (ReducedVertice(value=require_all_vertices[0].value, decision=require_all_vertices[0].decision, is_not=require_all_vertices[0].is_not), v.decision)\n return (ReducedVertice(\"require-all\", require_all_vertices, require_all_vertices[len(require_all_vertices)-1].decision), v.decision)\n\n def aggregate_require_entitlement(self, v):\n next_vertices = []\n prev_vertices = self.get_prev_vertices(v)\n integrated_vertices = []\n for n in self.get_next_vertices(v):\n if not re.search(\"entitlement-value\", str(n)):\n next_vertices.append(n)\n break\n integrated_vertices.append(n)\n current_list = [ n ]\n while current_list:\n current = current_list.pop()\n for n2 in self.get_next_vertices(current):\n if not re.search(\"entitlement-value\", str(n2)):\n self.remove_edge_by_vertices(current, n2)\n next_vertices.append(n2)\n else:\n current_list.append(n2)\n new_vertice = ReducedVertice(type=\"require-entitlement\", value=(v, None), decision=None, is_not=v.is_not)\n for p in prev_vertices:\n self.remove_edge_by_vertices(p, v)\n self.add_edge_by_vertices(p, new_vertice)\n for n in next_vertices:\n self.remove_edge_by_vertices(v, n)\n self.add_edge_by_vertices(new_vertice, n)\n for i in integrated_vertices:\n self.remove_edge_by_vertices(v, i)\n self.remove_vertice(v)\n self.add_vertice(new_vertice)\n if v in self.final_vertices:\n self.final_vertices.remove(v)\n self.final_vertices.append(new_vertice)\n (new_integrate, decision) = self.reduce_integrated_vertices(integrated_vertices)\n for i in integrated_vertices:\n self.remove_vertice(i)\n if i in self.final_vertices:\n self.final_vertices.remove(i)\n new_vertice.set_integrated_vertice(new_integrate)\n new_vertice.set_decision(decision)\n\n def aggregate_require_entitlement_nodes(self):\n copy_vertices = list(self.vertices)\n idx = 0\n while idx < len(copy_vertices):\n v = copy_vertices[idx]\n if re.search(\"require-entitlement\", str(v)):\n self.aggregate_require_entitlement(v)\n idx += 1\n\n def cleanup_filters(self):\n self.remove_builtin_filters()\n self.aggregate_require_entitlement_nodes()\n\n def remove_builtin_filters_with_metanodes(self):\n copy_vertices = list(self.vertices)\n for v in copy_vertices:\n if re.search(\"###\\$\\$\\$\\*\\*\\*\", v.str_simple()):\n self.remove_vertice(v)\n elif re.search(\"entitlement-value #t\", v.str_simple()):\n self.remove_vertice(v)\n elif re.search(\"entitlement-value-regex #\\\"\\.\\\"\", v.str_simple()):\n v.value.non_terminal.argument = \"#\\\".+\\\"\"\n elif re.search(\"global-name-regex #\\\"\\.\\\"\", v.str_simple()):\n v.value.non_terminal.argument = \"#\\\".+\\\"\"\n elif re.search(\"local-name-regex #\\\"\\.\\\"\", v.str_simple()):\n v.value.non_terminal.argument = \"#\\\".+\\\"\"\n\n def replace_require_entitlement_with_metanodes(self, v):\n prev_list = self.get_prev_vertices(v)\n next_list = self.get_next_vertices(v)\n new_node = ReducedVertice(type=\"require-entitlement\", value=v.value, decision=None, is_not=v.is_not)\n self.add_vertice(new_node)\n self.remove_vertice(v)\n for p in prev_list:\n self.add_edge_by_vertices(p, new_node)\n for n in next_list:\n self.add_edge_by_vertices(new_node, n)\n\n def aggregate_require_entitlement_with_metanodes(self):\n copy_vertices = list(self.vertices)\n for v in copy_vertices:\n if re.search(\"require-entitlement\", str(v)):\n self.replace_require_entitlement_with_metanodes(v)\n\n def cleanup_filters_with_metanodes(self):\n self.remove_builtin_filters_with_metanodes()\n self.aggregate_require_entitlement_with_metanodes()\n\n def print_vertices_with_operation(self, operation, out_f):\n allow_vertices = [v for v in self.vertices if v.decision == \"allow\"]\n deny_vertices = [v for v in self.vertices if v.decision == \"deny\"]\n if allow_vertices:\n out_f.write(\"(allow %s \" % (operation))\n if len(allow_vertices) > 1:\n for v in allow_vertices:\n out_f.write(\"\\n\" + 8*\" \" + str(v))\n else:\n out_f.write(str(allow_vertices[0]))\n out_f.write(\")\\n\")\n if deny_vertices:\n out_f.write(\"(deny %s \" % (operation))\n if len(deny_vertices) > 1:\n for v in deny_vertices:\n out_f.write(\"\\n\" + 8*\" \" + str(v))\n else:\n out_f.write(str(deny_vertices[0]))\n out_f.write(\")\\n\")\n\n def print_vertices_with_operation_metanodes(self, operation, default_is_allow, out_f):\n # Return if only start node in list.\n if len(self.vertices) == 1 and self.vertices[0].is_type_start():\n return\n # Use reverse of default rule.\n if default_is_allow:\n out_f.write(\"(deny %s\" % (operation))\n else:\n out_f.write(\"(allow %s\" % (operation))\n vlist = []\n start_list = self.get_start_vertices()\n start_list.reverse()\n vlist.insert(0, (None, 0))\n for s in start_list:\n vlist.insert(0, (s, 1))\n while True:\n if not vlist:\n break\n (cnode, indent) = vlist.pop(0)\n if not cnode:\n out_f.write(\")\")\n continue\n (first, last) = cnode.str_print()\n if first:\n if cnode.is_not:\n if cnode.str_print_not() != \"\":\n out_f.write(\"\\n\" + indent * \"\\t\" + cnode.str_print_not())\n else:\n out_f.write(\"\\n\" + indent * \"\\t\" + \"(require-not \" + first)\n if cnode.is_type_require_any() or cnode.is_type_require_all() or cnode.is_type_require_entitlement():\n vlist.insert(0, (None, indent))\n else:\n out_f.write(\")\")\n else:\n out_f.write(\"\\n\" + indent * \"\\t\" + first)\n if last:\n vlist.insert(0, (None, indent))\n next_vertices_list = self.get_next_vertices(cnode)\n if next_vertices_list:\n if cnode.is_type_require_any() or cnode.is_type_require_all() or cnode.is_type_require_entitlement():\n indent += 1\n next_vertices_list.reverse()\n if cnode.is_type_require_entitlement():\n pos = 0\n for n in next_vertices_list:\n if (n.is_type_single() and not re.search(\"entitlement-value\", n.str_simple())) or \\\n n.is_type_require_entitlement():\n vlist.insert(pos + 1, (n, indent-1))\n else:\n vlist.insert(0, (n, indent))\n pos += 1\n else:\n for n in next_vertices_list:\n vlist.insert(0, (n, indent))\n out_f.write(\"\\n\")\n\n def dump_xml(self, operation, out_f):\n allow_vertices = [v for v in self.vertices if v.decision == \"allow\"]\n deny_vertices = [v for v in self.vertices if v.decision == \"deny\"]\n if allow_vertices:\n out_f.write(\"\\t\\n\" % (operation))\n out_f.write(\"\\t\\t\\n\")\n for v in allow_vertices:\n out_f.write(v.xml_str())\n out_f.write(\"\\t\\t\\n\")\n out_f.write(\"\\t\\n\")\n if deny_vertices:\n out_f.write(\"\\t\\n\" % (operation))\n out_f.write(\"\\t\\t\\n\")\n for v in deny_vertices:\n out_f.write(v.xml_str())\n out_f.write(\"\\t\\t\\n\")\n out_f.write(\"\\t\\n\")\n\n\ndef reduce_operation_node_graph(g):\n # Create reduced graph.\n rg = ReducedGraph()\n for node_iter in g.keys():\n rv = ReducedVertice(value=node_iter, decision=g[node_iter][\"decision\"], is_not=g[node_iter][\"not\"])\n rg.add_vertice(rv)\n\n for node_iter in g.keys():\n rv = rg.get_vertice_by_value(node_iter)\n for node_next in g[node_iter][\"list\"]:\n rn = rg.get_vertice_by_value(node_next)\n rg.add_edge_by_vertices(rv, rn)\n\n # Handle special case for require-not (require-enitlement (...)).\n l = len(g.keys())\n for idx, node_iter in enumerate(g.keys()):\n rv = rg.get_vertice_by_value(node_iter)\n if not re.search(\"require-entitlement\", str(rv)):\n continue\n if not rv.is_not:\n continue\n c_idx = idx\n while True:\n c_idx += 1\n if c_idx >= l:\n break\n rn = rg.get_vertice_by_value(list(g.keys())[c_idx])\n if not re.search(\"entitlement-value\", str(rn)):\n break\n prevs_rv = rg.get_prev_vertices(rv)\n prevs_rn = rg.get_prev_vertices(rn)\n if sorted(prevs_rv) != sorted(prevs_rn):\n continue\n for pn in prevs_rn:\n rg.remove_edge_by_vertices(rn, pn)\n rg.add_edge_by_vertices(rv, rn)\n\n rg.cleanup_filters_with_metanodes()\n for node_iter in g.keys():\n rv = rg.get_vertice_by_value(node_iter)\n rg.reduce_graph_with_metanodes()\n return rg\n\n\ndef main():\n if len(sys.argv) != 4:\n print >> sys.stderr, \"Usage: %s binary_sandbox_file operations_file ios_version\" % (sys.argv[0])\n sys.exit(-1)\n\n ios_major_version = int(sys.argv[3].split('.')[0])\n # Read sandbox operations.\n sb_ops = [l.strip() for l in open(sys.argv[2])]\n num_sb_ops = len(sb_ops)\n logger.info(\"num_sb_ops:\", num_sb_ops)\n\n f = open(sys.argv[1], \"rb\")\n operation_nodes = build_operation_nodes(f, num_sb_ops, ios_major_version)\n\n global num_regex\n f.seek(4)\n num_regex = struct.unpack(\"I',\n ''.join([chr(x) for x in re[i:i+4]]))[0]\n node_transition = struct.unpack('>I',\n ''.join([chr(x) for x in re[i+4:i+8]]))[0]\n node_arg = struct.unpack('>I',\n ''.join([chr(x) for x in re[i+8:i+12]]))[0]\n i += 12\n\n logger.debug('node idx:{:#010x} type: {:#02x} arg: {:#010x}' \\\n ' transition: {:#010x}'.format(node_idx, node_type,node_arg,\n node_transition))\n assert(node_type in node_type_dispatch_table)\n regex_list.append(\n node_type_dispatch_table[node_type](\n node_type, node_arg, node_transition, node_idx))\n return i\n\ndef class_parse(re, i, classes, class_idx):\n def transform(x):\n c = chr(x)\n if c in '[]-':\n return '\\\\' + c\n else:\n return c\n\n class_size = struct.unpack('>I',\n ''.join([chr(x) for x in re[i:i+4]]))[0]\n i += 0x4\n content = struct.unpack('>{}I'.format(class_size),\n ''.join([chr(x) for x in re[i:i+4*class_size]]))\n i += 0x4 * class_size\n assert(class_size % 2 == 0)\n\n cls = ''\n for idx in range(0, class_size, 2):\n start = content[idx]\n end = content[idx+1]\n if start != end:\n cls += '{}-{}'.format(transform(start), transform(end))\n else:\n cls += transform(start)\n\n logger.debug('class idx = {:#x} size = {:#x} content=[{}]'.format(\n class_idx, class_size, cls))\n classes.append(cls)\n return i\n\nclass RegexParser(object):\n\n @staticmethod\n def parse(re, i, regex_list):\n node_count = struct.unpack('>I',\n ''.join([chr(x) for x in re[i:i+0x4]]))[0]\n logger.debug('node count = {:#x}'.format(node_count))\n\n start_node = struct.unpack('>I',\n ''.join([chr(x) for x in re[i+0x4:i+0x8]]))[0]\n logger.debug('start node = {:#x}'.format(start_node))\n\n end_node = struct.unpack('>I',\n ''.join([chr(x) for x in re[i+0x8:i+0xC]]))[0]\n logger.debug('end node = {:#x}'.format(end_node))\n\n cclass_count = struct.unpack('>I',\n ''.join([chr(x) for x in re[i+0xC:i+0x10]]))[0]\n logger.debug('character class count = {:#x}'.format(cclass_count))\n\n submatch_count = struct.unpack('>I',\n ''.join([chr(x) for x in re[i+0x10:i+0x14]]))[0]\n i += 0x14\n logger.debug('submatch count = {:#x}'.format(submatch_count))\n\n\n for node_idx in range(node_count):\n i = node_parse(re, i, regex_list, node_idx)\n\n classes = []\n for class_idx in range(cclass_count):\n i = class_parse(re, i, classes, class_idx)\n\n for node in regex_list:\n if node['type'] == 'class':\n node['value'] = '[{}]'.format(classes[node['value']])\n elif node['type'] == 'class_exclude':\n node['value'] = '[{}]'.format(classes[node['value']])\n\n regex_list[start_node]['start_node'] = True\n\n" }, { "path": "reverse-sandbox/regex_parser_v2.py", "content": "import logging\nimport struct\n\nlogging.config.fileConfig(\"logger.config\")\nlogger = logging.getLogger(__name__)\n\ndef parse_character(node_type, node_arg, node_transition, node_idx):\n value = chr(node_arg & 0xff)\n if value == \".\":\n value = \"[.]\"\n return {\n \"pos\": node_idx,\n \"nextpos\": node_transition,\n \"type\": \"character\",\n \"value\": value}\n\ndef parse_end(node_type, node_arg, node_transition, node_idx):\n return {\n \"pos\": node_idx,\n \"nextpos\": node_transition,\n \"type\": \"end\",\n \"value\": 0}\n\ndef parse_jump_forward(node_type, node_arg, node_transition, node_idx):\n jump_to = node_arg\n return {\n \"pos\": node_idx,\n \"nextpos\": node_transition,\n \"type\": \"jump_forward\",\n \"value\": jump_to}\n\ndef parse_jump_backward(node_type, node_arg, node_transition, node_idx):\n jump_to = node_transition\n return {\n \"pos\": node_idx,\n \"nextpos\": node_transition,\n \"type\": \"jump_backward\",\n \"value\": jump_to}\n\ndef parse_beginning_of_line(node_type, node_arg, node_transition, node_idx):\n return {\n \"pos\": node_idx,\n \"nextpos\": node_transition,\n \"type\": \"character\",\n \"value\": \"^\"}\n\ndef parse_end_of_line(node_type, node_arg, node_transition, node_idx):\n return {\n \"pos\": node_idx,\n \"nextpos\": node_transition,\n \"type\": \"character\",\n \"value\": \"$\"}\n\ndef parse_dot(node_type, node_arg, node_transition, node_idx):\n return {\n \"pos\": node_idx,\n \"nextpos\": node_transition,\n \"type\": \"character\",\n \"value\": \".\"}\n\ndef parse_character_class(node_type, node_arg, node_transition, node_idx):\n return {\n \"pos\": node_idx,\n \"nextpos\": node_transition,\n \"type\": \"class\",\n \"value\": node_arg}\n\ndef parse_character_neg_class(node_type, node_arg, node_transition, node_idx):\n return {\n \"pos\": node_idx,\n \"nextpos\": node_transition,\n \"type\": \"class_exclude\",\n \"value\": node_arg}\n\ndef parse_parantheses_open(node_type, node_arg, node_transition, node_idx):\n return parse_jump_backward(node_type, node_arg, node_transition,\n node_idx)\n '''\n return {\n \"pos\": node_idx,\n \"nextpos\": node_transition,\n \"type\": \"character\",\n \"value\": \"(\"}\n '''\n\ndef parse_parantheses_close(node_type, node_arg, node_transition, node_idx):\n return parse_jump_backward(node_type, node_arg, node_transition,\n node_idx)\n '''\n return {\n \"pos\": node_idx,\n \"nextpos\": node_transition,\n \"type\": \"character\",\n \"value\": \")\"}\n '''\n\nnode_type_dispatch_table = {\n 0x10: parse_character,\n 0x22: parse_end,\n 0x25: parse_jump_forward,\n 0x26: parse_jump_forward,\n 0x27: parse_jump_forward,\n 0x28: parse_jump_forward,\n 0x30: parse_dot,\n 0x31: parse_jump_backward,\n 0x32: parse_beginning_of_line,\n 0x33: parse_end_of_line,\n 0x34: parse_character_class,\n 0x35: parse_character_neg_class,\n}\n\n\ndef node_parse(re, i, regex_list, node_idx):\n node_type = struct.unpack('> 4)\n i = i+1\n logger.debug(\"i: %d, num: %d\", i, num)\n values = []\n value = \"[\"\n for j in range(0, num):\n values.append(re[i+2*j])\n values.append(re[i+2*j+1])\n first = values[0]\n last = values[2*num-1]\n # In case of excludes.\n if (first > last):\n node_type = \"class_exclude\"\n value += \"^\"\n for j in range(len(values)-1, 0, -1):\n values[j] = values[j-1]\n values[0] = last\n for j in range(0, len(values)):\n if j % 2 == 0:\n values[j] = values[j]+1\n else:\n values[j] = values[j]-1\n else:\n node_type = \"class\"\n for j in range(0, len(values), 2):\n if values[j] < values[j+1]:\n value += \"%s-%s\" % (chr(values[j]), chr(values[j+1]))\n else:\n value += \"%s\" % (chr(values[j]))\n value += \"]\"\n regex_list.append({\n \"pos\": i-6-1,\n \"nextpos\": i + 2 * num - 6,\n \"type\": node_type,\n \"value\": value\n })\n message = \"values: [\", \", \".join([hex(j) for j in values]), \"]\"\n logger.debug(message)\n\n return i + 2 * num - 1\n\ndef parse_end(re, i, regex_list):\n regex_list.append({\n \"pos\": i-6,\n \"nextpos\": i+2-6,\n \"type\": \"end\",\n \"value\": 0\n })\n return i + 1\n\ndef parse(re, i, regex_list):\n # Actual character.\n if re[i] == 0x02:\n i = parse_character(re, i, regex_list)\n # Beginning of line.\n elif re[i] == 0x19:\n parse_beginning_of_line(i, regex_list)\n # End of line.\n elif re[i] == 0x29:\n parse_end_of_line(i, regex_list)\n # Any character.\n elif re[i] == 0x09:\n parse_any_character(i, regex_list)\n # Jump forward.\n elif re[i] == 0x2f:\n i = parse_jump_forward(re, i, regex_list)\n # Jump backward.\n elif re[i] & 0xf == 0xa:\n i = parse_jump_backward(re, i, regex_list)\n # Character class.\n elif re[i] & 0xf == 0xb:\n i = parse_character_class(re, i, regex_list)\n elif re[i] & 0xf == 0x5:\n i = parse_end(re, i, regex_list)\n else:\n logger.warning(\"##########unknown\", hex(re[i]))\n\n return i + 1\n\nclass RegexParser(object):\n\n @staticmethod\n def parse(re, i, regex_list):\n length = struct.unpack('= 13:\n f.seek(get_base_addr(f, ios_version) + offset * 8)\n len = struct.unpack(\"\\n')\n outfile_xml.write('\\n')\n outfile_xml.write('\\n')\n outfile_xml.write('\\n')\n outfile_xml.write('\\n')\n outfile_xml.write('\\n')\n outfile_xml.write('\\n')\n outfile_xml.write('\\n')\n outfile_xml.write('\\n')\n outfile_xml.write(']>\\n')\n outfile_xml.write(\"\\n\")\n\n # Extract node for 'default' operation (index 0).\n default_node = operation_node.find_operation_node_by_offset(operation_nodes, op_table[0])\n outfile.write(\"(%s default)\\n\" % (default_node.terminal))\n outfile_xml.write(\"\\t\\n\" % (default_node.terminal))\n\n # For each operation expand operation node.\n for idx in range(1, len(op_table)):\n offset = op_table[idx]\n operation = sb_ops[idx]\n # Go past operations not in list, in case list is not empty.\n if ops_to_reverse:\n if operation not in ops_to_reverse:\n continue\n logger.info(\"parsing operation %s (index %d)\", operation, idx)\n node = operation_node.find_operation_node_by_offset(operation_nodes, offset)\n if not node:\n logger.info(\"operation %s (index %d) has no operation node\", operation, idx)\n continue\n g = operation_node.build_operation_node_graph(node, default_node)\n if g:\n rg = operation_node.reduce_operation_node_graph(g)\n rg.str_simple_with_metanodes()\n rg.print_vertices_with_operation_metanodes(operation, default_node.terminal.is_allow(), outfile)\n #rg.dump_xml(operation, outfile_xml)\n else:\n logger.info(\"no graph for operation %s (index %d)\", operation, idx)\n if node.terminal and default_node.terminal:\n if node.terminal.type != default_node.terminal.type:\n outfile.write(\"(%s %s)\\n\" % (node.terminal, operation))\n outfile_xml.write(\"\\t\\n\" % (operation, node.terminal))\n\n outfile.close()\n outfile_xml.write(\"\\n\")\n outfile_xml.close()\n\ndef get_ios_major_version(release):\n \"\"\"\n Returns major version of release\n \"\"\"\n return int(release.split('.')[0])\n\ndef is_ios_more_than_10_release(release):\n \"\"\"\n Returns True if release is using newer (iOS >= 10) binary sandbox profile format.\n \"\"\"\n major_version = get_ios_major_version(release)\n if major_version < 10:\n return False\n return True\n\n\ndef display_sandbox_profiles(f, re_table_offset, num_sb_ops, ios_version):\n logger.info(\"Printing sandbox profiles from bundle\")\n if ios_version >= 13:\n f.seek(6)\n elif ios_version >= 12:\n f.seek(12)\n elif ios_version >= 10:\n f.seek(10)\n else:\n f.seek(6)\n num_profiles = struct.unpack(\"= 13:\n f.seek(2)\n num_operation_nodes = struct.unpack(\"= 12:\n f.seek(14 + (num_sb_ops + 2) * 2 * num_profiles)\n elif ios_version >= 10:\n f.seek(12 + (num_sb_ops + 2) * 2 * num_profiles)\n else:\n f.seek(8 + (num_sb_ops + 2) * 2 * num_profiles)\n while True:\n word = struct.unpack(\"= 13:\n f.seek(8)\n regex_table_count = struct.unpack('= 12:\n f.seek(14 + (num_sb_ops + 2) * 2 * i)\n elif ios_version >= 10:\n f.seek(12 + (num_sb_ops + 2) * 2 * i)\n else:\n f.seek(8 + (num_sb_ops + 2) * 2 * i)\n\n name_offset = struct.unpack(\" 0:\n f.seek(vars_offset + i*2)\n else:\n f.seek(vars_offset*8 + i*2)\n current_offset = struct.unpack(\" 0:\n len = struct.unpack(\"= 13:\n # extract operation node table count\n f.seek(2)\n op_nodes_count = struct.unpack('= 6:\n header = struct.unpack(\"= 13:\n re_table_offset = 12\n else:\n re_table_offset = struct.unpack(\"= 12:\n f.seek(8)\n re_table_count = struct.unpack(\" 0:\n if get_ios_major_version(args.release) >= 13:\n f.seek(re_table_offset)\n else:\n f.seek(re_table_offset * 8)\n \n re_offsets_table = struct.unpack(\"<%dH\" % re_table_count, f.read(2 * re_table_count))\n for offset in re_offsets_table:\n if get_ios_major_version(args.release) >= 13:\n f.seek(get_base_addr(f, get_ios_major_version(args.release)) + offset * 8)\n re_length = struct.unpack(\"= 13:\n # get the regex table entries\n f.seek(8)\n regex_table_count = struct.unpack('= 12:\n f.seek(4)\n vars_offset = struct.unpack(\"= 10:\n f.seek(6)\n vars_offset = struct.unpack(\"= 13:\n f.seek(2)\n num_operation_nodes = struct.unpack(\"= 12:\n f.seek(14 + (num_sb_ops + 2) * 2 * num_profiles)\n elif get_ios_major_version(args.release) >= 10:\n f.seek(12 + (num_sb_ops + 2) * 2 * num_profiles)\n else:\n f.seek(8 + (num_sb_ops + 2) * 2 * num_profiles)\n while True:\n word = struct.unpack(\"= 13:\n f.seek(8)\n regex_table_count = struct.unpack('= 12:\n f.seek(14 + (num_sb_ops + 2) * 2 * i)\n elif get_ios_major_version(args.release) >= 10:\n f.seek(12 + (num_sb_ops + 2) * 2 * i)\n else:\n f.seek(8 + (num_sb_ops + 2) * 2 * i)\n\n name_offset = struct.unpack(\"= 13:\n f.seek(8)\n regex_table_count = struct.unpack('= 12:\n f.seek(14 + (num_sb_ops + 2) * 2 * i + 4)\n elif get_ios_major_version(args.release) >= 10:\n f.seek(12 + (num_sb_ops + 2) * 2 * i + 4)\n else:\n f.seek(8 + (num_sb_ops + 2) * 2 * i + 4)\n op_table = struct.unpack(\"<%dH\" % num_sb_ops, f.read(2 * num_sb_ops))\n for idx in range(1, len(op_table)):\n offset = op_table[idx]\n operation = sb_ops[idx]\n logger.info(\"operation %s (index %u) starts at node offset %u (0x%x)\", operation, idx, offset, offset)\n out_fname = os.path.join(out_dir, name + \".sb\")\n process_profile(f, out_fname, sb_ops, ops_to_reverse, op_table, operation_nodes)\n\n else:\n if get_ios_major_version(args.release) >= 12:\n f.seek(4)\n vars_offset = struct.unpack(\"= 10:\n f.seek(6)\n vars_offset = struct.unpack(\"= 6:\n f.seek(6)\n else:\n f.seek(4)\n op_table = struct.unpack(\"<%dH\" % num_sb_ops, f.read(2 * num_sb_ops))\n for idx in range(1, len(op_table)):\n offset = op_table[idx]\n operation = sb_ops[idx]\n logger.info(\"operation %s (index %u) starts at node offset %u (0x%x)\", operation, idx, offset, offset)\n\n # Place file pointer to start of operation nodes area.\n while True:\n word = struct.unpack(\"= 0x80:\n self.update_state_split_byte_read()\n elif b == 0x00 or b == 0x07:\n self.update_state_unknown()\n elif b == 0x05:\n self.update_state_reset_string()\n elif b == 0x08:\n self.update_state_concat_save_byte_read()\n # XXX: Read two bytes. I don't know what they do.\n self.get_next_byte()\n self.get_next_byte()\n elif b >= 0x10 and b < 0x3f:\n self.update_state_constant_read()\n elif b == 0x0b:\n self.update_state_range_byte_read()\n elif b == 0x02:\n self.update_state_plus_read()\n elif b == 0x06:\n self.update_state_reset_string()\n else:\n self.update_state_token_byte_read()\n\n def get_next_byte(self):\n if self.is_end():\n return 0x00\n b = struct.unpack(\"= len(self.binary_string):\n return True\n return False\n\n def reset_base(self):\n if len(self.base_stack) >= 1:\n self.base = self.base_stack.pop()\n\n def reset_base_full(self):\n self.base_stack = []\n self.base = \"\"\n\n\nclass SandboxString:\n rss_stack = []\n\n\n def parse_byte_string(self, s, global_vars):\n rss = ReverseStringState(s)\n base = \"\"\n reset_base = False\n tokens = []\n token = \"\"\n\n while True:\n if rss.state == rss.STATE_UNKNOWN:\n logger.debug(\"state is STATE_UNKNOWN\")\n b = rss.get_next_byte()\n rss.update_state(b)\n elif rss.state == rss.STATE_TOKEN_READ:\n logger.debug(\"state is STATE_TOKEN_READ\")\n b = rss.get_next_byte()\n rss.update_state(b)\n elif rss.state == rss.STATE_TOKEN_BYTE_READ:\n logger.debug(\"state is STATE_TOKEN_BYTE_READ\")\n # String starts with length.\n prev_state = rss.state_stack[len(rss.state_stack)-1]\n if prev_state != rss.STATE_TOKEN_READ:\n token_len = rss.get_length_minus_1()\n rss.read_token(token_len)\n rss.update_state_token_read()\n else:\n logger.warn(\"read token byte from token state\")\n break\n elif rss.state == rss.STATE_CONSTANT_READ:\n logger.debug(\"state is STATE_CONSTANT_READ\")\n b = rss.get_last_byte()\n if b >= 0x10 and b < 0x3f:\n rss.token = \"${\" + global_vars[b-0x10] + \"}\"\n b = rss.get_next_byte()\n rss.update_state(b)\n elif rss.state == rss.STATE_CONCAT_BYTE_READ:\n logger.debug(\"state is STATE_CONCAT_BYTE_READ\")\n if rss.state_stack[len(rss.state_stack)-1] == rss.STATE_TOKEN_READ or \\\n rss.state_stack[len(rss.state_stack)-1] == rss.STATE_CONSTANT_READ or \\\n rss.state_stack[len(rss.state_stack)-1] == rss.STATE_RANGE_BYTE_READ or \\\n rss.state_stack[len(rss.state_stack)-1] == rss.STATE_SINGLE_BYTE_READ or \\\n rss.state_stack[len(rss.state_stack)-1] == rss.STATE_PLUS_READ:\n rss.update_base()\n b = rss.get_next_byte()\n rss.update_state(b)\n elif rss.state == rss.STATE_CONCAT_SAVE_BYTE_READ:\n logger.debug(\"state is STATE_CONCAT_SAVE_BYTE_READ\")\n if rss.state_stack[len(rss.state_stack)-1] == rss.STATE_TOKEN_READ or \\\n rss.state_stack[len(rss.state_stack)-1] == rss.STATE_CONSTANT_READ or \\\n rss.state_stack[len(rss.state_stack)-1] == rss.STATE_RANGE_BYTE_READ or \\\n rss.state_stack[len(rss.state_stack)-1] == rss.STATE_SINGLE_BYTE_READ or \\\n rss.state_stack[len(rss.state_stack)-1] == rss.STATE_PLUS_READ:\n rss.update_base_stack()\n b = rss.get_next_byte()\n rss.update_state(b)\n elif rss.state == rss.STATE_END_BYTE_READ:\n logger.debug(\"state is STATE_END_BYTE_READ\")\n rss.end_current_token()\n rss.reset_base()\n b = rss.get_next_byte()\n rss.update_state(b)\n elif rss.state == rss.STATE_RANGE_BYTE_READ:\n logger.debug(\"state is STATE_RANGE_BYTE_READ\")\n rss.update_base_stack()\n b = rss.get_next_byte()\n b_array = []\n all_ascii = True\n token = \"\"\n for i in range(0, b+1):\n b1 = rss.get_next_byte()\n b2 = rss.get_next_byte()\n if b1 < 0x20 or b1 > 0x7f or b2 < 0x20 or b2 > 0x7f:\n all_ascii = False\n b_array.append((b1,b2))\n if all_ascii == False:\n (b1, b2) = b_array[0]\n (b3, b4) = b_array[1]\n if b2 == 0xff and b3 == 0x00:\n if b1-1 == b4+1: # single char exclude\n token = \"[^{:c}]\".format(b1-1)\n else: # range exclude\n token = \"[^{:c}-{:c}]\".format(b4+1, b1-1)\n else:\n token = \"[TODO]\"\n else:\n token = \"[\"\n for (b1, b2) in b_array:\n token += \"{:c}-{:c}\".format(b1, b2)\n token += \"]\"\n rss.token = token\n b = rss.get_next_byte()\n rss.update_state(b)\n elif rss.state == rss.STATE_SPLIT_BYTE_READ:\n logger.debug(\"state is STATE_SPLIT_BYTE_READ\")\n substr_len = rss.get_last_byte() - 0x7f\n substr = rss.get_substring(substr_len)\n subtokens = self.parse_byte_string(substr, global_vars)\n rss.end_with_subtokens(subtokens)\n b = rss.get_next_byte()\n rss.update_state(b)\n elif rss.state == rss.STATE_SINGLE_BYTE_READ:\n logger.debug(\"state is STATE_SINGLE_BYTE_READ\")\n rss.read_token(1)\n b = rss.get_next_byte()\n rss.update_state(b)\n elif rss.state == rss.STATE_RESET_STRING:\n logger.debug(\"state is STATE_RESET_STRING\")\n rss.end_current_token()\n rss.reset_base_full()\n b = rss.get_next_byte()\n rss.update_state(b)\n elif rss.state == rss.STATE_PLUS_READ:\n logger.debug(\"state is STATE_PLUS_READ\")\n if rss.state_stack[len(rss.state_stack)-1] == rss.STATE_CONCAT_BYTE_READ:\n rss.token = \"+\"\n rss.update_base()\n else:\n logger.warn(\"previous state is not concat\")\n rss.read_token(1)\n b = rss.get_next_byte()\n rss.update_state(b)\n else:\n logger.warn(\"unknown state ({:d})\".format(rss.state))\n break\n\n if rss.is_end():\n break\n\n # String must end in a STATE_END_BYTE_READ byte.\n if rss.state == rss.STATE_END_BYTE_READ:\n logger.debug(\"state is STATE_END_BYTE_READ\")\n rss.end_current_token()\n elif rss.state == rss.STATE_UNKNOWN or rss.state == rss.STATE_CONCAT_BYTE_READ:\n pass\n elif rss.state_stack[len(rss.state_stack)-1] == rss.STATE_END_BYTE_READ:\n pass\n else:\n logger.warn(\"last state is not STATE_END_BYTE_READ ({:d})\".format(rss.state))\n logger.warn(\"previous state ({:d})\".format(rss.state_stack[len(rss.state_stack)-1]))\n\n logger.info(\"initial string: \" + \" \".join(\"0x{:02x}\".format(ord(c)) for c in s))\n logger.info(\"output_strings (num: {:d}): {:s}\".format(len(rss.output_strings), \",\".join('\"{:s}\"'.format(s) for s in rss.output_strings)))\n return rss.output_strings\n\n\n def __init__(self):\n self.rss_stack = []\n\n\ndef main():\n s = sys.stdin.read()\n ss = SandboxString()\n my_global_vars = [\"FRONT_USER_HOME\", \"HOME\", \"PROCESS_TEMP_DIR\"]\n l = ss.parse_byte_string(s[4:], my_global_vars)\n print(list(set(l)))\n\n\nif __name__ == \"__main__\":\n sys.exit(main())\n" }, { "path": "reverse-sandbox/sandbox_filter.py", "content": "#!/usr/bin/env python\n\nimport struct\nimport re\nimport logging\nimport logging.config\nimport reverse_sandbox\nimport reverse_string\n\nfrom filters import Filters\n\n\nlogging.config.fileConfig(\"logger.config\")\nlogger = logging.getLogger(__name__)\n\nios_major_version = 4\nkeep_builtin_filters = False\nglobal_vars = []\nbase_addr = 0\n\ndef get_filter_arg_string_by_offset(f, offset):\n \"\"\"Extract string (literal) from given offset.\"\"\"\n f.seek(base_addr + offset * 8)\n if ios_major_version >= 13:\n len = struct.unpack(\"= 10:\n f.seek(offset * 8)\n s = f.read(4+len)\n logger.info(\"binary string is \" + s.encode(\"hex\"))\n ss = reverse_string.SandboxString()\n myss = ss.parse_byte_string(s[4:], global_vars)\n actual_string = \"\"\n for sss in myss:\n actual_string = actual_string + sss + \" \"\n actual_string = actual_string[:-1]\n logger.info(\"actual string is \" + actual_string)\n return myss\n type = struct.unpack(\"= 13:\n len = struct.unpack(\"= 10:\n f.seek(base_addr + offset * 8)\n s = f.read(4+len)\n logger.info(\"binary string is \" + s.encode(\"hex\"))\n ss = reverse_string.SandboxString()\n myss = ss.parse_byte_string(s[4:], global_vars)\n append = \"literal\"\n actual_string = \"\"\n for sss in myss:\n actual_string = actual_string + sss + \" \"\n actual_string = actual_string[:-1]\n logger.info(\"actual string is \" + actual_string)\n return (append, myss)\n type = struct.unpack(\"= 13:\n len = struct.unpack(\"> 8)\n number = arg & 0xff\n return '(_IO \"%s\" %d)' % (letter, number)\n\n\ndef get_filter_arg_vnode_type(f, arg):\n \"\"\"Convert integer to file (vnode) type string.\"\"\"\n arg_types = {\n 0x01: \"REGULAR-FILE\",\n 0x02: \"DIRECTORY\",\n 0x03: \"BLOCK-DEVICE\",\n 0x04: \"CHARACTER-DEVICE\",\n 0x05: \"SYMLINK\",\n 0x06: \"SOCKET\",\n 0x07: \"FIFO\",\n 0xffff: \"TTY\"\n }\n if arg in arg_types.keys():\n return '%s' % (arg_types[arg])\n else:\n return '%d' % arg\n\n\ndef get_filter_arg_owner(f, arg):\n \"\"\"Convert integer to process owner string.\"\"\"\n arg_types = {\n 0x01: \"self\",\n 0x02: \"pgrp\",\n 0x03: \"others\",\n 0x04: \"children\",\n 0x05: \"same-sandbox\"\n }\n if arg in arg_types.keys():\n return '%s' % (arg_types[arg])\n else:\n return '%d' % arg\n\n\ndef get_filter_arg_socket_domain(f, arg):\n \"\"\"Convert integer to socket domain string.\"\"\"\n arg_types = {\n 0: \"AF_UNSPEC\",\n 1: \"AF_UNIX\",\n 2: \"AF_INET\",\n 3: \"AF_IMPLINK\",\n 4: \"AF_PUP\",\n 5: \"AF_CHAOS\",\n 6: \"AF_NS\",\n 7: \"AF_ISO\",\n 8: \"AF_ECMA\",\n 9: \"AF_DATAKIT\",\n 10: \"AF_CCITT\",\n 11: \"AF_SNA\",\n 12: \"AF_DECnet\",\n 13: \"AF_DLI\",\n 14: \"AF_LAT\",\n 15: \"AF_HYLINK\",\n 16: \"AF_APPLETALK\",\n 17: \"AF_ROUTE\",\n 18: \"AF_LINK\",\n 19: \"pseudo_AF_XTP\",\n 20: \"AF_COIP\",\n 21: \"AF_CNT\",\n 22: \"pseudo_AF_RTIP\",\n 23: \"AF_IPX\",\n 24: \"AF_SIP\",\n 25: \"pseudo_AF_PIP\",\n 27: \"AF_NDRV\",\n 28: \"AF_ISDN\",\n 29: \"pseudo_AF_KEY\",\n 30: \"AF_INET6\",\n 31: \"AF_NATM\",\n 32: \"AF_SYSTEM\",\n 33: \"AF_NETBIOS\",\n 34: \"AF_PPP\",\n 35: \"pseudo_AF_HDRCMPLT\",\n 36: \"AF_RESERVED_36\",\n 37: \"AF_IEEE80211\",\n 38: \"AF_UTUN\",\n 40: \"AF_MAX\"\n }\n if arg in arg_types.keys():\n return '%s' % (arg_types[arg])\n else:\n return '%d' % arg\n\n\ndef get_filter_arg_socket_type(f, arg):\n \"\"\"Convert integer to socket type string.\"\"\"\n arg_types = {\n 0x01: \"SOCK_STREAM\",\n 0x02: \"SOCK_DGRAM\",\n 0x03: \"SOCK_RAW\",\n 0x04: \"SOCK_RDM\",\n 0x05: \"SOCK_SEQPACKET\"\n }\n if arg in arg_types.keys():\n return '\"%s\"' % (arg_types[arg])\n else:\n return '%d' % arg\n\n\ndef get_none(f, arg):\n \"\"\"Dumb callback function\"\"\"\n return None\n\n\ndef get_filter_arg_privilege_id(f, arg):\n \"\"\"Convert integer to privilege id string.\"\"\"\n arg_types = {\n 1000: \"PRIV_ADJTIME\",\n 1001: \"PRIV_PROC_UUID_POLICY\",\n 1002: \"PRIV_GLOBAL_PROC_INFO\",\n 1003: \"PRIV_SYSTEM_OVERRIDE\",\n 1004: \"PRIV_HW_DEBUG_DATA\",\n 1005: \"PRIV_SELECTIVE_FORCED_IDLE\",\n 1006: \"PRIV_PROC_TRACE_INSPECT\",\n 1008: \"PRIV_KERNEL_WORK_INTERNAL\",\n 6000: \"PRIV_VM_PRESSURE\",\n 6001: \"PRIV_VM_JETSAM\",\n 6002: \"PRIV_VM_FOOTPRINT_LIMIT\",\n 10000: \"PRIV_NET_PRIVILEGED_TRAFFIC_CLASS\",\n 10001: \"PRIV_NET_PRIVILEGED_SOCKET_DELEGATE\",\n 10002: \"PRIV_NET_INTERFACE_CONTROL\",\n 10003: \"PRIV_NET_PRIVILEGED_NETWORK_STATISTICS\",\n 10004: \"PRIV_NET_PRIVILEGED_NECP_POLICIES\",\n 10005: \"PRIV_NET_RESTRICTED_AWDL\",\n 10006: \"PRIV_NET_PRIVILEGED_NECP_MATCH\",\n 11000: \"PRIV_NETINET_RESERVEDPORT\",\n 14000: \"PRIV_VFS_OPEN_BY_ID\",\n }\n if arg in arg_types.keys():\n return '\"%s\"' % (arg_types[arg])\n else:\n return '%d' % arg\n\n\ndef get_filter_arg_process_attribute(f, arg):\n \"\"\"Convert integer to process attribute string.\"\"\"\n arg_types = {\n 0: 'is-plugin',\n 1: 'is-installer',\n 2: 'is-restricted',\n 3: 'is-initproc',\n }\n if arg in arg_types.keys():\n return '%s' % (arg_types[arg])\n else:\n return '%d' % arg\n\n\ndef get_filter_arg_csr(f, arg):\n \"\"\"Convert integer to csr string.\"\"\"\n arg_types = {\n 1: 'CSR_ALLOW_UNTRUSTED_KEXTS',\n 2: 'CSR_ALLOW_UNRESTRICTED_FS',\n 4: 'CSR_ALLOW_TASK_FOR_PID',\n 8: 'CSR_ALLOW_KERNEL_DEBUGGER',\n 16: 'CSR_ALLOW_APPLE_INTERNAL',\n 32: 'CSR_ALLOW_UNRESTRICTED_DTRACE',\n 64: 'CSR_ALLOW_UNRESTRICTED_NVRAM',\n 128: 'CSR_ALLOW_DEVICE_CONFIGURATION',\n }\n if arg in arg_types.keys():\n return '\"%s\"' % (arg_types[arg])\n else:\n return '%d' % arg\n\n\ndef get_filter_arg_host_port(f, arg):\n \"\"\"Convert integer to host special port string.\"\"\"\n arg_types = {\n 8: 'HOST_DYNAMIC_PAGER_PORT',\n 9: 'HOST_AUDIT_CONTROL_PORT',\n 10: 'HOST_USER_NOTIFICATION_PORT',\n 11: 'HOST_AUTOMOUNTD_PORT',\n 12: 'HOST_LOCKD_PORT',\n 13: 'unknown: 13',\n 14: 'HOST_SEATBELT_PORT',\n 15: 'HOST_KEXTD_PORT',\n 16: 'HOST_CHUD_PORT',\n 17: 'HOST_UNFREED_PORT',\n 18: 'HOST_AMFID_PORT',\n 19: 'HOST_GSSD_PORT',\n 20: 'HOST_TELEMETRY_PORT',\n 21: 'HOST_ATM_NOTIFICATION_PORT',\n 22: 'HOST_COALITION_PORT',\n 23: 'HOST_SYSDIAGNOSE_PORT',\n 24: 'HOST_XPC_EXCEPTION_PORT',\n 25: 'HOST_CONTAINERD_PORT',\n }\n if arg in arg_types.keys():\n return '\"%s\"' % (arg_types[arg])\n else:\n return '%d' % arg\n\n\n\"\"\"An array (dictionary) of filter converting items\n\nA filter is identied by a filter id and a filter argument. They are\nboth stored in binary format (numbers) inside the binary sandbox\nprofile file.\n\nEach item in the dictionary is identied by the filter id (used in\nhexadecimal). The value of each item is the string form of the filter id\nand the callback function used to convert the binary form the filter\nargument to a string form.\n\nWhile there is a one-to-one mapping between the binary form and the\nstring form of the filter id, that is not the case for the filter\nargument. To convert the binary form of the filter argument to its\nstring form we use one of the callback functions above; almost all\ncallback function names start with get_filter_arg_.\n\"\"\"\n\ndef convert_filter_callback(f, ios_major_version_arg, keep_builtin_filters_arg,\n global_vars_arg, re_list, filter_id, filter_arg, base_addr_arg):\n \"\"\"Convert filter from binary form to string.\n\n Binary form consists of filter id and filter argument:\n * filter id is the index inside the filters array above\n * filter argument is an actual parameter (such as a port number),\n a file offset or a regular expression index\n\n The string form consists of the name of the filter (as extracted\n from the filters array above) and a string representation of the\n filter argument. The string form of the filter argument if obtained\n from the binary form through the use of the callback function (as\n extracted frm the filters array above).\n\n Function arguments are:\n f: the binary sandbox profile file\n regex_list: list of regular expressions\n filter_id: the binary form of the filter id\n filter_arg: the binary form of the filter argument\n \"\"\"\n\n global regex_list\n global ios_major_version\n global keep_builtin_filters\n global global_vars\n global base_addr\n keep_builtin_filters = keep_builtin_filters_arg\n ios_major_version = ios_major_version_arg\n global_vars = global_vars_arg\n regex_list = re_list\n base_addr = base_addr_arg\n\n if not Filters.exists(ios_major_version, filter_id):\n logger.warn(\"filter_id {} not in keys\".format(filter_id))\n return (None, None)\n filter = Filters.get(ios_major_version, filter_id)\n if not filter[\"arg_process_fn\"]:\n logger.warn(\"no function for filter {}\".format(filter_id))\n return (None, None)\n if filter[\"arg_process_fn\"] == \"get_filter_arg_string_by_offset_with_type\":\n (append, result) = globals()[filter[\"arg_process_fn\"]](f, filter_arg)\n if filter_id == 0x01 and append == \"path\":\n append = \"subpath\"\n if result == None and filter[\"name\"] != \"debug-mode\":\n logger.warn(\"result of calling string offset for filter {} is none\".format(filter_id))\n return (None, None)\n return (filter[\"name\"] + append, result)\n result = globals()[filter[\"arg_process_fn\"]](f, filter_arg)\n if result == None and not ((filter[\"name\"] in [\"debug-mode\", \"syscall-mask\", \"machtrap-mask\", \"kernel-mig-routine-mask\"]) or\n (filter[\"name\"] in [\"extension\", \"mach-extension\"]\n and ios_major_version <= 5)):\n logger.warn(\"result of calling arg_process_fn for filter {} is none\".format(filter_id))\n return (None, None)\n return (filter[\"name\"], result)\n" }, { "path": "reverse-sandbox/sandbox_regex.py", "content": "#!/usr/bin/env python3\n\nimport struct\nimport logging\nimport logging.config\n\nlogging.config.fileConfig(\"logger.config\")\nlogger = logging.getLogger(__name__)\n\nfrom regex_parser_v1 import RegexParser as RegexParserV1\nfrom regex_parser_v2 import RegexParser as RegexParserV2\nfrom regex_parser_v3 import RegexParser as RegexParserV3\n\nclass Node():\n \"\"\"Representation of a node inside a regex non-deterministic automaton\n\n The most important attribute is the node type, which may be any of\n the four macros TYPE_... below.\n \"\"\"\n\n TYPE_JUMP_FORWARD = 1\n TYPE_JUMP_BACKWARD = 2\n TYPE_CHARACTER = 3\n TYPE_END = 4\n FLAG_WHITE = 1\n FLAG_GREY = 2\n FLAG_BLACK = 3\n name = \"\"\n type = None\n value = None\n flag = \"white\"\n\n def __init__(self, name=None, type=None, value=''):\n self.name = name\n self.type = type\n self.value = value\n self.flag = self.FLAG_WHITE\n\n def set_name(self, name):\n self.name = name\n\n def set_type_jump_forward(self):\n self.type = self.TYPE_JUMP_FORWARD\n\n def set_type_jump_backward(self):\n self.type = self.TYPE_JUMP_BACKWARD\n\n def set_type_character(self):\n self.type = self.TYPE_CHARACTER\n\n def set_type_end(self):\n self.type = self.TYPE_END\n\n def is_type_end(self):\n return self.type == self.TYPE_END\n\n def is_type_jump(self):\n return self.type == self.TYPE_JUMP_BACKWARD or self.type == self.TYPE_JUMP_FORWARD\n\n def is_type_jump_backward(self):\n return self.type == self.TYPE_JUMP_BACKWARD\n\n def is_type_jump_forward(self):\n return self.type == self.TYPE_JUMP_FORWARD\n\n def is_type_character(self):\n return self.type == self.TYPE_CHARACTER\n\n def set_value(self, value):\n self.value = value\n\n def set_flag_white(self):\n self.flag = self.FLAG_WHITE\n\n def set_flag_grey(self):\n self.flag = self.FLAG_GREY\n\n def set_flag_black(self):\n self.flag = self.FLAG_BLACK\n\n def __str__(self):\n if self.type == self.TYPE_JUMP_BACKWARD:\n return \"(%s: jump backward)\" % (self.name)\n elif self.type == self.TYPE_JUMP_FORWARD:\n return \"(%s: jump forward)\" % (self.name)\n elif self.type == self.TYPE_END:\n return \"(%s: end)\" % (self.name)\n else:\n return \"(%s: %s)\" % (self.name, self.value)\n\n\nclass Graph():\n \"\"\"Representation of a regex NDA (Non-Deterministic Automaton)\n\n Use this class to convert a regex list of items into its canonical\n regular expression string.\n \"\"\"\n\n graph_dict = {}\n canon_graph_dict = {}\n node_list = []\n start_node = None\n end_states = []\n start_state = 0\n regex = []\n unified_regex = \"\"\n\n def __init__(self):\n self.graph_dict = {}\n\n def add_node(self, node, next_list=None):\n self.graph_dict[node] = next_list\n\n def has_node(self, node):\n return node in graph_dict.keys()\n\n def update_node(self, node, next_list):\n self.graph_dict[node] = next_list\n\n def add_new_next_to_node(self, node, next):\n self.graph_dict[node].append(next)\n\n def __str__(self):\n # Get maximum node number.\n max = -1\n for node in self.graph_dict.keys():\n if max < int(node.name):\n max = int(node.name)\n\n # Create graph list for ordered listing of nodes.\n graph_list = [None] * (max+1)\n for node in self.graph_dict.keys():\n actual_string = str(node) + \":\"\n for next_node in self.graph_dict[node]:\n actual_string += \" \" + str(next_node)\n graph_list[int(node.name)] = actual_string\n\n # Store node graph in ret_string.\n ret_string = \"\\n-- Node graph --\\n\"\n for s in graph_list:\n if s:\n ret_string += s + \"\\n\"\n\n # Store canonical graph in ret_string.\n ret_string += \"\\n-- Canonical graph --\\n\"\n for state in self.canon_graph_dict.keys():\n if state == self.start_state:\n ret_string += \"> \"\n elif state in self.end_states:\n ret_string += \"# \"\n else:\n ret_string += \" \"\n ret_string += \"%d: %s\\n\" % (state, self.canon_graph_dict[state])\n ret_string += \"\\n\"\n return ret_string\n\n def get_node_for_idx(self, idx):\n if idx >= len(self.node_list) or idx < 0:\n return None\n return self.node_list[idx]\n\n def get_re_index_for_pos(self, regex_list, pos):\n for idx, item in enumerate(regex_list):\n if item[\"pos\"] == pos:\n return idx\n for idx, item in enumerate(regex_list):\n if item[\"pos\"]-1 == pos:\n return idx\n return -1\n\n def get_next_idx_for_regex_item(self, regex_list, regex_item):\n result = self.get_re_index_for_pos(regex_list, regex_item[\"nextpos\"])\n assert(result >= 0)\n return result\n\n def fill_from_regex_list(self, regex_list):\n # First create list of nodes. No pointers/links at this point.\n # Create a node for each item.\n self.node_list = []\n for idx, item in enumerate(regex_list):\n node = Node(name=\"%s\" % (idx))\n if item[\"type\"] == \"jump_backward\":\n node.set_type_jump_backward()\n elif item[\"type\"] == \"jump_forward\":\n node.set_type_jump_forward()\n elif item[\"type\"] == \"end\":\n node.set_type_end()\n else:\n node.set_type_character()\n node.set_value(item[\"value\"])\n\n if 'start_node' in item and item['start_node'] == True:\n assert(self.start_node == None)\n self.start_node = node\n self.node_list.append(node)\n\n self.graph_dict = {}\n for idx, node in enumerate(self.node_list):\n # If node is end node ignore.\n if node.is_type_end():\n self.graph_dict[node] = []\n elif node.is_type_character():\n next = self.get_node_for_idx(\n self.get_next_idx_for_regex_item(regex_list, regex_list[idx]))\n if next:\n self.graph_dict[node] = [ next ]\n else:\n self.graph_dict[node] = []\n # Node is jump node.\n elif node.is_type_jump_backward():\n next_idx = self.get_re_index_for_pos(regex_list, regex_list[idx][\"value\"])\n next = self.get_node_for_idx(next_idx)\n if next:\n self.graph_dict[node] = [next]\n else:\n self.graph_dict[node] = []\n elif node.is_type_jump_forward():\n next_idx1 = self.get_next_idx_for_regex_item(\n regex_list, regex_list[idx])\n next_idx2 = self.get_re_index_for_pos(regex_list, regex_list[idx][\"value\"])\n next1 = self.get_node_for_idx(next_idx1)\n next2 = self.get_node_for_idx(next_idx2)\n self.graph_dict[node] = []\n if next1:\n self.graph_dict[node].append(next1)\n if next2:\n self.graph_dict[node].append(next2)\n\n def get_character_nodes(self, node):\n node_list = []\n for next in self.graph_dict[node]:\n if next.is_type_character() or next.is_type_end():\n node_list.append(next)\n else:\n node_list = list(set(node_list).union(self.get_character_nodes(next)))\n return node_list\n\n def find_node_type_jump(self, current_node, node, backup_dict):\n if not current_node.is_type_jump():\n return False\n if current_node == node:\n return True\n if not self.graph_dict[current_node]:\n return False\n for next_node in backup_dict[current_node]:\n if self.find_node_type_jump(next_node, node, backup_dict):\n return True\n return False\n\n def reduce(self):\n for node in self.graph_dict.keys():\n if node.is_type_character():\n self.graph_dict[node] = self.get_character_nodes(node)\n old_dict = dict(self.graph_dict)\n backup_dict = dict(self.graph_dict)\n for node in old_dict.keys():\n if node.is_type_jump():\n if self.find_node_type_jump(self.start_node,\n node, backup_dict):\n continue\n del self.graph_dict[node]\n\n def get_edges(self, node):\n edges = []\n is_end_state = False\n for next in self.graph_dict[node]:\n if next.is_type_end():\n is_end_state = True\n else:\n edges.append((next.value, int(next.name)))\n return is_end_state, edges\n\n def convert_to_canonical(self):\n self.end_states = []\n for node in self.graph_dict.keys():\n if node.is_type_end():\n continue\n state_idx = int(node.name)\n is_end_state, self.canon_graph_dict[state_idx] = self.get_edges(node)\n if is_end_state == True:\n self.end_states.append(state_idx)\n for node in self.graph_dict.keys():\n if node.name == \"0\":\n self.start_state = -1\n self.canon_graph_dict[-1] = [ (node.value, 0) ]\n logger.debug(self.canon_graph_dict)\n logger.debug(\"end_states:\")\n logger.debug(self.end_states)\n logger.debug(\"start_state:\")\n logger.debug(self.start_state)\n\n def need_use_plus(self, initial_string, string_to_add):\n if not string_to_add.endswith(\"*\"):\n return False\n\n if string_to_add.startswith(\"(\") and string_to_add[-2:-1] == \")\":\n actual_part = string_to_add[1:-2]\n else:\n actual_part = string_to_add[:-1]\n if initial_string.endswith(actual_part):\n return True\n if initial_string.endswith(string_to_add):\n return True\n\n return False\n\n def unify_two_strings(self, s1, s2):\n # Find largest common starting substring.\n lcss = \"\"\n for i in range(1, len(s1)+1):\n if s2.find(s1[:i], 0, i) != -1:\n lcss = s1[:i]\n if lcss:\n s1 = s1[len(lcss):]\n s2 = s2[len(lcss):]\n # Find largest common ending substring.\n lces = \"\"\n for i in range(1, len(s1)+1):\n if s2.find(s1[-i:], len(s2)-i, len(s2)) != -1:\n lces = s1[-i:]\n if lces:\n s1 = s1[:len(s1)-len(lces)]\n s2 = s2[:len(s2)-len(lces)]\n\n if not s1 and not s2:\n return lcss + lces\n\n if s1 and s2:\n return lcss + \"(\" + s1 + \"|\" + s2 + \")\" + lces\n\n # Make s1 the empty string.\n if not s2:\n aux = s1\n s1 = s2\n s2 = aux\n\n if s2[-1] == '+':\n s2 = s2[:-1] + '*'\n else:\n if len(s2) > 1:\n s2 = \"(\" + s2 + \")?\"\n else:\n s2 = s2 + '?'\n\n return lcss + s2 + lces\n\n def unify_strings(self, string_list):\n unified = \"\"\n if not string_list:\n return None\n if len(string_list) == 1:\n return string_list[0]\n # We now know we have multiple strings. Merge two at a time.\n current = string_list[0]\n for s in string_list[1:]:\n current = self.unify_two_strings(current, s)\n return current\n\n def remove_state(self, state_to_remove):\n itself_string = \"\"\n for (next_string, next_state) in self.canon_graph_dict[state_to_remove]:\n if next_state == state_to_remove:\n if len(next_string) > 1:\n itself_string = \"(%s)*\" % next_string\n else:\n itself_string = \"%s*\" % next_string\n\n # Create list of to_strings indexed by to_states.\n to_strings = {}\n for to_state in self.canon_graph_dict.keys():\n to_strings[to_state] = []\n if to_state == state_to_remove:\n continue\n for (iter_to_string, iter_to_state) in self.canon_graph_dict[state_to_remove]:\n if iter_to_state == to_state:\n to_strings[to_state].append(iter_to_string)\n\n # Unify multiple strings leading to the same to_state.\n unified_to_string = {}\n for to_state in to_strings.keys():\n unified_to_string[to_state] = self.unify_strings(to_strings[to_state])\n\n # Go through all graph edges.\n for from_state in self.canon_graph_dict.keys():\n # Pass current state to remove.\n if from_state == state_to_remove:\n continue\n items_to_remove_list = []\n for (next_string, next_state) in self.canon_graph_dict[from_state]:\n # Only if edge points to state_to_remove.\n if next_state != state_to_remove:\n continue\n # Plan edge to remove. Create new edge bypassing state_to_remove.\n items_to_remove_list.append((next_string, next_state))\n for to_state in self.canon_graph_dict.keys():\n if len(to_strings[to_state]) == 0:\n continue\n to_string = unified_to_string[to_state]\n #for (to_string, to_state) in self.canon_graph_dict[state_to_remove]:\n # # If state points to itself, do not add edge.\n # if to_state == state_to_remove:\n # continue\n # Add new edge, consider if state points to itself.\n if self.need_use_plus(next_string, itself_string):\n self.canon_graph_dict[from_state].append((next_string + \"+\" + to_string, to_state))\n continue\n self.canon_graph_dict[from_state].append((next_string + itself_string + to_string, to_state))\n for (next_string, next_state) in items_to_remove_list:\n self.canon_graph_dict[from_state].remove((next_string, next_state))\n\n del self.canon_graph_dict[state_to_remove]\n\n def simplify(self):\n tmp_dict = dict(self.canon_graph_dict)\n for state in tmp_dict.keys():\n if state != self.start_state and state not in self.end_states:\n self.remove_state(state)\n\n def combine_start_end_nodes(self):\n working_strings = self.canon_graph_dict[self.start_state]\n final_strings = []\n string_added = True\n while string_added == True:\n string_added = False\n initial_strings = working_strings\n working_strings = []\n for (start_string, start_next_state) in initial_strings:\n if not start_next_state in self.end_states:\n continue\n if self.canon_graph_dict[start_next_state]:\n for (next_string, next_state) in self.canon_graph_dict[start_next_state]:\n if next_state == start_next_state:\n next_string = \"(%s)*\" % next_string\n if self.need_use_plus(start_string, next_string):\n final_strings.append((start_string + \"+\", None))\n else:\n final_strings.append((start_string + next_string, None))\n else:\n final_strings.append((start_string + next_string, None))\n working_strings.append((start_string + next_string, next_state))\n else:\n final_strings.append((start_string, None))\n string_added = True\n self.regex = [x[0] for x in final_strings]\n self.unified_regex = self.unify_strings(self.regex)\n\n\n\ndef create_regex_list(re):\n \"\"\"Convert binary regex to list of items.\n\n Each item stores character position inside the binary regex (useful\n for jumps), character type and the value (either character or\n jump offset).\n \"\"\"\n\n regex_list = []\n\n version = struct.unpack('>I', ''.join([chr(x) for x in re[:4]]))[0]\n logger.debug(\"re.version: 0x%x\", version)\n\n i = 4\n if version == 1:\n RegexParserV1.parse(re, i, regex_list)\n elif version == 2:\n RegexParserV2.parse(re, i, regex_list)\n elif version == 3:\n RegexParserV3.parse(re, i, regex_list)\n else:\n logger.critical(\"No parser available for regex version {:x}\".format(version))\n\n\n\n return regex_list\n\n\ndef parse_regex(re):\n \"\"\"Parse binary form for regular expression into canonical string.\n\n The input binary format is the one stored in the sandbox profile\n file. The out format is a canonical regular expression string using\n standard ASCII characters and metacharacters such as ^, $, +, *, etc.\n \"\"\"\n\n regex_list = create_regex_list(re)\n g = Graph()\n g.fill_from_regex_list(regex_list)\n g.reduce()\n g.convert_to_canonical()\n g.simplify()\n g.combine_start_end_nodes()\n logger.debug(g)\n return g.regex\n\n\nimport sys\nimport struct\n\n\ndef main():\n \"\"\"Parse regular expressions in binary file.\"\"\"\n\n if len(sys.argv) != 2:\n print >> sys.stderr, \"Usage: %s \" % (sys.argv[0])\n sys.exit(1)\n\n with open(sys.argv[1]) as f:\n re_count = struct.unpack(\"