[ { "path": ".github/workflows/codeql.yml", "content": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# You may wish to alter this file to override the set of languages analyzed,\n# or to provide custom queries or build logic.\n#\n# ******** NOTE ********\n# We have attempted to detect the languages in your repository. Please check\n# the `language` matrix defined below to confirm you have the correct set of\n# supported CodeQL languages.\n#\nname: \"CodeQL Advanced\"\n\non:\n push:\n branches: [ \"main\" ]\n pull_request:\n branches: [ \"main\" ]\n schedule:\n - cron: '24 3 * * 5'\n\njobs:\n analyze:\n name: Analyze (${{ matrix.language }})\n # Runner size impacts CodeQL analysis time. To learn more, please see:\n # - https://gh.io/recommended-hardware-resources-for-running-codeql\n # - https://gh.io/supported-runners-and-hardware-resources\n # - https://gh.io/using-larger-runners (GitHub.com only)\n # Consider using larger runners or machines with greater resources for possible analysis time improvements.\n runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}\n permissions:\n # required for all workflows\n security-events: write\n\n # required to fetch internal or private CodeQL packs\n packages: read\n\n # only required for workflows in private repositories\n actions: read\n contents: read\n\n strategy:\n fail-fast: false\n matrix:\n include:\n - language: python\n build-mode: none\n # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'\n # Use `c-cpp` to analyze code written in C, C++ or both\n # Use 'java-kotlin' to analyze code written in Java, Kotlin or both\n # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both\n # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,\n # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.\n # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how\n # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n\n # Initializes the CodeQL tools for scanning.\n - name: Initialize CodeQL\n uses: github/codeql-action/init@v3\n with:\n languages: ${{ matrix.language }}\n build-mode: ${{ matrix.build-mode }}\n # If you wish to specify custom queries, you can do so here or in a config file.\n # By default, queries listed here will override any specified in a config file.\n # Prefix the list here with \"+\" to use these queries and those in the config file.\n\n # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs\n # queries: security-extended,security-and-quality\n\n # If the analyze step fails for one of the languages you are analyzing with\n # \"We were unable to automatically build your code\", modify the matrix above\n # to set the build mode to \"manual\" for that language. Then modify this step\n # to build your code.\n # ℹ️ Command-line programs to run using the OS shell.\n # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun\n - if: matrix.build-mode == 'manual'\n shell: bash\n run: |\n echo 'If you are using a \"manual\" build mode for one or more of the' \\\n 'languages you are analyzing, replace this with the commands to build' \\\n 'your code, for example:'\n echo ' make bootstrap'\n echo ' make release'\n exit 1\n\n - name: Perform CodeQL Analysis\n uses: github/codeql-action/analyze@v3\n with:\n category: \"/language:${{matrix.language}}\"\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Microsoft Open Source Code of Conduct\n\nThis project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).\n\nResources:\n\n- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)\n- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)\n- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns\n" }, { "path": "DomainSpecific/.gitignore", "content": "__pycache__/\ndependency/models/\nenv_ready\nworkspace\n" }, { "path": "DomainSpecific/configs/cc_math_filter.CC-MAIN-2023-23.json", "content": "{\n \"name\": \"cc_math_extraction\",\n \"description\": \"math extraction from cc parquet file - 202323.\",\n \"date\": \"20240513\",\n \"version\": \"1.0.0\",\n \"author\": \"yanghuan\",\n \"backend\": \"Native\",\n \n \"input\":\n {\n \"pq_name_list_file_path\":\n {\n \"type\": \"Mem_Str\",\n \"value\": \"{workspace_dir}/cc_pqs/raw/CC-MAIN-2023-23/pqs.CC-MAIN-2023-23.txt\"\n },\n \"filtered_pq_name_list_file_path\":\n {\n \"type\": \"Mem_Str\",\n \"value\": \"{workspace_dir}/cc_pqs/math/CC-MAIN-2023-23/paths.{worker_id}.txt\"\n }\n },\n \n \"output\":\n {\n \"filtered_pq_name_list_file_path\":\n {\n \"type\": \"Mem_Str\"\n }\n },\n \n \"layer\":\n {\n \"layer01\":\n {\n \"type\": \"From_Line_File\",\n \"joint\": \"Default\",\n \"input\": [\"pq_name_list_file_path\"],\n \"output\": [\"pq_names\"]\n },\n \"layer01_par\":\n {\n \"type\": \"Data_Partition\",\n \"joint\": \"Default\",\n \"input\": [\"pq_names\"],\n \"output\": [\"pq_names\"]\n },\n \"layer01_sam\":\n {\n \"type\": \"Data_Sample\",\n \"joint\": \"Default\",\n \"param\":\n {\n \"N\": -1\n },\n \"input\": [\"pq_names\"],\n \"output\": [\"pq_names\"]\n },\n \"layer02\":\n {\n \"type\": \"Math_Filter\",\n \"joint\": \"FlatMap\",\n \"param\":\n {\n \"INPUT_FOLDER\": \"{workspace_dir}/cc_pqs/raw/CC-MAIN-2023-23/\",\n \"OUTPUT_FOLDER\": \"{workspace_dir}/cc_pqs/math/CC-MAIN-2023-23/\"\n },\n \"input\": [\"pq_names\"],\n \"output\": [\"filtered_pq_names\"]\n },\n \"layer03\":\n {\n \"type\": \"To_Line_File\",\n \"joint\": \"Default\",\n \"input\": [\"filtered_pq_names\", \"filtered_pq_name_list_file_path\"],\n \"output\": [\"filtered_pq_name_list_file_path\"]\n }\n }\n}\n" }, { "path": "DomainSpecific/configs/cc_openquestion_filter.CC-MAIN-2023-23.json", "content": "{\n \"name\": \"cc_openquestion_extraction\",\n \"description\": \"open question extraction from cc parquet file - 202323.\",\n \"date\": \"20240527\",\n \"version\": \"1.0.0\",\n \"author\": \"yanghuan\",\n \"backend\": \"Native\",\n \n \"input\":\n {\n \"pq_name_list_file_path\":\n {\n \"type\": \"Mem_Str\",\n \"value\": \"{workspace_dir}/cc_pqs/raw/CC-MAIN-2023-23/pqs.CC-MAIN-2023-23.txt\"\n },\n \"filtered_pq_name_list_file_path\":\n {\n \"type\": \"Mem_Str\",\n \"value\": \"{workspace_dir}/cc_pqs/openquestion/CC-MAIN-2023-23/paths.{worker_id}.txt\"\n }\n },\n \n \"output\":\n {\n \"filtered_pq_name_list_file_path\":\n {\n \"type\": \"Mem_Str\"\n }\n },\n \n \"layer\":\n {\n \"layer01\":\n {\n \"type\": \"From_Line_File\",\n \"joint\": \"Default\",\n \"input\": [\"pq_name_list_file_path\"],\n \"output\": [\"pq_names\"]\n },\n \"layer01_par\":\n {\n \"type\": \"Data_Partition\",\n \"joint\": \"Default\",\n \"input\": [\"pq_names\"],\n \"output\": [\"pq_names\"]\n },\n \"layer01_sam\":\n {\n \"type\": \"Data_Sample\",\n \"joint\": \"Default\",\n \"param\":\n {\n \"N\": -1\n },\n \"input\": [\"pq_names\"],\n \"output\": [\"pq_names\"]\n },\n \"layer02\":\n {\n \"type\": \"OpenQuestion_Filter\",\n \"joint\": \"FlatMap\",\n \"param\":\n {\n \"INPUT_FOLDER\": \"{workspace_dir}/cc_pqs/raw/CC-MAIN-2023-23/\",\n \"OUTPUT_FOLDER\": \"{workspace_dir}/cc_pqs/openquestion/CC-MAIN-2023-23/\"\n },\n \"input\": [\"pq_names\"],\n \"output\": [\"filtered_pq_names\"]\n },\n \"layer03\":\n {\n \"type\": \"To_Line_File\",\n \"joint\": \"Default\",\n \"input\": [\"filtered_pq_names\", \"filtered_pq_name_list_file_path\"],\n \"output\": [\"filtered_pq_name_list_file_path\"]\n }\n }\n}\n" }, { "path": "DomainSpecific/configs/cc_warc_download.CC-MAIN-2023-23.json", "content": "{\n \"name\": \"cc_warc_download\",\n \"description\": \"download warc files for a specific cc snapshot - CC-MAIN-2023-23.\",\n \"date\": \"20231011\",\n \"version\": \"1.0.0\",\n \"author\": \"yanghuan\",\n \"backend\": \"Native\",\n \n \"input\":\n {\n \"warc_url_list_file_path\":\n {\n \"type\": \"Mem_Str\",\n \"value\": \"{workspace_dir}/urls.CC-MAIN-2023-23.txt\"\n },\n \"success_warc_name_list_file_path\":\n {\n \"type\": \"Mem_Str\",\n \"value\": \"{workspace_dir}/cc_warcs/CC-MAIN-2023-23/paths.{worker_id}.txt\"\n },\n \"fail_warc_url_list_file_path\":\n {\n \"type\": \"Mem_Str\",\n \"value\": \"{workspace_dir}/cc_warcs/CC-MAIN-2023-23/fail_urls.{worker_id}.txt\"\n }\n },\n \n \"output\":\n {\n \"success_warc_name_list_file_path\":\n {\n \"type\": \"Mem_Str\"\n },\n \"fail_warc_url_list_file_path\":\n {\n \"type\": \"Mem_Str\"\n }\n },\n \n \"layer\":\n {\n \"layer01\":\n {\n \"type\": \"From_Line_File\",\n \"joint\": \"Default\",\n \"input\": [\"warc_url_list_file_path\"],\n \"output\": [\"warc_urls\"]\n },\n \"layer01_par\":\n {\n \"type\": \"Data_Partition\",\n \"joint\": \"Default\",\n \"input\": [\"warc_urls\"],\n \"output\": [\"warc_urls\"]\n },\n \"layer01_sam\":\n {\n \"type\": \"Data_Sample\",\n \"joint\": \"Default\",\n \"param\":\n {\n \"N\": 1\n },\n \"input\": [\"warc_urls\"],\n \"output\": [\"warc_urls\"]\n },\n \"layer02\":\n {\n \"type\": \"Download_Warc_File\",\n \"joint\": \"Map\",\n \"param\":\n {\n \"DOWNLOAD_FOLDER\": \"{workspace_dir}/cc_warcs/CC-MAIN-2023-23\",\n \"CONNECTS\": 16,\n \"TRIES\": 3\n },\n \"input\": [\"warc_urls\"],\n \"output\": [\"success_warc_names\", \"fail_warc_urls\"]\n },\n \"layer03\":\n {\n \"type\": \"Data_Filter\",\n \"param\":\n {\n \"FILTERS\": [null]\n },\n \"input\": [\"success_warc_names\"],\n \"output\": [\"success_warc_names\"]\n },\n \"layer04\":\n {\n \"type\": \"To_Line_File\",\n \"joint\": \"Default\",\n \"input\": [\"success_warc_names\", \"success_warc_name_list_file_path\"],\n \"output\": [\"success_warc_name_list_file_path\"]\n },\n \"layer05\":\n {\n \"type\": \"Data_Filter\",\n \"param\":\n {\n \"FILTERS\": [null]\n },\n \"input\": [\"fail_warc_urls\"],\n \"output\": [\"fail_warc_urls\"]\n },\n \"layer06\":\n {\n \"type\": \"To_Line_File\",\n \"joint\": \"Default\",\n \"input\": [\"fail_warc_urls\", \"fail_warc_url_list_file_path\"],\n \"output\": [\"fail_warc_url_list_file_path\"]\n }\n }\n}\n" }, { "path": "DomainSpecific/configs/cc_warc_filter.CC-MAIN-2023-23.json", "content": "{\n \"name\": \"cc_warc_filter\",\n \"description\": \"filter html containing specific tags on warc files - CC-MAIN-2023-23.\",\n \"date\": \"20230825\",\n \"version\": \"1.0.0\",\n \"author\": \"yanghuan\",\n \"backend\": \"Native\",\n \n \"input\":\n {\n \"warc_name_list_file_path\":\n {\n \"type\": \"Mem_Str\",\n \"value\": \"{workspace_dir}/cc_warcs/CC-MAIN-2023-23/paths.txt\"\n },\n \"filtered_warc_name_list_file_path\":\n {\n \"type\": \"Mem_Str\",\n \"value\": \"{workspace_dir}/cc_filtered_warc/CC-MAIN-2023-23/paths.{worker_id}.txt\"\n }\n },\n \n \"output\":\n {\n \"filtered_warc_name_list_file_path\":\n {\n \"type\": \"Mem_Str\"\n }\n },\n \n \"layer\":\n {\n \"layer01\":\n {\n \"type\": \"From_Line_File\",\n \"joint\": \"Default\",\n \"input\": [\"warc_name_list_file_path\"],\n \"output\": [\"warc_names\"]\n },\n \"layer01_par\":\n {\n \"type\": \"Data_Partition\",\n \"joint\": \"Default\",\n \"input\": [\"warc_names\"],\n \"output\": [\"warc_names\"]\n },\n \"layer01_sam\":\n {\n \"type\": \"Data_Sample\",\n \"joint\": \"Default\",\n \"param\":\n {\n \"N\": -1\n },\n \"input\": [\"warc_names\"],\n \"output\": [\"warc_names\"]\n },\n \"layer02\":\n {\n \"type\": \"Warc_Filter\",\n \"joint\": \"FlatMap\",\n \"param\":\n {\n \"INPUT_FOLDER\": \"{workspace_dir}/cc_warcs/CC-MAIN-2023-23\",\n \"OUTPUT_FOLDER\": \"{workspace_dir}/cc_filtered_warc/CC-MAIN-2023-23/\",\n \"TAGS\": [\"= 100)\n\nclass Data:\n \"\"\"\n Data class (Deprecated).\n \"\"\"\n def __init__(self, type=DataType.Mem_Any, value=None):\n self.type = type if isinstance(type, DataType) else DataType[type]\n self.value = value\n\n\nif __name__ == \"__main__\":\n data = Data()\n print(data)\n" }, { "path": "DomainSpecific/core/layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nfrom enum import Enum\nfrom tqdm import tqdm\nfrom core.layers import LayerType, LayerType2Func\n\nclass JointType(Enum):\n Default = 0 # Only process data as whole (frequently used in data IO and control layers).\n Map = 1 # Firstly split data list into data unit, then process data unit to any type, finnaly return the list of processed data unit.\n FlatMap = 2 # Firstly split data list into data unit, then process data unit to list type, then concat the whole processed data lists, finnally return the concated data list.\n\nclass Layer:\n def __init__(self, type, joint=JointType.Default, repetition=1, param=dict(), input_names=list(), output_names=list()):\n self.type = type if isinstance(type, LayerType) else LayerType[type]\n self.func, self.input_types, self.output_types, self.enabled = LayerType2Func[self.type]\n self.joint = joint if isinstance(joint, JointType) else JointType[joint]\n self.repetition = repetition\n self.param = param\n self.input_names = input_names\n self.output_names = output_names\n\n def __call__(self, inputs, worker_id=0, worker_num=1, variables=dict()):\n outputs = list()\n try:\n variables[\"worker_id\"] = worker_id\n variables[\"worker_num\"] = worker_num\n\n if not isinstance(inputs, list):\n raise Exception(f\"The inputs of layer should be list data type.\")\n if len(inputs) != len(self.input_types):\n raise Exception(f\"The number of inputs is not {len(self.input_types)}.\")\n for i, (data, input_type) in enumerate(zip(inputs, self.input_types)):\n # TODO: add the check of input type.\n # check the data type of input.\n #if data.type != DataType[input_type]:\n # raise Exception(f\"The {i}th data, whose type is {data.type.name}, does not match the input type {input_type}\")\n # Condition of empty input.\n if data is None:\n outputs = [None for _ in self.output_types]\n return outputs\n\n # TODO: to address the situation of repetition > 1.\n for i in range(self.repetition):\n if self.joint == JointType.Default:\n values = list(self.func(*inputs, variables, **self.param))\n else:\n n = min([len(data) for data in inputs])\n if n != max([len(data) for data in inputs]):\n raise Exception(f\"Element amount of input datas are not equal.\")\n\n values = [[] for _ in self.output_types]\n for i in tqdm(range(n), desc=f\"Layer: {self.type.name}, worker_id: {worker_id}/{worker_num}\"):\n _values = self.func(*[data[i] for data in inputs], variables, **self.param)\n for value, _value in zip(values, _values):\n if _value is None:\n continue\n if self.joint == JointType.Map:\n value.append(_value)\n elif self.joint == JointType.FlatMap:\n if not isinstance(_value, list):\n raise Exception(f\"The output of layer should be list data type.\")\n value.extend(_value)\n else:\n raise Exception(f\"Using unsupported joint type for {self.type.name} layer.\")\n\n outputs = values\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return outputs\n\n\nif __name__ == \"__main__\":\n inputs = [[\"a\", \"b\", \"c\", \"d\", \"e\"]]\n layer = Layer(LayerType.Data_Sample, param={\"N\": 2})\n outputs = layer(inputs)\n print(layer)\n" }, { "path": "DomainSpecific/core/layers/__init__.py", "content": "from enum import Enum\nfrom ..data import DataType\n\nfrom .template_layer import template_layer\n\n# Control layers\nfrom .control import *\n\n# Network (download/upload) layers\nfrom .network import *\n\n# IO (read/write) layers\nfrom .io import *\n\n# Extract layers\nfrom .extract import *\n\n# Transform layers\nfrom .transform import *\n\nclass LayerType(Enum):\n Template = 0\n\n # Control\n Data_Sample = 1\n Data_Concat = 2\n Data_Order = 3\n Data_Partition = 4\n Data_Filter = 5\n Data_Shuffle = 6\n\n # Network - download/upload\n Upload_File_To_Blob = 101\n Upload_Bytes_To_Blob = 102\n Download_File_From_Blob = 103\n Download_Bytes_From_Blob = 104\n Download_File_From_Internet = 105\n Download_Bytes_From_Internet = 106\n Download_Url_List = 107\n Download_Warc_Indice = 108\n Download_Warc_File = 109\n Download_Urls_From_Website = 110\n Download_Image_From_Jsonl = 111\n Download_StarCoder = 112\n\n # IO - read/write\n To_Binary_File = 201\n To_Line_File = 202\n To_Jsonl_File = 203\n To_Parquet_File = 204\n To_Index_File = 205\n To_Warc_File = 206\n From_Binary_File = 207\n From_Line_File = 208\n From_Jsonl_File = 209\n From_Parquet_File = 210\n From_Index_File = 211\n From_Wet_File = 212\n From_Warc_File = 213\n\n # Extract\n Extract_Article = 301\n Build_Index = 302\n Search_Index = 303\n \n # Transform\n Tokenize_Article = 401\n Ngrams = 402\n Minhash_Tokens = 403\n LSH_Minhash = 404\n Warc_Filter = 405\n Warc_Encode = 406\n Warc_To_Wet = 407\n Wet_Decode = 408\n Text_Embedding = 409\n Sentence_Embedding = 410\n Sentence_Filter = 411\n Code_Generation = 412\n Url_To_Record = 413\n Extract_Link_From_Warc = 414\n Wet_To_Imageinfos = 415\n Warc_To_Screenshot_MD = 416\n MCQ_Filter = 417\n OpenQuestion_Filter = 418\n Convert_PDF = 419\n Extract_HTML = 420\n MD_Filter = 421\n Cascaded_Filter = 422\n Math_Filter = 423\n\n\nLayerType2Func = \\\n{\n LayerType.Template : (template_layer, [DataType.Mem_Any], [DataType.Mem_Any], True),\n\n # Control\n LayerType.Data_Sample : (data_sample_layer, [DataType.Mem_List], [DataType.Mem_List], True),\n LayerType.Data_Concat : (data_concat_layer, [DataType.Mem_List], [DataType.Mem_List], True),\n LayerType.Data_Order : (data_order_layer, [DataType.Mem_List], [DataType.Mem_List], True),\n LayerType.Data_Filter : (data_filter_layer, [DataType.Mem_List], [DataType.Mem_List], True),\n LayerType.Data_Partition : (data_partition_layer, [DataType.Mem_List], [DataType.Mem_List], True),\n LayerType.Data_Shuffle : (data_shuffle_layer, [DataType.Mem_List], [DataType.Mem_List], True),\n\n # Network - download/upload\n LayerType.Upload_File_To_Blob : (upload_file_to_blob_layer, [DataType.Mem_Str, DataType.Mem_Str], [DataType.Mem_Str, DataType.Mem_Str], True),\n LayerType.Upload_Bytes_To_Blob : (upload_bytes_to_blob_layer, [DataType.Mem_Binary, DataType.Mem_Str], [DataType.Mem_Str, DataType.Mem_Str], True),\n LayerType.Download_File_From_Blob : (download_file_from_blob_layer, [DataType.Mem_Str], [DataType.Mem_Str, DataType.Mem_Str], True),\n LayerType.Download_Bytes_From_Blob : (download_bytes_from_blob_layer, [DataType.Mem_Str], [DataType.Mem_Str, DataType.Mem_Binary, DataType.Mem_Str], True),\n LayerType.Download_File_From_Internet : (download_file_from_internet_layer, [DataType.Mem_Str], [DataType.Mem_Str, DataType.Mem_Str], True),\n LayerType.Download_Bytes_From_Internet : (download_bytes_from_internet_layer, [DataType.Mem_Str], [DataType.Mem_Str, DataType.Mem_Binary, DataType.Mem_Str], True),\n LayerType.Download_Url_List : (download_url_list_layer, [DataType.Mem_Str], [DataType.Mem_StrList, DataType.Mem_StrList], True),\n LayerType.Download_Warc_File : (download_warc_file_layer, [DataType.Mem_Str], [DataType.Mem_Str, DataType.Mem_Str], True),\n LayerType.Download_Warc_Indice : (download_warc_indice_layer, [DataType.Mem_Str], [DataType.Mem_StrList, DataType.Mem_StrList], True),\n LayerType.Download_Urls_From_Website : (download_urls_from_website_layer, [DataType.Mem_Str], [DataType.Mem_StrList], True),\n LayerType.Download_StarCoder : (download_starcoder_layer, [DataType.Mem_Str], [DataType.Mem_Int], True),\n\n # IO - read/write\n LayerType.To_Binary_File : (to_binary_file_layer, [DataType.Mem_Binary, DataType.Mem_Str], [DataType.Mem_Str], True),\n LayerType.To_Line_File : (to_line_file_layer, [DataType.Mem_StrList, DataType.Mem_Str], [DataType.Mem_Str], True),\n LayerType.To_Jsonl_File : (to_jsonl_file_layer, [DataType.Mem_DictList, DataType.Mem_Str], [DataType.Mem_Str], True),\n LayerType.To_Parquet_File : (to_parquet_file_layer, [DataType.Mem_DictList, DataType.Mem_Str], [DataType.Mem_Str], True),\n LayerType.To_Index_File : (to_index_file_layer, [DataType.Mem_Index, DataType.Mem_Str], [DataType.Mem_Str], True),\n LayerType.From_Binary_File : (from_binary_file_layer, [DataType.Mem_Str], [DataType.Mem_Binary], True),\n LayerType.From_Line_File : (from_line_file_layer, [DataType.Mem_Str], [DataType.Mem_StrList], True),\n LayerType.From_Jsonl_File : (from_jsonl_file_layer, [DataType.Mem_Str], [DataType.Mem_DictList], True),\n LayerType.From_Parquet_File : (from_parquet_file_layer, [DataType.Mem_Str], [DataType.Mem_DictList], True),\n LayerType.From_Index_File : (from_index_file_layer, [DataType.Mem_Str], [DataType.Mem_Index], True),\n LayerType.From_Wet_File : (from_wet_file_layer, [DataType.Mem_Str], [DataType.Mem_DictList], True),\n LayerType.From_Warc_File : (from_warc_file_layer, [DataType.Mem_Str], [DataType.Mem_DictList], True),\n\n # Extract\n LayerType.Extract_Article : (extract_article_layer, [DataType.Mem_Warc], [DataType.Mem_Dict], True),\n LayerType.Build_Index : (build_index_layer, [DataType.Mem_VectorList], [DataType.Mem_Index], True),\n LayerType.Search_Index : (search_index_layer, [DataType.Mem_Index, DataType.Mem_VectorList], [DataType.Mem_VectorList, DataType.Mem_VectorList], True),\n \n # Transform\n LayerType.Tokenize_Article : (tokenize_article_layer, [DataType.Mem_Dict], [DataType.Mem_StrList], True),\n LayerType.Ngrams : (ngrams_layer, [DataType.Mem_StrList], [DataType.Mem_StrList], True),\n LayerType.Minhash_Tokens : (minhash_tokens_layer, [DataType.Mem_StrList], [DataType.Mem_StrList], True),\n LayerType.LSH_Minhash : (lsh_minhash_layer, [DataType.Mem_StrList], [DataType.Mem_StrList], True),\n LayerType.Warc_Filter : (warc_filter_layer, [DataType.Mem_Str], [DataType.Mem_StrList], True),\n LayerType.Warc_Encode : (warc_encode_layer, [DataType.Mem_Str], [DataType.Mem_StrList], True),\n LayerType.Warc_To_Wet : (warc_to_wet_layer, [DataType.Mem_Str], [DataType.Mem_StrList], True),\n LayerType.Wet_Decode : (wet_decode_layer, [DataType.Mem_Str], [DataType.Mem_StrList], True),\n LayerType.Math_Filter : (math_filter_layer, [DataType.Mem_Str], [DataType.Mem_StrList], True),\n LayerType.OpenQuestion_Filter : (openquestion_filter_layer, [DataType.Mem_Str], [DataType.Mem_StrList], True),\n LayerType.MCQ_Filter : (mcq_filter_layer, [DataType.Mem_Str], [DataType.Mem_StrList], True),\n}\n\n\n__all__ = [\n \"LayerType\", \n \"LayerType2Func\", \n \"template_layer\", \n \"data_sample_layer\", \n \"data_concat_layer\", \n \"data_order_layer\", \n \"data_partition_layer\", \n \"data_filter_layer\", \n \"data_shuffle_layer\", \n \"upload_file_to_blob_layer\", \n \"upload_bytes_to_blob_layer\", \n \"download_file_from_blob_layer\", \n \"download_bytes_from_blob_layer\", \n \"download_file_from_internet_layer\", \n \"download_bytes_from_internet_layer\", \n \"download_url_list_layer\", \n \"download_warc_file_layer\", \n \"download_warc_indice_layer\", \n \"download_urls_from_website_layer\", \n \"download_starcoder_layer\", \n \"to_binary_file_layer\", \n \"to_line_file_layer\", \n \"to_jsonl_file_layer\", \n \"to_parquet_file_layer\", \n \"to_index_file_layer\", \n \"from_binary_file_layer\", \n \"from_line_file_layer\", \n \"from_jsonl_file_layer\", \n \"from_parquet_file_layer\", \n \"from_index_file_layer\", \n \"from_wet_file_layer\", \n \"from_warc_file_layer\", \n \"extract_article_layer\", \n \"build_index_layer\", \n \"search_index_layer\", \n \"tokenize_article_layer\", \n \"ngrams_layer\", \n \"minhash_tokens_layer\", \n \"lsh_minhash_layer\", \n \"warc_filter_layer\", \n \"warc_encode_layer\", \n \"warc_to_wet_layer\", \n \"wet_decode_layer\", \n \"math_filter_layer\", \n \"openquestion_filter_layer\", \n \"mcq_filter_layer\", \n]\n" }, { "path": "DomainSpecific/core/layers/control/__init__.py", "content": "# Control\nfrom .data_sample_layer import data_sample_layer\nfrom .data_filter_layer import data_filter_layer\nfrom .data_order_layer import data_order_layer\nfrom .data_partition_layer import data_partition_layer\nfrom .data_shuffle_layer import data_shuffle_layer\nfrom .data_concat_layer import data_concat_layer\n\n__all__ = [\n \"data_sample_layer\", \n \"data_filter_layer\",\n \"data_order_layer\",\n \"data_partition_layer\",\n \"data_shuffle_layer\", \n \"data_concat_layer\", \n]\n" }, { "path": "DomainSpecific/core/layers/control/data_concat_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\n\ndef data_concat_layer(lists, variables=dict()):\n ret = list()\n try:\n for a_list in lists[::-1]:\n if a_list is not None:\n ret[0:0] = a_list\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n lists = [[\"a\"], [\"b\", \"c\"], None, [\"d\", \"e\", \"f\"]]\n lines = data_concat_layer(lists)\n print(lines)\n" }, { "path": "DomainSpecific/core/layers/control/data_filter_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\n\ndef data_filter_layer(lines, variables=dict(), IN=False, FILTERS=(None,)):\n ret = list()\n try:\n ret = list(filter(lambda line: line in FILTERS if IN else line not in FILTERS, lines))\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n lines = [\"a\", None, \"b\"]\n FILTERS = (None,)\n lines = data_filter_layer(lines, FILTERS=FILTERS)\n print(lines)\n" }, { "path": "DomainSpecific/core/layers/control/data_order_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\n\ndef data_order_layer(lines, variables=dict(), REVERSE=False):\n ret = list()\n try:\n ret = sorted(lines, reverse=REVERSE)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n lines = [1, 3, 2]\n lines = data_order_layer(lines)\n print(lines)\n" }, { "path": "DomainSpecific/core/layers/control/data_partition_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\n\ndef data_partition_layer(lines, variables=dict(), WORKER_ID=-1):\n ret = list()\n try:\n worker_id = variables.get(\"worker_id\", 0)\n worker_num = variables.get(\"worker_num\", 1)\n n = len(lines)\n if WORKER_ID == -1:\n ret = [lines[i] for i in range(worker_id, n, worker_num)]\n else:\n ret = lines if WORKER_ID == worker_id else list()\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n lines = [0, 1, 2, 3, 4, 5, 6, 7, 8]\n variables = {\"worker_id\": 0, \"worker_num\": 2}\n lines = data_partition_layer(lines, variables=variables)\n print(lines)\n" }, { "path": "DomainSpecific/core/layers/control/data_sample_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport random\nimport traceback\n\ndef data_sample_layer(lines, variables=dict(), N=-1, SEED=1):\n ret = list()\n try:\n random.seed(SEED)\n N = min(N, len(lines))\n if N >= 0:\n ret = random.sample(lines, N)\n else:\n ret = lines\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n lines = [\"a\", \"b\"]\n N = 1\n lines = data_sample_layer(lines, N=N)\n print(lines)\n" }, { "path": "DomainSpecific/core/layers/control/data_shuffle_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport random\nimport traceback\n\ndef data_shuffle_layer(lines, variables=dict(), SEED=1):\n ret = list()\n try:\n random.seed(SEED)\n random.shuffle(lines)\n ret = lines\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n lines = [\"a\", \"b\"]\n lines = data_shuffle_layer(lines)\n print(lines)\n" }, { "path": "DomainSpecific/core/layers/extract/__init__.py", "content": "# Extract\nfrom .extract_article_layer import extract_article_layer\nfrom .build_index_layer import build_index_layer\nfrom .search_index_layer import search_index_layer\n\n__all__ = [\n \"extract_article_layer\", \n \"build_index_layer\", \n \"search_index_layer\", \n]\n" }, { "path": "DomainSpecific/core/layers/extract/build_index_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport sys\nimport faiss\nimport numpy as np\nimport traceback\n\ndef build_index_layer(base_vectors, variables=dict(), SEED=1, DIM=4096, CLUSTERS=100):\n ret = None\n try:\n np.random.seed(SEED)\n\n quantizer = faiss.IndexFlatL2(DIM)\n index = faiss.IndexIVFFlat(quantizer, DIM, CLUSTERS, faiss.METRIC_L2)\n\n assert not index.is_trained\n base_vectors = np.array(base_vectors)\n index.train(base_vectors)\n assert index.is_trained\n\n index.add(base_vectors)\n ret = index\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == '__main__':\n D = 64\n base_vectors = np.random.random((100000, D)).astype('float32')\n base_vectors[:, 0] += np.arange(100000) / 1000.\n index = build_index_layer(base_vectors, D=D)\n print(index)\n" }, { "path": "DomainSpecific/core/layers/extract/extract_article_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport re\nimport fasttext\nimport traceback\nfrom unittest.mock import patch\nfrom bs4 import BeautifulSoup\nfrom markdownify import MarkdownConverter, chomp\nfrom newspaper import Article\nimport global_var\n\ndef filter_tags_in_html(soup):\n def del_tags(soup):\n del_tags = ['style', 'script', 'img']\n for tag in del_tags:\n tags = soup.find_all(tag)\n for tag in tags:\n tag.decompose()\n\n tags = soup.find_all('table')\n for tag in tags:\n if len(tag.text.strip()) == 0:\n for tag in tags:\n tag.decompose()\n\n def modify_text(soup):\n modify_tags = ['a']\n for i in range(len(modify_tags)):\n for tag in soup.find_all(modify_tags[i]):\n tag_text = tag.text\n new_tag_text = tag_text.replace('\\n', '')\n if len(new_tag_text) != len(tag_text):\n tag.string = new_tag_text\n del_tags(soup)\n modify_text(soup)\n\n return soup\n\ndef lid(soup, model):\n LID_WIN_SIZE=256\n text = ''.join(soup.text.split())\n span_start, span_end = 0, len(text)\n if len(text) > LID_WIN_SIZE:\n mid = len(text) // 2\n mid_win = LID_WIN_SIZE // 2\n span_start = max(0, int(mid - mid_win))\n span_end = min(len(text), int(mid + mid_win))\n\n det_text = text[span_start: span_end]\n res = model.predict(det_text)\n la = res[0][0].replace(\"__label__\", \"\")\n prob = float(res[1][0])\n return la, prob\n\ndef get_main_text_html(soup):\n article = Article(\"padding_url\", fetch_images=False, keep_article_html=True)\n article.download(input_html=str(soup))\n article.parse()\n # assert len(article.text.strip()) >= 128\n main_html = article.article_html\n main_text = article.text\n return main_html, main_text\n\ndef remove_dup_newline(text):\n fields = text.split('\\n')\n for i in range(len(fields)):\n fields[i] = fields[i].strip()\n return re.sub('\\n{2,}', '\\n\\n', '\\n'.join(fields)).strip()\n\nclass User_MarkdownConverter(MarkdownConverter):\n def convert_tr(self, el, text, convert_as_inline):\n cells = el.find_all(['td', 'th'])\n is_headrow = all([cell.name == 'th' for cell in cells])\n overline = ''\n underline = ''\n if is_headrow and not el.previous_sibling:\n # first row and is headline: print headline underline\n underline += '| ' + ' | '.join(['---'] * len(cells)) + ' |' + '\\n'\n elif (not el.previous_sibling\n and (el.parent.name == 'table'\n or (el.parent.name == 'tbody'\n and not el.parent.previous_sibling))):\n # first row, not headline, and:\n # - the parent is table or\n # - the parent is tbody at the beginning of a table.\n # print empty headline above this row\n overline += '| ' + ' | '.join([''] * len(cells)) + ' |' + '\\n'\n overline += '| ' + ' | '.join(['---'] * len(cells)) + ' |' + '\\n'\n if len(text.replace('|', ' ').strip()) == 0:\n return overline + underline\n else:\n return overline + '|' + text.replace('\\n', ' ') + '\\n' + underline\n\n def convert_a(self, el, text, convert_as_inline):\n prefix, suffix, text = chomp(text)\n if not text:\n return ''\n href = el.get('href')\n title = el.get('title')\n # For the replacement see #29: text nodes underscores are escaped\n if (self.options['autolinks']\n and text.replace(r'\\_', '_') == href\n and not title\n and not self.options['default_title']):\n # Shortcut syntax\n return '<%s>' % href\n if self.options['default_title'] and not title:\n title = href\n title_part = ' \"%s\"' % title.replace('\"', r'\\\"') if title else ''\n # return '%s[%s](%s%s)%s' % (prefix, text, href, title_part, suffix) if href else text\n return '%s %s %s' % (prefix, text.replace('\\n', ' '), suffix) if href else text\n\n def convert_pre(self, el, text, convert_as_inline):\n if not text:\n return ''\n code_language = self.options['code_language']\n\n if self.options['code_language_callback']:\n code_language = self.options['code_language_callback'](el) or code_language\n\n return '\\n```%s\\n%s\\n```\\n' % (code_language, text)\n\ndef html2text(soup, **options):\n def clean_markdown(md):\n fields = md.split('\\n')\n for i in range(len(fields)):\n fields[i] = fields[i].strip()\n\n new_fields = []\n for i in range(len(fields)):\n field_set = list(set(fields[i]))\n if len(field_set) == 1 and field_set[0] in ['#', '*', '+', '-']:\n continue\n new_fields.append(fields[i])\n\n fields = new_fields\n md = '\\n'.join(fields)\n\n return re.sub('\\n{2,}', '\\n\\n', md).strip()\n\n return clean_markdown(User_MarkdownConverter(**options).convert_soup(soup))\n\ndef trans2md(html):\n soup = BeautifulSoup(html, 'html5lib')\n markdown_text = html2text(soup)\n # assert len(markdown_text) > 50 and len(markdown_text.split('\\n')) != 1\n if markdown_text.startswith('.') and markdown_text.endswith('.'):\n markdown_text = markdown_text[1:-1]\n main_text = remove_dup_newline(soup.text)\n return markdown_text, main_text\n\n@classmethod\ndef _patch_newspaper_parser_clean(cls, node):\n return node\n\n@patch('newspaper.parsers.Parser.clean_article_html', new=_patch_newspaper_parser_clean)\ndef extract(soup):\n main_html, main_text = get_main_text_html(soup)\n markdown_text, _new_main_text = trans2md(main_html)\n return markdown_text, main_text\n\ndef extract_article_layer(id_html, variables=dict()):\n ret = None\n try:\n LA_TIER1 = [\"en\", \"es\", \"ja\", \"fr\", \"de\", \"pt\", \"it\", \"zh\"]\n LA_TIER2 = [\"nl\", \"sv\", \"da\", \"fi\", \"ru\", \"no\", \"ko\", \"zh\", \"pl\", \"tr\", \"ar\", \"he\", \"pt\", \"cs\", \"hu\", \"th\", \"hi\"]\n LA_TIER = LA_TIER1 + LA_TIER2\n article_id, html = id_html\n \n soup = BeautifulSoup(html, 'html5lib')\n soup = filter_tags_in_html(soup)\n la, la_prob = lid(soup, global_var.lid_model)\n if la in LA_TIER:\n main_md, main_text = extract(soup)\n if len(main_text) >= 128:\n ret = {\"id\": article_id, \"text\": main_text, \"lang\": la, \"lang_prob\": la_prob}\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == '__main__':\n id_html = (None, None)\n id_text_la = extract_article_layer(id_html)\n print(id_text_la)\n" }, { "path": "DomainSpecific/core/layers/extract/search_index_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nimport faiss\nimport numpy as np\nimport traceback\n\ndef search_index_layer(index, query_vectors, variables=dict(), TOPK=1):\n ret = (None, None)\n try:\n query_vectors = np.array(query_vectors)\n D, I = index.search(query_vectors, TOPK)\n ret = (I, D)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return ret\n\n\nif __name__ == '__main__':\n DIM = 4096\n CLUSTERS = 2\n base_vectors = np.random.random((100000, DIM)).astype('float32')\n base_vectors[:, 0] += np.arange(100000) / 1000.\n \n quantizer = faiss.IndexFlatL2(DIM)\n index = faiss.IndexIVFFlat(quantizer, DIM, CLUSTERS, faiss.METRIC_L2)\n\n assert not index.is_trained\n index.train(base_vectors)\n assert index.is_trained\n index.add(base_vectors)\n\n query_vectors = np.random.random((10000, DIM)).astype('float32')\n query_vectors[:, 0] += np.arange(10000) / 1000.\n\n I, D = search_index_layer(index, query_vectors, D=D)\n print(D[:1])\n" }, { "path": "DomainSpecific/core/layers/global_var.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nimport traceback\n#import torch\nimport fasttext\nfrom transformers import AutoTokenizer, RobertaForSequenceClassification\nfrom dependency.gpt_api import GPTAPI\n\ntry:\n # silences warnings as the package does not properly use the python 'warnings' package\n # see https://github.com/facebookresearch/fastText/issues/1056\n fasttext.FastText.eprint = lambda *args,**kwargs: None\nexcept:\n pass\n\n\"\"\"\nclass OpenQuestionModel:\n def __init__(self, pretrained_model_path, token_model_path=\"cardiffnlp/twitter-roberta-base-emotion\", local_files_only=False):\n # load tokenizer model.\n self.tokenizer = AutoTokenizer.from_pretrained(token_model_path)\n\n # load trained model.\n self.model = RobertaForSequenceClassification.from_pretrained(pretrained_model_path, local_files_only=local_files_only)\n\n def run(self, text, thred=0.5, max_length=512):\n # tokenization.\n inputs = self.tokenizer(text, return_tensors=\"pt\", padding=\"max_length\", truncation=True, max_length=max_length)\n\n # inference.\n with torch.no_grad():\n logits = self.model(**inputs).logits\n logits = logits.softmax(dim=1)[0]\n predicted_idx = logits.argmax().item()\n predicted_label = self.model.config.id2label[predicted_idx]\n predicted_conf = logits[predicted_idx].item()\n if predicted_label == \"LABEL_0\" and predicted_conf < thred:\n predicted_idx = 1\n predicted_label = \"LABEL_1\"\n #return predicted_idx, predicted_label, predicted_conf\n return predicted_label\n\"\"\"\n\n# language detection by fasttext.\nLID_MODEL_PATH = \"./dependency/models/lid.176.bin\"\nif os.path.exists(LID_MODEL_PATH):\n lid_model = fasttext.load_model(LID_MODEL_PATH)\nelse:\n lid_model = None\n\n# math detection by fasttext.\nMATH_FT_MODEL_PATH = \"./dependency/models/math.bin\"\nif os.path.exists(MATH_FT_MODEL_PATH):\n ft_math_model = fasttext.load_model(MATH_FT_MODEL_PATH)\nelse:\n ft_math_model = None\n\n# openquestion detection by fasttext.\nOPENQUESTION_MODEL_PATH = \"./dependency/models/openquestion.bin\"\nif os.path.exists(OPENQUESTION_MODEL_PATH):\n ft_openquestion_model = fasttext.load_model(OPENQUESTION_MODEL_PATH)\nelse:\n ft_openquestion_model = None\n\n# multiple-choice question detection by fasttext.\nMCQ_MODEL_PATH = \"./dependency/models/mcq.bin\"\nif os.path.exists(MCQ_MODEL_PATH):\n ft_mcq_model = fasttext.load_model(MCQ_MODEL_PATH)\nelse:\n ft_mcq_model = None\n\n\"\"\"\n# multiple-choice question detection by pytorch.\nMCQ_PT_MODEL_PATH = \"./dependency/models/mcq.pytorch\"\nif os.path.exists(MCQ_PT_MODEL_PATH):\n py_mcq_model = OpenQuestionModel(MCQ_PT_MODEL_PATH, local_files_only=True)\nelse:\n py_mcq_model = None\n\"\"\"\n\n# gpt agent.\ngpt_api = GPTAPI()\n" }, { "path": "DomainSpecific/core/layers/io/__init__.py", "content": "# IO - read/write\nfrom .to_binary_file_layer import to_binary_file_layer\nfrom .to_line_file_layer import to_line_file_layer\nfrom .to_jsonl_file_layer import to_jsonl_file_layer\nfrom .to_parquet_file_layer import to_parquet_file_layer\nfrom .to_index_file_layer import to_index_file_layer\nfrom .from_binary_file_layer import from_binary_file_layer\nfrom .from_line_file_layer import from_line_file_layer\nfrom .from_jsonl_file_layer import from_jsonl_file_layer\nfrom .from_parquet_file_layer import from_parquet_file_layer\nfrom .from_index_file_layer import from_index_file_layer\nfrom .from_wet_file_layer import from_wet_file_layer\nfrom .from_warc_file_layer import from_warc_file_layer\n\n__all__ = [\n \"to_binary_file_layer\", \n \"to_line_file_layer\", \n \"to_jsonl_file_layer\", \n \"to_parquet_file_layer\", \n \"to_index_file_layer\",\n \"from_binary_file_layer\", \n \"from_line_file_layer\", \n \"from_jsonl_file_layer\", \n \"from_parquet_file_layer\",\n \"from_index_file_layer\",\n \"from_wet_file_layer\", \n \"from_warc_file_layer\",\n]\n" }, { "path": "DomainSpecific/core/layers/io/from_binary_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport util\n\ndef from_binary_file_layer(file_path, variables=dict(), STORAGE_PATH=None):\n ret = None\n try:\n file_path = util.to_real_path(file_path, variables)\n if STORAGE_PATH is not None:\n util.download_file_from_blob(STORAGE_PATH, file_path, file_path)\n\n with open(file_path, \"rb\") as f:\n data = f.read()\n ret = data\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n file_path = \"test.binary\"\n data = from_binary_file_layer(file_path)\n print(data)\n" }, { "path": "DomainSpecific/core/layers/io/from_index_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport faiss\nimport traceback\nimport util\n\ndef from_index_file_layer(file_path, variables=dict(), STORAGE_PATH=None):\n ret = None\n try:\n file_path = util.to_real_path(file_path, variables)\n if STORAGE_PATH is not None:\n util.download_file_from_blob(STORAGE_PATH, file_path, file_path)\n\n index = faiss.read_index(file_path)\n ret = index\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == '__main__':\n file_path = \"index.faiss\"\n index = from_index_file_layer(file_path)\n print(index)\n" }, { "path": "DomainSpecific/core/layers/io/from_jsonl_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport jsonlines\nimport util\n\ndef from_jsonl_file_layer(file_path, variables=dict(), STORAGE_PATH=None):\n ret = list()\n try:\n file_path = util.to_real_path(file_path, variables)\n if STORAGE_PATH is not None:\n util.download_file_from_blob(STORAGE_PATH, file_path, file_path)\n\n with jsonlines.open(file_path) as reader:\n for line in reader:\n ret.append(line)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n file_path = \"test.jsonl\"\n data = from_jsonl_file_layer(file_path)\n print(data)\n" }, { "path": "DomainSpecific/core/layers/io/from_line_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport util\n\ndef from_line_file_layer(file_path, variables=dict(), STORAGE_PATH=None):\n ret = list()\n try:\n file_path = util.to_real_path(file_path, variables)\n if STORAGE_PATH is not None:\n util.download_file_from_blob(STORAGE_PATH, file_path, file_path)\n\n for line in open(file_path, \"r\"):\n line = line.strip()\n ret.append(line)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n file_path = \"test.line\"\n lines = from_line_file_layer(file_path)\n print(lines)\n" }, { "path": "DomainSpecific/core/layers/io/from_parquet_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport pyarrow as pa\nimport pyarrow.parquet as pq\nimport util\n\ndef from_parquet_file_layer(file_path, variables=dict(), STORAGE_PATH=None):\n ret = None\n try:\n file_path = util.to_real_path(file_path, variables)\n if STORAGE_PATH is not None:\n util.download_file_from_blob(STORAGE_PATH, file_path, file_path)\n\n table = pq.read_table(file_path)\n ret = table.to_pylist()\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n file_path = \"test.parquet\"\n data = from_parquet_file_layer(file_path)\n print(data)\n" }, { "path": "DomainSpecific/core/layers/io/from_warc_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nfrom warcio.archiveiterator import ArchiveIterator\nimport util\n\ndef from_warc_file_layer(file_path, variables=dict(), STORAGE_PATH=None):\n ret = None\n try:\n file_path = util.to_real_path(file_path, variables)\n if STORAGE_PATH is not None:\n util.download_file_from_blob(STORAGE_PATH, file_path, file_path)\n\n if os.path.exists(file_path):\n items = list()\n with open(file_path, \"rb\") as input:\n records = ArchiveIterator(input, arc2warc=True)\n for idx, record in enumerate(records):\n if record.rec_type == \"response\" and record.http_headers.get_header(\"Content-Type\", \"\").startswith(\"text/html\"):\n item = dict()\n item[\"uri\"] = record.rec_headers.get(\"WARC-Target-URI\")\n item[\"lang\"] = record.rec_headers.get(\"Detected-Language\")\n item[\"content_length\"] = record.rec_headers[\"Content-Length\"]\n item[\"html\"] = record.content_stream().read()\n items.append(item)\n ret = items\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n file_path = \"test.warc.gz\"\n data = from_warc_file_layer(file_path)\n print(data)\n" }, { "path": "DomainSpecific/core/layers/io/from_wet_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nfrom warcio.archiveiterator import ArchiveIterator\nimport util\n\ndef from_wet_file_layer(file_path, variables=dict(), STORAGE_PATH=None):\n ret = None\n try:\n file_path = util.to_real_path(file_path, variables)\n if STORAGE_PATH is not None:\n util.download_file_from_blob(STORAGE_PATH, file_path, file_path)\n\n if os.path.exists(file_path):\n items = list()\n with open(file_path, \"rb\") as input:\n records = ArchiveIterator(input, arc2warc=False)\n for idx, record in enumerate(records):\n if record.rec_type == \"conversion\":\n item = dict()\n item[\"uri\"] = record.rec_headers.get(\"WARC-Target-URI\")\n item[\"lang\"] = record.rec_headers.get(\"Detected-Language\")\n item[\"content_length\"] = record.rec_headers[\"Content-Length\"]\n item[\"text\"] = record.content_stream().read()\n items.append(item)\n ret = items\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n file_path = \"test.warc.wet.gz\"\n data = from_wet_file_layer(file_path)\n print(data)\n" }, { "path": "DomainSpecific/core/layers/io/to_binary_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport util\n\ndef to_binary_file_layer(bytes, file_path, variables=dict(), STORAGE_PATH=None):\n ret = None\n try:\n file_path = util.to_real_path(file_path, variables)\n util.create_folder_by_file_path(file_path)\n\n with open(file_path, \"wb\") as f:\n f.write(bytes)\n\n if STORAGE_PATH is not None:\n util.upload_file_to_blob(STORAGE_PATH, file_path, file_path)\n\n ret = file_path\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n bytes = b\"hello\"\n file_path = \"test.binary\"\n file_path = to_binary_file_layer(bytes, file_path)\n print(file_path)\n" }, { "path": "DomainSpecific/core/layers/io/to_index_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport faiss\nimport traceback\nimport util\n\ndef to_index_file_layer(index, file_path, variables=dict(), STORAGE_PATH=None):\n ret = None\n try:\n file_path = util.to_real_path(file_path, variables)\n util.create_folder_by_file_path(file_path)\n\n faiss.write_index(index, file_path)\n\n if STORAGE_PATH is not None:\n util.upload_file_to_blob(STORAGE_PATH, file_path, file_path)\n\n ret = file_path\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == '__main__':\n D = 64\n NLIST = 100\n base_vectors = np.random.random((100000, D)).astype('float32')\n base_vectors[:, 0] += np.arange(100000) / 1000.\n \n quantizer = faiss.IndexFlatL2(D)\n index = faiss.IndexIVFFlat(quantizer, D, NLIST, faiss.METRIC_L2)\n\n assert not index.is_trained\n index.train(base_vectors)\n assert index.is_trained\n index.add(base_vectors)\n\n file_path = \"index.faiss\"\n file_path = to_index_file_layer(index, file_path)\n print(file_path)\n" }, { "path": "DomainSpecific/core/layers/io/to_jsonl_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport jsonlines\nimport util\n\ndef to_jsonl_file_layer(data, file_path, variables=dict(), STORAGE_PATH=None):\n ret = None\n try:\n file_path = util.to_real_path(file_path, variables)\n util.create_folder_by_file_path(file_path)\n\n with jsonlines.open(file_path, \"w\") as writer:\n writer.write_all(data)\n\n if STORAGE_PATH is not None:\n util.upload_file_to_blob(STORAGE_PATH, file_path, file_path)\n\n ret = file_path\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n data = [{'id': \"1\", 'html': \"hello\"}, {'id': \"2\", 'html': \"hi\"}]\n file_path = \"test.jsonl\"\n file_path = to_jsonl_file_layer(data, file_path)\n print(file_path)\n" }, { "path": "DomainSpecific/core/layers/io/to_line_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport util\n\ndef to_line_file_layer(lines, file_path, variables=dict(), STORAGE_PATH=None):\n ret = None\n try:\n file_path = util.to_real_path(file_path, variables)\n util.create_folder_by_file_path(file_path)\n\n with open(file_path, \"w\") as f:\n for line in lines:\n f.write(line + \"\\n\")\n\n if STORAGE_PATH is not None:\n util.upload_file_to_blob(STORAGE_PATH, file_path, file_path)\n\n ret = file_path\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n lines = [\"line1\", \"line2\"]\n file_path = \"test.line\"\n file_path = to_line_file_layer(lines, file_path)\n print(file_path)\n" }, { "path": "DomainSpecific/core/layers/io/to_parquet_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport pyarrow as pa\nimport pyarrow.parquet as pq\nimport util\n\ndef to_parquet_file_layer(data, file_path, variables=dict(), STORAGE_PATH=None):\n ret = None\n try:\n file_path = util.to_real_path(file_path, variables)\n util.create_folder_by_file_path(file_path)\n\n table = pa.Table.from_pylist(data)\n pq.write_table(table, file_path)\n\n if STORAGE_PATH is not None:\n util.upload_file_to_blob(STORAGE_PATH, file_path, file_path)\n\n ret = file_path\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n data = [{'id': \"1\", 'html': \"hello\"}, {'id': \"2\", 'html': \"hi\"}]\n file_path = \"test.parquet\"\n file_path = to_parquet_file_layer(data, file_path)\n print(file_path)\n" }, { "path": "DomainSpecific/core/layers/network/__init__.py", "content": "# Network - download/upload\nfrom .upload_file_to_blob_layer import upload_file_to_blob_layer\nfrom .upload_bytes_to_blob_layer import upload_bytes_to_blob_layer\nfrom .download_file_from_blob_layer import download_file_from_blob_layer\nfrom .download_bytes_from_blob_layer import download_bytes_from_blob_layer\nfrom .download_file_from_internet_layer import download_file_from_internet_layer\nfrom .download_bytes_from_internet_layer import download_bytes_from_internet_layer\nfrom .download_url_list_layer import download_url_list_layer\nfrom .download_warc_file_layer import download_warc_file_layer\nfrom .download_warc_indice_layer import download_warc_indice_layer\nfrom .download_urls_from_website_layer import download_urls_from_website_layer\nfrom .download_starcoder_layer import download_starcoder_layer\n\n__all__ = [\n \"upload_file_to_blob_layer\",\n \"upload_bytes_to_blob_layer\",\n \"download_file_from_blob_layer\", \n \"download_bytes_from_blob_layer\", \n \"download_file_from_internet_layer\", \n \"download_bytes_from_internet_layer\", \n \"download_url_list_layer\", \n \"download_warc_file_layer\", \n \"download_warc_indice_layer\", \n \"download_urls_from_website_layer\", \n \"download_starcoder_layer\", \n]\n" }, { "path": "DomainSpecific/core/layers/network/download_bytes_from_blob_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport time\nimport traceback\nimport util\n\ndef download_bytes_from_blob_layer(blob_path, variables=dict(), STORAGE_PATH=None, TRIES=1):\n ret = (None, None, blob_path)\n try:\n for _ in range(TRIES):\n try:\n assert STORAGE_PATH is not None and os.path.exists(STORAGE_PATH)\n storage_config = util.load_yaml(STORAGE_PATH)\n blob_path = util.to_real_path(blob_path, variables)\n file_name = util.md5(blob_path) + util.suffix(blob_path)\n bytes = util.download_bytes_from_blob(storage_config, blob_path)\n ret = (file_name, bytes, None)\n break\n except:\n time.sleep(1)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return ret\n\n\nif __name__ == '__main__':\n blob_path = \"$(azure_blob_path)\"\n STORAGE_PATH = \"resources/environment/llmstore.yaml\"\n bytes = download_bytes_from_blob_layer(blob_path, STORAGE_PATH=STORAGE_PATH)\n print(bytes)\n" }, { "path": "DomainSpecific/core/layers/network/download_bytes_from_internet_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport time\nimport traceback\nimport util\n\ndef download_bytes_from_internet_layer(url, variables=dict(), TRIES=1):\n ret = (None, None, url)\n try:\n for _ in range(TRIES):\n try:\n url = util.to_real_path(url, variables)\n file_name = util.md5(url) + util.suffix(url)\n bytes = util.download_bytes_from_internet(url)\n ret = (file_name, bytes, None)\n break\n except:\n time.sleep(1)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return ret\n\n\nif __name__ == '__main__':\n url = \"https://upload.wikimedia.org/wikipedia/commons/4/4f/SVG_Logo.svg\"\n bytes = download_bytes_from_internet_layer(url)\n print(bytes)\n" }, { "path": "DomainSpecific/core/layers/network/download_file_from_blob_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport time\nimport traceback\nimport util\n\ndef download_file_from_blob_layer(blob_path, variables=dict(), DOWNLOAD_PATH=\".\", STORAGE_PATH=None, TRIES=1):\n ret = (None, blob_path)\n try:\n for _ in range(TRIES):\n try:\n assert STORAGE_PATH is not None and os.path.exists(STORAGE_PATH)\n storage_config = util.load_yaml(STORAGE_PATH)\n blob_path = util.to_real_path(blob_path, variables)\n file_name = util.md5(blob_path) + util.suffix(blob_path)\n file_path = os.path.join(DOWNLOAD_PATH, file_name)\n file_path = util.to_real_path(file_path, variables)\n util.download_file_from_blob(storage_config, blob_path, file_path)\n ret = (file_path, None)\n break\n except:\n time.sleep(1)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return ret\n\n\nif __name__ == '__main__':\n blob_path = \"$(azure_blob_path)\"\n DOWNLOAD_PATH = \"$(local_folder_path)\"\n STORAGE_PATH = \"resources/environment/llmstore.yaml\"\n path = download_file_from_blob_layer(blob_path, DOWNLOAD_PATH=DOWNLOAD_PATH, STORAGE_PATH=STORAGE_PATH)\n print(path)\n" }, { "path": "DomainSpecific/core/layers/network/download_file_from_internet_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport time\nimport traceback\nimport util\n\ndef download_file_from_internet_layer(url, variables=dict(), DOWNLOAD_PATH=\".\", TRIES=1):\n ret = (None, url)\n try:\n for _ in range(TRIES):\n try:\n url = util.to_real_path(url, variables)\n file_name = util.md5(url) + util.suffix(url)\n file_path = os.path.join(DOWNLOAD_PATH, file_name)\n file_path = util.to_real_path(file_path, variables)\n util.download_file_from_internet(url, file_path)\n #bytes = util.download_bytes_from_internet(url)\n #util.upload_bytes_to_blob(variables[\"storage_config\"], bytes, file_path)\n ret = (file_path, None)\n break\n except:\n time.sleep(1)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return ret\n\n\nif __name__ == '__main__':\n url = \"https://upload.wikimedia.org/wikipedia/commons/4/4f/SVG_Logo.svg\"\n DOWNLOAD_PATH = \"$(local_folder_path)\"\n path = download_file_from_internet_layer(url, DOWNLOAD_PATH=DOWNLOAD_PATH)\n print(path)\n" }, { "path": "DomainSpecific/core/layers/network/download_starcoder_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport json\nfrom datetime import datetime\nimport boto3\nfrom botocore import UNSIGNED\nfrom botocore.config import Config\nimport smart_open\nfrom datasets import load_dataset\nimport util\n\ns3 = boto3.client(\"s3\", config=Config(signature_version=UNSIGNED))\n\ndef download_contents(blob_id, src_encoding):\n s3_url = f\"s3://softwareheritage/content/{blob_id}\"\n with smart_open.open(s3_url, \"rb\", compression=\".gz\", transport_params={\"client\": s3}) as fin:\n content = fin.read().decode(src_encoding)\n return content\n\ndef download_starcoder_layer(data_repo, variables=dict(), OUTPUT_FOLDER=\"./\", STORAGE_PATH=None, HUGGINGFACE_TOKEN=None):\n ret = 0\n try:\n worker_id = variables[\"worker_id\"]\n worker_num = variables[\"worker_num\"]\n data_repo = util.to_real_path(data_repo, variables)\n output_folder = util.to_real_path(OUTPUT_FOLDER, variables)\n if STORAGE_PATH is not None:\n storage_config = util.load_yaml(STORAGE_PATH)\n\n ds = load_dataset(data_repo, split=\"train\", streaming=True, token=HUGGINGFACE_TOKEN, cache_dir=f\"./cache.{worker_id}/\")\n ds = ds.filter(lambda row, idx: idx % worker_num == worker_id, with_indices=True)\n\n item_count = 0\n for i, row in enumerate(ds):\n for key in row.keys():\n if isinstance(row[key], datetime):\n row[key] = datetime.timestamp(row[key])\n\n blob_id = row[\"blob_id\"]\n src_encoding = row[\"src_encoding\"]\n\n snapshot_prefix = row[\"snapshot_id\"][:4]\n repo_name = row[\"repo_name\"].replace(\"/\", \"@\")\n branch_name = row[\"branch_name\"].replace(\"/\", \"@\")\n language = row[\"language\"].replace(\" \", \"_\")\n path = row[\"path\"].lstrip(\"/\")\n filename = row[\"filename\"].strip()\n filename = path\n extension = row[\"extension\"].strip()\n\n content = download_contents(blob_id, src_encoding)\n\n code_path = os.path.join(output_folder, snapshot_prefix, repo_name, branch_name, blob_id)\n metadata_path = os.path.join(output_folder, snapshot_prefix, repo_name, branch_name, blob_id + \".json\")\n\n try:\n util.create_folder_by_file_path(code_path)\n with open(code_path, \"w\") as f:\n f.write(content)\n if STORAGE_PATH is not None:\n util.upload_file_to_blob(storage_config, code_path, code_path)\n\n util.create_folder_by_file_path(metadata_path)\n with open(metadata_path, \"w\") as f:\n f.write(json.dumps(row, indent=4) + \"\\n\")\n if STORAGE_PATH is not None:\n util.upload_file_to_blob(storage_config, metadata_path, metadata_path)\n\n if STORAGE_PATH is not None:\n try:\n os.remove(code_path)\n os.remove(metadata_path)\n except OSError:\n pass\n except:\n traceback.print_exc()\n \n item_count += 1\n\n ret = item_count\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == '__main__':\n data_repo = \"$(local_the_stack_v2_dedup_metadata_path)\"\n variables = {\"workspace_dir\": r\"workspace\", \"worker_id\": 0, \"worker_num\": 1}\n OUTPUT_FOLDER = \"$(local_the_stack_v2_dedup_data_path)\"\n STORAGE_PATH = \"resources/storage/llmstore.yaml\"\n HUGGINGFACE_TOKEN = None\n item_count = download_starcoder_layer(data_repo, variables=variables, OUTPUT_FOLDER=OUTPUT_FOLDER, STORAGE_PATH=STORAGE_PATH, HUGGINGFACE_TOKEN=HUGGINGFACE_TOKEN)\n print(item_count)\n" }, { "path": "DomainSpecific/core/layers/network/download_url_list_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport time\nimport gzip\nimport json\nimport requests\nimport traceback\n\ndef download_url_list_layer(index_url, variables=dict(), FILTER_SUFFIXES=(), TRIES=1):\n ret = list()\n try:\n for _ in range(TRIES):\n try:\n resp = requests.get(index_url, stream=True)\n urls = list()\n with gzip.open(resp.raw, 'rt') as f:\n for line in f.readlines():\n text = \"{\" + line.strip().split(\" {\")[1]\n item = json.loads(text)\n url = item[\"url\"]\n suffix = os.path.splitext(url)[1]\n if suffix in FILTER_SUFFIXES:\n urls.append(url)\n ret[0:0] = urls\n break\n except:\n time.sleep(1)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret, [index_url] if len(ret) == 0 else [])\n\n\nif __name__ == '__main__':\n index_url = \"https://data.commoncrawl.org/cc-index/collections/CC-MAIN-2023-23/indexes/cdx-00000.gz\"\n FILTER_SUFFIXES = (\".svg\",)\n urls = download_url_list_layer(index_url, FILTER_SUFFIXES=FILTER_SUFFIXES)\n print(urls)\n" }, { "path": "DomainSpecific/core/layers/network/download_urls_from_website_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport requests\nimport logging\nimport traceback\nimport xml.etree.ElementTree as ET\n\ndef download_urls_from_website_layer(website_url, variables=dict(), FILTER=None):\n ret = list()\n try:\n robot_url = website_url + \"/robots.txt\"\n logging.disable(logging.WARNING)\n\n # get sitemap.\n xml_urls = list()\n whilte_url_prefixs = list()\n black_url_prefixs = list()\n resp = requests.get(robot_url)\n crawler = None\n for line in resp.text.split(\"\\n\"):\n line = line.strip()\n if len(line) == 0:\n continue\n if line.startswith(\"#\"):\n continue\n\n if line.startswith(\"User-agent:\"):\n crawler = line.split(\":\")[-1].strip()\n continue\n\n if crawler != \"*\":\n continue\n if crawler == \"*\" and line.startswith(\"Disallow:\"):\n url_prefix = line.replace(\"Disallow:\", \"\").strip()\n black_url_prefixs.append(url_prefix)\n continue\n if crawler == \"*\" and line.startswith(\"Allow:\"):\n url_prefix = line.replace(\"Allow:\", \"\").strip()\n whilte_url_prefixs.append(url_prefix)\n continue\n if crawler == \"*\" and line.startswith(\"Sitemap:\"):\n xml_url = line.replace(\"Sitemap:\", \"\").strip()\n if (FILTER is None or FILTER in xml_url) and xml_url.endswith(\".xml\"):\n xml_urls.append(xml_url)\n continue\n\n # get urls.\n html_urls = set()\n for xml_url in xml_urls:\n try:\n resp = requests.get(xml_url)\n root = ET.fromstring(resp.content)\n for sitemap in root:\n html_url = list(sitemap)[0].text\n html_urls.add(html_url)\n #nodes = tree.xpath('//a/@href')\n #nodes = tree.xpath(\"//loc\")\n except:\n pass\n\n ret = list(html_urls)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == '__main__':\n website_url = \"https://byjus.com/\"\n FILTER = \"math\"\n urls = download_urls_from_website_layer(website_url, FILTER=FILTER)\n print(urls[0][0])\n" }, { "path": "DomainSpecific/core/layers/network/download_warc_file_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport time\nimport traceback\nimport util\n\ndef download_warc_file_layer(warc_url, variables=dict(), DOWNLOAD_FOLDER=\"./\", CONNECTS=16, TRIES=1, OVERWRITE=False):\n ret = (None, warc_url)\n try:\n if not warc_url.startswith(\"https://\"):\n warc_url = \"https://data.commoncrawl.org/\" + warc_url\n #warc_url = warc_url.replace(\"https://data.commoncrawl.org/\", \"https://ds5q9oxwqwsfj.cloudfront.net/\")# debug\n warc_name = warc_url.split(\"/\")[-3] + \"_\" + os.path.basename(warc_url)\n warc_path = os.path.join(DOWNLOAD_FOLDER, warc_name)\n warc_path = util.to_real_path(warc_path, variables)\n\n for _ in range(TRIES):\n if OVERWRITE or not os.path.exists(warc_path):\n util.create_folder_by_file_path(warc_path)\n commandline = f\"axel -q -n {CONNECTS} -o {warc_path} {warc_url}\"\n exit_status = os.system(commandline)\n else:\n exit_status = 0\n\n if exit_status == 0:\n break\n time.sleep(1)\n\n if exit_status == 0:\n ret = (warc_name, None)\n else:\n ret = (None, warc_url)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return ret\n\n\nif __name__ == '__main__':\n warc_url = \"https://data.commoncrawl.org/crawl-data/CC-MAIN-2022-49/segments/1669446706285.92/warc/CC-MAIN-20221126080725-20221126110725-00000.warc.gz\"\n DOWNLOAD_FOLDER = \"$(local_folder_path)\"\n (success_warc_url, failed_warc_url) = download_warc_file_layer(warc_url, DOWNLOAD_FOLDER=DOWNLOAD_FOLDER)\n print(success_warc_url, failed_warc_url)\n" }, { "path": "DomainSpecific/core/layers/network/download_warc_indice_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport time\nimport gzip\nimport requests\nimport traceback\n\ndef download_warc_indice_layer(index_url, variables=dict(), TRIES=1, URL_PREFIX=\"https://data.commoncrawl.org/\"):\n ret = list()\n try:\n for _ in range(TRIES):\n try:\n resp = requests.get(index_url, stream=True)\n urls = list()\n with gzip.open(resp.raw, 'rt') as f:\n for line in f.readlines():\n warc_url = URL_PREFIX + line.strip()\n urls.append(warc_url)\n ret = urls\n break\n except:\n time.sleep(1)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret, [index_url] if len(ret) == 0 else [])\n\n\nif __name__ == '__main__':\n index_url = \"https://data.commoncrawl.org/crawl-data/CC-MAIN-2022-49/warc.paths.gz\"\n warc_urls = download_warc_indice_layer(index_url)\n print(warc_urls[0][0])\n" }, { "path": "DomainSpecific/core/layers/network/upload_bytes_to_blob_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport time\nimport traceback\nimport util\n\ndef upload_bytes_to_blob_layer(bytes, blob_path, variables=dict(), STORAGE_PATH=None, BLOB_PREFIX=\"\", TRIES=1):\n ret = (None, blob_path)\n try:\n for _ in range(TRIES):\n try:\n assert STORAGE_PATH is not None and os.path.exists(STORAGE_PATH)\n storage_config = util.load_yaml(STORAGE_PATH)\n blob_path = util.to_real_path(os.path.join(BLOB_PREFIX, blob_path), variables)\n util.upload_bytes_to_blob(storage_config, bytes, blob_path)\n ret = (blob_path, None)\n break\n except:\n time.sleep(1)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return ret\n\n\nif __name__ == '__main__':\n bytes = b\"hello\"\n blob_path = \"$(azure_blob_path)\"\n STORAGE_PATH = \"resources/environment/llmstore.yaml\"\n path = upload_bytes_to_blob_layer(bytes, blob_path, STORAGE_PATH=STORAGE_PATH)\n print(path)\n" }, { "path": "DomainSpecific/core/layers/network/upload_file_to_blob_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport time\nimport traceback\nimport util\n\ndef upload_file_to_blob_layer(file_path, blob_path, variables=dict(), STORAGE_PATH=None, BLOB_PREFIX=\"\", TRIES=1):\n ret = (None, blob_path)\n try:\n for _ in range(TRIES):\n try:\n assert STORAGE_PATH is not None and os.path.exists(STORAGE_PATH)\n storage_config = util.load_yaml(STORAGE_PATH)\n file_path = util.to_real_path(file_path, variables)\n blob_path = util.to_real_path(os.path.join(BLOB_PREFIX, blob_path), variables)\n util.upload_file_to_blob(storage_config, file_path, blob_path)\n ret = (blob_path, None)\n break\n except:\n time.sleep(1)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return ret\n\n\nif __name__ == '__main__':\n blob_path = \"$(azure_blob_path)\"\n file_path = \"$(local_file_path)\"\n STORAGE_PATH = \"resources/environment/llmstore.yaml\"\n path = upload_file_to_blob_layer(file_path, blob_path, STORAGE_PATH=STORAGE_PATH)\n print(path)\n" }, { "path": "DomainSpecific/core/layers/template_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport sys\nimport traceback\n\n# Spec of adding a new layer:\n# 1. the layer function should be registered in __init__.py file of current folder.\n# 2. the layer function should return tuple value, even though the return value is empty.\n# 3. the layer function should contain a \"variables\" variable in dictionary type for the access of global shared variables.\n# 4. It's better to implement the unit test and put it to the \"__main__\" function.\n# 5. It's better to have exception handling for the function logic.\n# 6. It's better to end with \"_layer\" for the name of function.\n# 7. It's better to write comments for the function of purpose, input and output.\n# 8. It's better to be lowercase for the name of input datas.\n# 9. It's better to be uppercase for the name of input parameters.\n\ndef template_layer(input, variables=dict(), PARAM=None):\n ret = None\n try:\n ret = input\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret,)\n\n\nif __name__ == \"__main__\":\n input = None\n output = template_layer(input)\n" }, { "path": "DomainSpecific/core/layers/transform/__init__.py", "content": "# Transform\nfrom .tokenize_article_layer import tokenize_article_layer\nfrom .ngrams_layer import ngrams_layer\nfrom .minhash_tokens_layer import minhash_tokens_layer\nfrom .lsh_minhash_layer import lsh_minhash_layer\nfrom .warc_filter_layer import warc_filter_layer\nfrom .warc_encode_layer import warc_encode_layer\nfrom .warc_to_wet_layer import warc_to_wet_layer\nfrom .wet_decode_layer import wet_decode_layer\nfrom .math_filter_layer import math_filter_layer\nfrom .openquestion_filter_layer import openquestion_filter_layer\nfrom .mcq_filter_layer import mcq_filter_layer\n\n__all__ = [\n \"tokenize_article_layer\", \n \"ngrams_layer\", \n \"minhash_tokens_layer\", \n \"lsh_minhash_layer\", \n \"warc_filter_layer\", \n \"warc_encode_layer\", \n \"warc_to_wet_layer\", \n \"wet_decode_layer\", \n \"math_filter_layer\",\n \"openquestion_filter_layer\",\n \"mcq_filter_layer\",\n]\n" }, { "path": "DomainSpecific/core/layers/transform/lsh_minhash_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport time\nimport traceback\nimport numpy as np\nfrom scipy.integrate import quad as integrate\n\n# different from datasketch's implementation, will use 2^61-1 as the maximum hash value instead of 2^32-1\nNUM_PERM = 256\nLSH_THRESHOLD = 0.8\n\nclass LSH:\n def __init__(self):\n # gen lsh range\n b, r = self.optimal_param(LSH_THRESHOLD, NUM_PERM, 0.5, 0.5)\n self.hashranges = [(i*r, (i+1)*r) for i in range(b)]\n \n # gen lsh param\n # https://github.com/ekzhu/datasketch/blob/44077457d32887a91297f15c3efee2c1982f690e/datasketch/lsh.py\n def false_positive_probability(self, threshold, b, r):\n _probability = lambda s : 1 - (1 - s**float(r))**float(b)\n a, err = integrate(_probability, 0.0, threshold)\n return a\n\n def false_negative_probability(self, threshold, b, r):\n _probability = lambda s : 1 - (1 - (1 - s**float(r))**float(b))\n a, err = integrate(_probability, threshold, 1.0)\n return a\n\n def optimal_param(self, threshold, num_perm, false_positive_weight,\n false_negative_weight):\n '''\n Compute the optimal `MinHashLSH` parameter that minimizes the weighted sum\n of probabilities of false positive and false negative.\n '''\n min_error = float(\"inf\")\n opt = (0, 0)\n for b in range(1, num_perm+1):\n max_r = int(num_perm / b)\n for r in range(1, max_r+1):\n fp = self.false_positive_probability(threshold, b, r)\n fn = self.false_negative_probability(threshold, b, r)\n error = fp*false_positive_weight + fn*false_negative_weight\n if error < min_error:\n min_error = error\n opt = (b, r)\n return opt\n\n def gen_lsh(self, minhash):\n return [bytearray(minhash[start:end]) for start, end in self.hashranges]\n\nlsh = LSH()\n\ndef lsh_minhash_layer(minhash, variables=dict()):\n ret = list()\n try:\n minhash = np.array(minhash, dtype=np.uint64)\n #assert minhash.dtype == np.uint64 and minhash.shape == (NUM_PERM,)\n lshvalues = lsh.gen_lsh(minhash)\n for i, value in enumerate(lshvalues):\n key = f'{i}_'.encode() + value\n ret.append(key)\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return ret\n\n\nif __name__ == \"__main__\":\n minhash = [2170239837623632,1287605064391826,7877338491737559,1522708576701298,1959803855170230,136353893425081,3067530819312822,19822079906565762,14191953696745176,371933081470560,2359093478290026,24211742396711177,5207401883495830,3386445753675098,6482843287028185,14956790165792002,7760994632330526,3801562091963312,654119844389846,6118541550243605,1058268864309841,19648312785892006,5519054639081138,17769255728697304,1326859272534844,6541616202650748,11131462447891679,11540424367241221,6416091255362971,1178274890175074,9516296843449206,5019313649584786,556043434180166,3170749841321737,788403856226243,16256424180717928,11536645058081246,13331271075979702,5603975614240490,11332978618315755,49833277925775,28529817665769800,5399529123965416,5804862109442032,10516842515700528,1383775130067327,9593857895450592,344120332429946,3650720026287843,4927677784872807,3114522307389328,1054088699310940,11453703275676121,17145094372333782,11943406601641085,429519913626747,3559765888081715,6380853683568781,13142954055708448,1122751140539670,7679037943867431,23532369906879837,4460946791673399,6284691595180437,5534632051525650,4326069154983305,6645880540672905,1199004738171304,2741143312089611,3315947713975755,33325056362165,17905224452748795,11081894870845940,2429362824597352,8796539339687473,17606225237179401,2406479086961618,25285711888782525,1847958183256316,4198878926995358,5057832224878357,10146090240130753,2413082792037196,3530471135853536,7672611456084586,2230458118023706,9790058494528486,3351632677682193,6902744571969727,4063006572456150,2761280786272613,6242978327908865,26924233559187524,2214283527827093,951652422014210,1577851399523074,282734099627651,4284321096276342,1571021659718705,2064444079057042,25995837896147107,3642452037001290,615591136529782,2579917399379439,10350113780305730,141093940432428,9292013714641581,16926413460125,4351013271280123,4492914008491347,3885988895709230,3643655265951773,4028855757933683,10480484972551973,2399277677842610,391439629014342,4511050103292841,13930059233224697,10142483490268814,10209387364437517,10291028774837120,1963510243393060,6698235608219585,10249974506598137,2090329927024291,19452257405817527,5395347850501660,1466647506773938,18271233688875585,17909487123073655,22732716574954981,28208124344155426,16118266291737203,6436198404802809,935143955767639,4692764892567773,8853071216371112,1600664618209927,39702070969452097,7552579352900360,2729546584440357,12309935356310386,426760114692333,1297488733224877,153415463561661,18948566290952420,8432980683248649,21321844297374743,8265174613176795,905258690673816,705406607744747,9105597597214747,517772088040257,1591136193162784,27511729624229236,3634922285407283,1831578225426174,13255266977668852,15312685554649660,722931468693513,1049089865098577,3498618026981595,4820015824926872,21126162808808528,27814106051492575,4822875592156961,14999120736412943,10825146296544249,6314954554132894,937945964737656,5459760788750366,3819227047549912,6591064604768721,7907494363943122,3486632627636937,9384132089104933,22104346516322826,6658745931891482,34093012584282609,4995951742943174,3517485897161771,135044219482780,7630383357514628,5162177136386332,10728488430543051,5828055747100055,6893511170015442,11011121196423559,2528283999013590,5080079240873515,19593423843180365,6822359610856040,191087978655560,8846708703413576,33146998994366094,3940701969864300,3507581990705859,6201879648552385,27956522101531374,10178358282977630,2205391899838384,2614926987404300,1090899715885363,6945147978151211,5432157012678156,1250518799355535,3948407147690489,10306927288370802,4580562167416191,8475303907451120,2243101892749971,2451601302451002,2180238663422921,3834240093757495,12119880871693653,12134080723101916,1805202361835209,31781168568203930,42987808989068825,41914343122681270,7985132073155851,16763654385115268,1387995454655588,2351466328427087,3139781779642664,27792958762616566,11961004800461011,6612181571493100,22715857059525182,689087660337260,244785061275028,11511948953811059,8237401627755449,8214914423544509,5470929524034644,9110614658125771,17166417582628999,18571246019891132,3766276759071421,1226388404627669,9965671498507403,1214978610204088,7808074359603991,1313444080667563,9031456783378283,3783393382666945,34163041205217466,3314866608200743,3451870308271748,11716681494447625,1667361573332888,13859255454740261,7299000064706400,6085019581018810,4996856251238621,5666642298303467]\n lsh_values = lsh_minhash_layer(minhash)\n print(lsh_values)\n" }, { "path": "DomainSpecific/core/layers/transform/math_filter_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport re\nimport requests\nimport fasttext\nfrom gensim.utils import simple_preprocess\nimport pyarrow as pa\nimport pyarrow.parquet as pq\nimport util\nimport global_var\n\nwhilte_list = {r\"\\\\displaystyle\", r\"\\\\alpha\", r\"\\\\beta\", r\"\\\\gamma\", r\"\\\\delta\", r\"\\\\zeta\", r\"\\\\eta\", r\"\\\\iota\", r\"\\\\kappa\", r\"\\\\mu\", r\"\\\\nu\", r\"\\\\xi\", r\"\\\\rho\", r\"\\\\tau\", r\"\\\\phi\", r\"\\\\chi\", r\"\\\\psi\", r\"\\\\omicron\", r\"\\\\epsilon\", r\"\\\\pi\", r\"\\\\lambda\", r\"\\\\omega\", r\"\\\\sigma\", r\"\\\\theta\", r\"\\\\vartheta\", r\"\\\\times\", r\"\\\\cdot\", r\"\\\\dot\", r\"\\\\div\", r\"\\\\frac\", r\"\\\\log\", r\"\\\\exp\", r\"\\\\poly\", r\"\\\\eq\", r\"\\\\neq\", r\"\\\\leq\", r\"\\\\geq\", r\"\\\\approx\", r\"\\\\infty\", r\"\\\\int\", r\"\\\\sum\", r\"\\\\lim\", r\"\\\\begin\", r\"\\\\subset\", r\"\\\\supset\", r\"\\\\top\", r\"\\\\star\", r\"\\\\sim\", r\"\\\\simeq\", r\"\\\\ne\", r\"\\\\ll\", r\"\\\\gg\", r\"\\\\pm\", r\"\\\\mp\", r\"\\\\triangleleft\", r\"\\\\triangleright\", r\"\\\\ast\", r\"\\\\circ\", r\"\\\\bullet\", r\"\\\\oplus\", r\"\\\\odot\", r\"\\\\otimes\", r\"\\\\ominus\", r\"\\\\oslash\", r\"\\\\bigcirc\", r\"\\\\wr\", r\"\\\\dagger\", r\"\\\\bigtriangleup\", r\"\\\\bigtriangledown\", r\"\\\\setminus\", r\"\\\\sqcup\", r\"\\\\wedge\", r\"\\\\dotplus\", r\"\\\\centerdot\", r\"\\\\ltimes\", r\"\\\\rtimes\", r\"\\\\prod\", r\"\\\\coprod\", r\"\\\\iint\", r\"\\\\iiint\", r\"\\\\iiiint\", r\"\\\\idotsint\", r\"\\\\bigoplus\", r\"\\\\big\", r\"\\\\oint\", r\"\\\\rightarrow\", r\"\\\\to\", r\"\\\\leftarrow\", r\"\\\\gets\", r\"\\\\uparrow\", r\"\\\\downarrow\", r\"\\\\forall\", r\"\\\\exists\", r\"\\\\pmod\", r\"\\\\cup\", r\"\\\\cap\", r\"\\\\hat\", r\"\\\\acute\", r\"\\\\check\", r\"\\\\grave\", r\"\\\\vec\", r\"\\\\ddot\", r\"\\\\tilde\", r\"\\\\breve\", r\"\\\\mathring\", r\"\\\\land\", r\"\\\\lor\", r\"\\\\lnot\", r\"\\\\in\", r\"\\\\smile\", r\"\\\\frown\", r\"\\\\infty\", r\"\\\\mid\", r\"\\\\sin\", r\"\\\\cos\", r\"\\\\tan\", r\"\\\\equiv\", r\"\\\\circ\", r\"\\\\dfrac\", r\"\\\\prec\", r\"\\\\preccurlyeq\", r\"\\\\sqrt\",}\nblack_list = {r\"\\\\text\", r\"\\\\if\", r\"\\\\local\", r\"\\\\usr\", r\"\\\\include\", r\"\\\\lib\", r\"\\\\bin\", r\"\\\\url\", r\"\\\\program\", r\"\\\\microsoft\", r\"\\\\temp\", r\"\\\\windows\", r\"\\\\documents\", r\"\\\\users\", r\"\\\\my\", r\"\\\\the\",}\nkeywords1 = whilte_list - black_list\nkeywords1 = set(map(lambda x: x + \"[^a-zA-Z]\", keywords1))\n\nkeywords2 = {r\"\\+\", r\"\\-\", r\"\\*\", r\"\\/\", r\"\\%\", r\"\\=\", r\"\\!\\=\", r\"\\<\", r\"\\>\", r\"\\^\", r\"\\_\", r\"\\(\", r\"\\)\", r\"\\[\", r\"\\]\", r\"\\{\", r\"\\}\", r\"\\|\\|\", r\"\\&\\&\", r\"sqrt\", r\"sum\", r\"int\", r\"\\$\", r\"\\\", r\"\\[math\\]\", }\n\npattern0 = re.compile(r\"\\\\[A-Z]{0,9}[a-z]{2,9}\")\npattern1 = re.compile(\"|\".join(keywords1))\npattern2 = re.compile(\"|\".join(keywords2))\n\ndef ismath_by_model(text, model, thred=0.5):\n if model is None:\n return False\n if not isinstance(text, str) or len(text.strip()) == 0:\n return False\n try:\n x = \" \".join(simple_preprocess(text))\n ret = model.predict(x)\n label, prob = ret[0][0], ret[1][0]\n return label != \"__label__0\"\n except:\n traceback.print_exc()\n return False\n\ndef math_filter_layer(pq_name, variables=dict(), INPUT_FOLDER=\"./\", OUTPUT_FOLDER=\"./\", OVERWRITE=False):\n ret = list()\n try:\n in_pq_path = os.path.join(INPUT_FOLDER, pq_name)\n in_pq_path = util.to_real_path(in_pq_path, variables)\n out_pq_path = os.path.join(OUTPUT_FOLDER, pq_name)\n out_pq_path = util.to_real_path(out_pq_path, variables)\n\n if os.path.exists(in_pq_path) and (OVERWRITE or not os.path.exists(out_pq_path)):\n util.create_folder_by_file_path(out_pq_path)\n\n # read parquet file.\n try:\n table = pq.read_table(in_pq_path)\n except:\n traceback.print_exc()\n \n # filter records containing math.\n records = list()\n for record in table.to_pylist():\n try:\n text = record[\"text\"]\n\n if record[\"la\"] != \"en\":\n continue\n\n #if item[\"la_prob\"] < 0.65:\n # continue\n #if text is None or len(text) < 64:\n # continue\n #if text.count(\"\\\\u\") >= 10:\n # continue\n\n #if not check_quality(record):\n # continue\n\n symbols0 = set(pattern0.findall(text))\n if len(symbols0) <= 0:\n continue\n\n symbols1 = set(pattern1.findall(text.lower()))\n symbols1 = set(map(lambda sym: sym[:-1], symbols1))\n if len(symbols1) <= 0:\n continue\n\n symbols2 = set(pattern2.findall(text.lower()))\n if len(symbols1) == 1 and len(symbols2) <= 0:\n continue\n\n ismath = len(symbols1) >= 5 or ismath_by_model(text, global_var.ft_math_model)\n if not ismath:\n continue\n\n records.append(record)\n except:\n traceback.print_exc()\n\n # write parquet file.\n try:\n table = pa.Table.from_pylist(records)\n pq.write_table(table, out_pq_path)\n except:\n traceback.print_exc()\n \n ret = [out_pq_path]\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret, )\n\n\nif __name__ == '__main__':\n snapshot = \"CC-MAIN-2022-49\"\n variables = {\"workspace_dir\": r\"workspace\", \"worker_id\": 0, \"worker_num\": 1}\n INPUT_FOLDER = \"$(input_data_folder)\"\n OUTPUT_FOLDER = \"$(output_data_folder)\"\n STORAGE_PATH = \"resources/storage/llmstore.yaml\"\n ret = math_filter_layer(snapshot, variables=variables, INPUT_FOLDER=INPUT_FOLDER, OUTPUT_FOLDER=OUTPUT_FOLDER, STORAGE_PATH=STORAGE_PATH)\n print(ret)\n" }, { "path": "DomainSpecific/core/layers/transform/mcq_filter_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport re\nimport json\nimport fasttext\nimport requests\nfrom io import BytesIO\nfrom gensim.utils import simple_preprocess\nfrom warcio.limitreader import LimitReader\nfrom warcio.warcwriter import WARCWriter\nfrom warcio.archiveiterator import ArchiveIterator\nimport util\nimport global_var\n\n\ndef detect_lang(text):\n try:\n LID_WIN_SIZE = 256\n text = ''.join(text.split())\n span_start, span_end = 0, len(text)\n if len(text) > LID_WIN_SIZE:\n mid = len(text) // 2\n mid_win = LID_WIN_SIZE // 2\n span_start = max(0, int(mid - mid_win))\n span_end = min(len(text), int(mid + mid_win))\n det_text = text[span_start: span_end]\n res = global_var.lid_model.predict(det_text)\n lang = res[0][0].replace(\"__label__\", \"\")\n prob = float(res[1][0])\n return lang\n except:\n return \"unkown\"\n\n\ndef detect_choice_exercise_by_rule(uri, html):\n uri = uri.lower()\n html = html.lower()\n contain_cnt = 0\n\n keywords_in_text = [b\"choice question\"]\n for keyword in keywords_in_text:\n if keyword in html:\n contain_cnt += 1\n break\n\n combo_keywords_in_text = [\n (b\"a.\", b\"b.\", b\"c.\", b\"d.\"),\n (b\"a)\", b\"b)\", b\"c)\", b\"d)\"),\n (b\"\\na \", b\"\\nb \", b\"\\nc \", b\"\\nd \"),\n (b\">a<\", b\">b<\", b\">c<\", b\">d<\"),\n\n (b\"1.\", b\"2.\", b\"3.\", b\"4.\"),\n (b\"1)\", b\"2)\", b\"3)\", b\"4)\"),\n (b\"\\n1 \", b\"\\n2 \", b\"\\n3 \", b\"\\n4 \"),\n (b\">1<\", b\">2<\", b\">3<\", b\">4<\"),\n\n (b\"i.\", b\"ii.\", b\"iii.\", b\"iv.\"),\n (b\"i)\", b\"ii)\", b\"iii)\", b\"iv)\"),\n (b\"\\ni \", b\"\\nii \", b\"\\niii \", b\"\\niv \"),\n (b\">i<\", b\">ii<\", b\">iii<\", b\">iv<\"),\n ]\n\n for combo_keyword in combo_keywords_in_text:\n if combo_keyword[0] in html and combo_keyword[1] in html and combo_keyword[2] in html and combo_keyword[3] in html:\n contain_cnt += 1\n break\n\n return contain_cnt == 2\n\n\ndef detect_choice_exercise_by_ft_model(uri, text, thred=0.5):\n try:\n if not isinstance(text, str) or len(text.strip()) == 0:\n return False\n x = \" \".join(simple_preprocess(text))\n ret = global_var.ft_mcq_model.predict(x)\n label, prob = ret[0][0], ret[1][0]\n if label == \"__label__0\" and prob < thred:\n return True\n return label == \"__label__1\"\n except:\n return False\n\n\"\"\"\ndef detect_choice_exercise_by_pt_model(uri, text, thred=0.5):\n try:\n if not isinstance(text, str) or len(text.strip()) == 0:\n return False\n label = global_var.py_mcq_model.run(text, thred)\n return label == \"LABEL_1\"\n except:\n return False\n\"\"\"\n\n\ndef detect_choice_exercise_by_LLM(text, engine=None):\n system = '''\nYou will be given a text converted from a webpage. Your task is to detect whether it contains choice question by responding with 'yes' or 'no'.\n'''\n answer = global_var.gpt_api.run(system=system, question=text, engine=engine)\n answer = answer.lower().strip()\n if answer.startswith(\"yes\"):\n return True\n elif answer.startswith(\"no\"):\n return False\n else:\n return False\n\n\ndef LCS(str1, str2):\n m = len(str1)\n n = len(str2)\n\n dp = [[0 for _ in range(n+1)] for _ in range(m+1)]\n\n for i in range(1, m+1):\n for j in range(1, n+1):\n if str1[i-1] == str2[j-1]:\n dp[i][j] = dp[i-1][j-1] + 1\n else:\n dp[i][j] = max(dp[i-1][j], dp[i][j-1])\n\n return round(1.0 * dp[m][n] / n, 6)\n\n\ndef localize_choice_exercise_by_LLM(text, engine=None):\n system = '''\nPurpose:\nCreate a multiple-choice question dataset.\n\nTask:\nExtract all multiple-choice questions from the provided text.\n\nRequirements:\n1. If the given text does not contain multiple-choice questions, respond only with \"No multiple-choice questions found\".\n2. Do not modify the original multiple-choice questions.\n3. Ensure all multiple-choice questions are copied without omissions.\n4. Ensure all multiple-choice questions are copied in order.\n5. Ensure all multiple-choice questions are copied under the original layout.\n6. Copy the questions along with their options.\n7. If answers and explanations are provided, copy them as well.\n8. If source materials or reading passage is provided, copy it as well.\n9. Don't add content not from original given text.\n\nPlease strictly adhere to these requirements while performing the task.\n'''\n exercises = global_var.gpt_api.run(system=system, question=text, engine=engine)\n exercises = exercises.strip()\n if len(exercises) == 0 or \"no multiple-choice question\" in exercises.lower():\n return None\n else:\n exercises = exercises.replace(\"Multiple Choice Questions\\n\", \"\")\n exercises = exercises.replace(\"Multiple-choice questions:\\n\", \"\")\n exercises = exercises.replace(\"No other multiple-choice questions found.\", \"\")\n exercises = exercises.replace(\"No other multiple-choice questions found in the text.\", \"\")\n exercises = exercises.replace(\"No multiple-choice questions found.\", \"\")\n exercises = exercises.replace(\"No more multiple-choice questions found.\", \"\")\n\n sim = LCS(text, exercises)\n if sim < 0.9:\n return None\n else:\n return exercises\n\n\n# rule + model + GPT3.5 turbor.\ndef mcq_filter_layer(wet_file_name, variables=dict(), INPUT_FOLDER=\"./\", OUTPUT_FOLDER=\"./\", OVERWRITE=False):\n ret = list()\n try:\n src_wet_file_path = os.path.join(INPUT_FOLDER, wet_file_name)\n src_wet_file_path = util.to_real_path(src_wet_file_path, variables)\n jsonl_file_name = wet_file_name.replace(\".warc.wet.gz\", \".jsonl\")\n dst_jsonl_file_path = os.path.join(OUTPUT_FOLDER, jsonl_file_name)\n dst_jsonl_file_path = util.to_real_path(dst_jsonl_file_path, variables)\n\n if os.path.exists(src_wet_file_path) and (OVERWRITE or not os.path.exists(dst_jsonl_file_path)):\n items = list()\n with open(src_wet_file_path, \"rb\") as input:\n records = ArchiveIterator(input, arc2warc=False)\n for id, record in enumerate(records):\n if record.rec_type == \"conversion\":\n try:\n # read raw html.\n uri = record.rec_headers[\"WARC-Target-URI\"]\n bs = record.content_stream().read()\n if bs is None:\n continue\n\n text = str(bs, \"utf-8\")\n if text is None:\n continue\n\n # 1st round filter.\n round1_contain_exercise = detect_choice_exercise_by_rule(uri, bs)\n if not round1_contain_exercise:\n continue\n\n # 2nd round filter.\n round2_contain_exercise = detect_choice_exercise_by_ft_model(uri, text, thred=0.825)\n if not round2_contain_exercise:\n continue\n #round2_contain_exercise = detect_choice_exercise_by_pt_model(uri, text, thred=0.99)\n #if not round2_contain_exercise:\n # continue\n\n \"\"\"\n # 3rd round filter.\n round3_contain_exercise = detect_choice_exercise_by_LLM(text, \"gpt-35-turbo\")\n if not round3_contain_exercise:\n continue\n \"\"\"\n\n item = dict()\n item[\"uri\"] = uri\n item[\"text\"] = text\n lang = detect_lang(text)\n item[\"lang\"] = lang\n #exercises = localize_choice_exercise_by_LLM(text, \"gpt-35-turbo\")\n #item[\"exercises\"] = exercises\n items.append(item)\n except:\n traceback.print_exc()\n pass\n with open(dst_jsonl_file_path, \"w\") as output:\n for item in items:\n output.write(json.dumps(item) + \"\\n\")\n ret = [dst_jsonl_file_path]\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret, )\n\n\nif __name__ == '__main__':\n wet_file_name = \"CC-MAIN-20210115134101-20210115164101-00005_5.warc.wet.gz\"\n variables = {\"workspace_dir\": r\"workspace\", \"worker_id\": 0, \"worker_num\": 1}\n INPUT_FOLDER = \"$(input_data_folder)\"\n OUTPUT_FOLDER = \"$(output_data_folder)\"\n ret = mcq_filter_layer(wet_file_name, variables=variables, INPUT_FOLDER=INPUT_FOLDER, OUTPUT_FOLDER=OUTPUT_FOLDER, OVERWRITE=True)\n print(ret)\n" }, { "path": "DomainSpecific/core/layers/transform/minhash_tokens_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport hashlib\nimport traceback\nimport numpy as np\nfrom itertools import tee\n\nMER = 2**61 - 1\nNUM_PERM = 256\nSEED = 1\n\nclass MinHasher:\n def __init__(self):\n np.random.seed(1)\n self.gen = np.random.RandomState(SEED)\n self.a = self.gen.randint(1, MER, (NUM_PERM,), dtype='u8')\n self.b = self.gen.randint(0, MER, (NUM_PERM,), dtype='u8')\n\n def _sha1_hash(self, val):\n val = int.from_bytes(hashlib.sha1(val).digest()[:8], 'little')\n val &= MER\n return np.uint64(val)\n \n def hash(self, sequence):\n res = np.ones(NUM_PERM, dtype='u8') * MER\n for token in sequence:\n hash0 = self._sha1_hash(token.encode('utf8'))\n hash_vec = hash0 * self.a + self.b\n hash_vec %= MER\n res = np.minimum(res, hash_vec)\n return res\n\nminhasher = MinHasher()\n\ndef minhash_tokens_layer(tokens, variables=dict()):\n ret = None\n try:\n minhash = minhasher.hash(tokens)\n ret = minhash\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return ret\n\n\nif __name__ == \"__main__\":\n tokens = {'产权 份额 为 土地 出让', '商品 住房 市场 价格 合理', '确定 , 在 售 房', ', 可 向 代 持', '住房 , 划 拨 土地', '增 购 政府 份额 的', '向社会公布 。 划 拨 土地', '为 商品 住房 , 划', '▁来源 : 中国 网 地产', '出让 土地 共有 产权 保障', '的 , 可 向 代', '售 房 阶段 向社会公布 。', '商品 住房 , 划 拨', '以及 累计 缴纳 社保 或', '性质 转 为 商品 住房', '的 非 市区 户籍 家庭', '购房 款 。 ▁在 使用', '地产 ▁ 杭州市 1 日', '《 杭州市 共有 产权 保障', '住房 享有 与 购买 商品', '类型 商品 住房 市场 价格', '的 申请 , 增 购', '价 按 同 地段 、', '款 。 ▁在 使用 管理', '可根据 支付 能力 在 50%', '按照 单 套 销售 价格', '方可 通过 买卖 等方式 上市', '年限 等相关 条件 。 ▁', '10 年后 , 方可 通过', '市场 价格 合理 优惠 后', '拨 土地 共有 产权 保障', '杭州市 共有 产权 保障 住房', '销售 基准 价 按 同', '能力 在 50% 至 80%', '等相关 条件 。 ▁ 办法', '年 的 , 可 向', '至 80% 范围内 选择 产权', '共有 产权 保障 住房 销售', '符合 限购 政策 前提 下', '购房 家庭 可根据 支付 能力', '提出 共有 产权 保障 住房', '住房 , 购房 家庭 可根据', '。 ▁在 使用 管理 方面', '-12- 03 ▁记者 : ▁来源', '保障 住房 面向 符合条件的 市区', '住房 以及 累计 缴纳 社保', '。 ▁ 办法 明确 ,', '购房 家庭 产权 份额 为', '社保 或 个 税 年限', '价 及其 浮动 幅度 确定', '非 市区 户籍 家庭 供应', '购房 款 。 出让 土地', ', 购房 家庭 可根据 支付', '单 套 销售 价格 对应的', '权利 性质 调整为 出让 。', '03 ▁记者 : ▁来源 :', '▁2021 -12- 03 ▁记者 :', '产权 保障 住房 面向 符合条件的', '日 对外 发布 《 杭州市', '就业 的 非 市区 户籍', '增 购 后 住房 性质', ', 购买 共有 产权 保障', '、 同 类型 商品 住房', '同等 的 公共服务 权益 。', '对应的 不同 比例 支付 购房', '的 公共服务 权益 。 ▁根据', '网 地产 ▁ 杭州市 1', '款 。 出让 土地 共有', '套 销售 价格 对应的 产权', '管理 方面 , 杭州 提出', '住房 , 购房 家庭 产权', '和 稳定 就业 的 非', '土地 权利 性质 调整为 出让', '浮动 幅度 确定 , 在', '不动产 权 证 满 10', '▁ 办法 明确 , 共有', '机构 提出 一次性 增 购', '》 , 其中 明确 ,', '权 证 满 10 年后', '在 50% 至 80% 范围内', '方面 , 杭州 提出 共有', '满 10 年后 , 方可', '基准 价 按 同 地段', '产权 份额 比例 , 按照', '保障 住房 管理办法 》 ,', '居住证 、 住房 以及 累计', '销售 价格 对应的 产权 比例', '住房 面向 符合条件的 市区 户籍', '。 ▁根据 办法 , 市区', '单 套 销售 价格 按照', '销售 基准 价 及其 浮动', ': 中国 网 地产 ▁', '持 机构 提出 一次性 增', '价格 按照 销售 基准 价', '家庭 供应 , 购买 共有', '购买 共有 产权 保障 住房', '稳定 就业 的 非 市区', '购买 商品 住房 同等 的', '其中 明确 , 共有 产权', '▁记者 : ▁来源 : 中国', '价格 对应的 不同 比例 支付', '与 购买 商品 住房 同等', '、 住房 等相关 条件 ,', '条件 。 ▁ 办法 明确', '证 满 5 年 的', '满 5 年 的 ,', '管理办法 》 , 其中 明确', '市区 户籍 家庭 需 满足', '份额 的 申请 , 增', '商品 住房 同等 的 公共服务', '支付 能力 在 50% 至', '权 证 满 5 年', '户籍 家庭 需 满足 居住证', ', 方可 通过 买卖 等方式', ', 在 售 房 阶段', '对应的 产权 比例 支付 购房', '产权 保障 住房 购房 家庭', '家庭 需 满足 居住证 、', '杭州 提出 共有 产权 保障', '1 日 对外 发布 《', ', 其中 明确 , 共有', '满足 居住证 、 住房 以及', '选择 产权 份额 比例 ,', '同时 满足 户籍 、 住房', ', 市区 户籍 家庭 要在', '销售 价格 对应的 不同 比例', '个 税 年限 等相关 条件', '住房 市场 价格 合理 优惠', '产权 保障 住房 , 购房', '、 住房 以及 累计 缴纳', '产权 保障 住房 销售 基准', '后 住房 性质 转 为', '土地 出让 时 已 确定的', '比例 , 按照 单 套', '发布 《 杭州市 共有 产权', '住房 性质 转 为 商品', '累计 缴纳 社保 或 个', '份额 比例 , 按照 单', '时 已 确定的 份额 比例', '划 拨 土地 权利 性质', '基准 价 及其 浮动 幅度', '。 出让 土地 共有 产权', '为 土地 出让 时 已', ', 购房 家庭 产权 份额', '等相关 条件 , 非 市区', '按 同 地段 、 同', '按照 销售 基准 价 及其', '不同 比例 支付 购房 款', '住房 销售 基准 价 按', '家庭 产权 份额 为 土地', '可 向 代 持 机构', '▁在 使用 管理 方面 ,', '家庭 取得 不动产 权 证', '性质 调整为 出让 。 取得', '取得 不动产 权 证 满', '市区 户籍 家庭 要在 符合', ', 杭州 提出 共有 产权', '政策 前提 下 同时 满足', '▁根据 办法 , 市区 户籍', '办法 , 市区 户籍 家庭', '缴纳 社保 或 个 税', '。 划 拨 土地 共有', '家庭 可根据 支付 能力 在', '满足 户籍 、 住房 等相关', '一次性 增 购 政府 份额', '购 政府 份额 的 申请', '需 满足 居住证 、 住房', '同 地段 、 同 类型', '供应 , 购买 共有 产权', '使用 管理 方面 , 杭州', '保障 住房 享有 与 购买', '共有 产权 保障 住房 享有', '限购 政策 前提 下 同时', '套 销售 价格 按照 销售', '户籍 和 稳定 就业 的', '优惠 后 确定 。 单', '住房 管理办法 》 , 其中', '市区 户籍 和 稳定 就业', '支付 购房 款 。 ▁在', '户籍 家庭 供应 , 购买', '同 类型 商品 住房 市场', '保障 住房 购房 家庭 取得', '及其 浮动 幅度 确定 ,', '共有 产权 保障 住房 管理办法', '共有 产权 保障 住房 面向', '在 售 房 阶段 向社会公布', '共有 产权 保障 住房 ,', '政府 份额 的 申请 ,', '买卖 等方式 上市 交易 。', '市区 户籍 家庭 供应 ,', '出让 时 已 确定的 份额', '家庭 要在 符合 限购 政策', '申请 , 增 购 后', ', 非 市区 户籍 家庭', '前提 下 同时 满足 户籍', '划 拨 土地 共有 产权', ', 划 拨 土地 权利', '产权 保障 住房 管理办法 》', '阶段 向社会公布 。 划 拨', '明确 , 共有 产权 保障', '确定的 份额 比例 , 按照', '证 满 10 年后 ,', '通过 买卖 等方式 上市 交易', '已 确定的 份额 比例 ,', '不动产 权 证 满 5', '提出 一次性 增 购 政府', '对外 发布 《 杭州市 共有', '价格 合理 优惠 后 确定', '。 取得 不动产 权 证', '范围内 选择 产权 份额 比例', '房 阶段 向社会公布 。 划', '▁ 杭州市 1 日 对外', '份额 为 土地 出让 时', ', 增 购 后 住房', '地段 、 同 类型 商品', '杭州市 1 日 对外 发布', '户籍 家庭 要在 符合 限购', '保障 住房 销售 基准 价', '调整为 出让 。 取得 不动产', ', 共有 产权 保障 住房', '权益 。 ▁根据 办法 ,', '比例 支付 购房 款 。', '保障 住房 , 购房 家庭', '或 个 税 年限 等相关', '年后 , 方可 通过 买卖', '出让 。 取得 不动产 权', '价格 对应的 产权 比例 支付', '购 后 住房 性质 转', '确定 。 单 套 销售', '支付 购房 款 。 出让', '要在 符合 限购 政策 前提', '拨 土地 权利 性质 调整为', '转 为 商品 住房 ,', '享有 与 购买 商品 住房', '公共服务 权益 。 ▁根据 办法', '中国 网 地产 ▁ 杭州市', '5 年 的 , 可', '合理 优惠 后 确定 。', '办法 明确 , 共有 产权', '共有 产权 保障 住房 购房', '套 销售 价格 对应的 不同', '户籍 、 住房 等相关 条件', '下 同时 满足 户籍 、', '产权 保障 住房 享有 与', '面向 符合条件的 市区 户籍 和', '购房 家庭 取得 不动产 权', '条件 , 非 市区 户籍', '幅度 确定 , 在 售', ': ▁来源 : 中国 网', '代 持 机构 提出 一次性', '产权 比例 支付 购房 款', '80% 范围内 选择 产权 份额', '向 代 持 机构 提出', '住房 同等 的 公共服务 权益', '税 年限 等相关 条件 。', '土地 共有 产权 保障 住房', ', 按照 单 套 销售', '非 市区 户籍 家庭 需', '。 单 套 销售 价格', '符合条件的 市区 户籍 和 稳定', '住房 等相关 条件 , 非', '50% 至 80% 范围内 选择', '后 确定 。 单 套', '住房 购房 家庭 取得 不动产', '销售 价格 按照 销售 基准'}\n minhash = minhash_tokens_layer(tokens)\n print(minhash)\n" }, { "path": "DomainSpecific/core/layers/transform/ngrams_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nfrom itertools import tee\n\nNGRAM_SIZE = 5\n\ndef ngrams_layer(sequence, variables=dict()):\n ret = None\n try:\n # https://github.com/ChenghaoMou/text-dedup/blob/main/text_dedup/utils/tokenization.py\n if len(sequence) < NGRAM_SIZE:\n return iter([sequence])\n iterables = tee(iter(sequence), NGRAM_SIZE)\n for i, sub_iterable in enumerate(iterables):\n for _ in range(i):\n next(sub_iterable, None)\n tokens = zip(*iterables)\n tokens = {\" \".join(t).strip() for t in tokens}\n #tokens = list(tokens)\n ret = tokens\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return ret\n\n\nif __name__ == \"__main__\":\n tokens = ['▁2021', '-12-', '03', '▁记者', ':', '▁来源', ':', '中国', '网', '地产', '▁', '杭州市', '1', '日', '对外', '发布', '《', '杭州市', '共有', '产权', '保障', '住房', '管理办法', '》', ',', '其中', '明确', ',', '共有', '产权', '保障', '住房', '面向', '符合条件的', '市区', '户籍', '和', '稳定', '就业', '的', '非', '市区', '户籍', '家庭', '供应', ',', '购买', '共有', '产权', '保障', '住房', '享有', '与', '购买', '商品', '住房', '同等', '的', '公共服务', '权益', '。', '▁根据', '办法', ',', '市区', '户籍', '家庭', '要在', '符合', '限购', '政策', '前提', '下', '同时', '满足', '户籍', '、', '住房', '等相关', '条件', ',', '非', '市区', '户籍', '家庭', '需', '满足', '居住证', '、', '住房', '以及', '累计', '缴纳', '社保', '或', '个', '税', '年限', '等相关', '条件', '。', '▁', '办法', '明确', ',', '共有', '产权', '保障', '住房', '销售', '基准', '价', '按', '同', '地段', '、', '同', '类型', '商品', '住房', '市场', '价格', '合理', '优惠', '后', '确定', '。', '单', '套', '销售', '价格', '按照', '销售', '基准', '价', '及其', '浮动', '幅度', '确定', ',', '在', '售', '房', '阶段', '向社会公布', '。', '划', '拨', '土地', '共有', '产权', '保障', '住房', ',', '购房', '家庭', '可根据', '支付', '能力', '在', '50%', '至', '80%', '范围内', '选择', '产权', '份额', '比例', ',', '按照', '单', '套', '销售', '价格', '对应的', '不同', '比例', '支付', '购房', '款', '。', '出让', '土地', '共有', '产权', '保障', '住房', ',', '购房', '家庭', '产权', '份额', '为', '土地', '出让', '时', '已', '确定的', '份额', '比例', ',', '按照', '单', '套', '销售', '价格', '对应的', '产权', '比例', '支付', '购房', '款', '。', '▁在', '使用', '管理', '方面', ',', '杭州', '提出', '共有', '产权', '保障', '住房', '购房', '家庭', '取得', '不动产', '权', '证', '满', '5', '年', '的', ',', '可', '向', '代', '持', '机构', '提出', '一次性', '增', '购', '政府', '份额', '的', '申请', ',', '增', '购', '后', '住房', '性质', '转', '为', '商品', '住房', ',', '划', '拨', '土地', '权利', '性质', '调整为', '出让', '。', '取得', '不动产', '权', '证', '满', '10', '年后', ',', '方可', '通过', '买卖', '等方式', '上市', '交易', '。']\n tokens = ngrams_layer(tokens)\n print(tokens)\n" }, { "path": "DomainSpecific/core/layers/transform/openquestion_filter_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport re\nimport gc\nimport requests\nimport fasttext\nfrom gensim.utils import simple_preprocess\nimport pyarrow as pa\nimport pyarrow.parquet as pq\nsys.path.append(\".\")\nimport util\nimport global_var\n\nquestion_keywords = (\"q&a\", \"q & a\", \"q:\", \"que:\", \"question:\", \"quiz:\", \"exam:\", \"examination:\", \"probe:\", \"request:\", \"challenge:\", \"test:\", \"query:\", \"survey:\")\n#question_keywords2 = (\"what \", \"where \", \"why \", \"when \", \"who \", \"whoes \", \"how \", \"\\?\")\nquestion_keywords2 = (\"what\", \"where\", \"why\", \"when\", \"who\", \"whoes\", \"how\")\nquestion_keywords += question_keywords2\nquestion_keywords = set(map(lambda x: \"[^a-zA-Z]\" + x + \"[^a-zA-Z]\", question_keywords))\nquestion_pattern = re.compile(\"|\".join(question_keywords))\n\nanswer_keywords = (\"q&a\", \"q & a\", \"a:\", \"ans:\", \"answer:\", \"solution:\", \"reply:\", \"response:\", \"result:\", \"outcome:\", \"explanation:\", \"conclusion:\", \"finding:\", \"assertion:\", \"statement:\", \"clarification:\")\nanswer_keywords = set(map(lambda x: \"[^a-zA-Z]\" + x + \"[^a-zA-Z]\", answer_keywords))\nanswer_pattern = re.compile(\"|\".join(answer_keywords))\n\n\ndef is_openquestion_by_model(text, model, thred=0.5):\n if model is None:\n return False\n if not isinstance(text, str) or len(text.strip()) == 0:\n return False\n try:\n x = \" \".join(simple_preprocess(text))\n ret = model.predict(x)\n label, prob = ret[0][0], ret[1][0]\n return label != \"__label__0\"\n except:\n traceback.print_exc()\n return False\n\ndef check_yes_no_question(text_before, text_after):\n text_after = text_after.lower().strip()\n keywords = (\"yes\", \"y\", \"no\", \"n\")\n for keyword in keywords:\n if text_after.startswith(keyword) and not text_after[len(keyword)].isalnum():\n return True\n return False\n\ndef check_multiple_choise_question(text_before, text_after):\n combo_keywords_list = [\n (\"a.\", \"b.\", \"c.\", \"d.\"),\n (\"a)\", \"b)\", \"c)\", \"d)\"),\n (\"\\na \", \"\\nb \", \"\\nc \", \"\\nd \"),\n (\">a<\", \">b<\", \">c<\", \">d<\"),\n\n (\"1.\", \"2.\", \"3.\", \"4.\"),\n (\"1)\", \"2)\", \"3)\", \"4)\"),\n (\"\\n1 \", \"\\n2 \", \"\\n3 \", \"\\n4 \"),\n (\">1<\", \">2<\", \">3<\", \">4<\"),\n\n (\"i.\", \"ii.\", \"iii.\", \"iv.\"),\n (\"i)\", \"ii)\", \"iii)\", \"iv)\"),\n (\"\\ni \", \"\\nii \", \"\\niii \", \"\\niv \"),\n (\">i<\", \">ii<\", \">iii<\", \">iv<\"),\n ]\n text_before = text_before.lower().strip()\n for combo_keywords in combo_keywords_list:\n t = 0\n for combo_keyword in combo_keywords:\n t = text_before.find(combo_keyword, t)\n if t == -1:\n break\n if t != -1:\n return True\n #if combo_keywords[0] in text_before and combo_keywords[1] in text_before and combo_keywords[2] in text_before:\n # return True\n return False\n\ndef check_fill_in_question(text_before, text_after):\n text_before = text_before.lower().strip()\n if \"___\" in text_before or \"()\" in text_before or \"...\" in text_before:\n return True\n return False\n\ndef check_quality(item):\n text = item[\"text\"]\n lines = text.split(\"\\n\")\n lens = list(map(lambda l: len(l.strip()), lines))\n max_len = max(lens)\n\n #if max_len > 1024:\n if max_len > 2048:\n return False\n if max_len <= 128:\n return False\n\n if len(lens) <= 3:\n return False\n if len(lens) > 256:\n return False\n\n if len(text) < 256:\n return False\n if len(text) > 1024 * 16:\n return False\n\n if 1.0 * text.count(\" \") / len(text) > 0.33:\n return False\n\n if 1.0 * text.count(\" \") / len(text) > 0.1:\n return False\n\n if 1.0 * text.count(\"\\t\") / len(text) > 0.1:\n return False\n\n if 1.0 * text.count(\".\") / len(text) > 0.1:\n return False\n\n if 1.0 * text.count(\"-\") / len(text) > 0.1:\n return False\n\n if 1.0 * text.count(\"#\") / len(text) > 0.1:\n return False\n\n if 1.0 * text.count(\"|\") / len(text) > 0.1:\n return False\n\n if 1.0 * text.count(\",\") / len(text) > 0.1:\n return False\n\n sl_cnt = 1.0 * len(list(filter(lambda x: len(x.strip()) <= 32, lines))) / len(lines)\n if sl_cnt > 0.67:\n return False\n\n return True\n\ndef openquestion_filter_layer(pq_name, variables=dict(), INPUT_FOLDER=\"./\", OUTPUT_FOLDER=\"./\", OVERWRITE=False):\n ret = list()\n try:\n in_pq_path = os.path.join(INPUT_FOLDER, pq_name)\n in_pq_path = util.to_real_path(in_pq_path, variables)\n out_pq_path = os.path.join(OUTPUT_FOLDER, pq_name)\n out_pq_path = util.to_real_path(out_pq_path, variables)\n\n if os.path.exists(in_pq_path) and (OVERWRITE or not os.path.exists(out_pq_path)):\n util.create_folder_by_file_path(out_pq_path)\n\n # read parquet file.\n try:\n table = pq.read_table(in_pq_path)\n records = table.to_pylist()\n except:\n traceback.print_exc()\n \n # filter records containing open question.\n openquestion_records = list()\n for record_idx, record in enumerate(records):\n try:\n text = record[\"text\"]\n text_low = text.lower()\n\n if record[\"la\"] != \"en\":\n continue\n\n #if item[\"la_prob\"] < 0.65:\n # continue\n #if text is None or len(text) < 64:\n # continue\n #if text.count(\"\\\\u\") >= 10:\n # continue\n\n #if not check_quality(record):\n # continue\n\n contain_question = len(question_pattern.findall(text_low)) >= 2\n if not contain_question:\n continue\n \n contain_answer = len(answer_pattern.findall(text_low)) >= 2\n if not contain_answer:\n continue\n\n contain_openquestion = is_openquestion_by_model(text, global_var.ft_openquestion_model)\n if not contain_openquestion:\n continue\n\n openquestion_records.append(record)\n except:\n traceback.print_exc()\n\n # write parquet file.\n try:\n openquestion_table = pa.Table.from_pylist(openquestion_records)\n pq.write_table(openquestion_table, out_pq_path)\n except:\n traceback.print_exc()\n \n ret = [out_pq_path]\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return (ret, )\n\n\nif __name__ == '__main__':\n snapshot = \"CC-MAIN-2022-49\"\n variables = {\"workspace_dir\": r\"workspace\", \"worker_id\": 0, \"worker_num\": 1}\n INPUT_FOLDER = \"$(input_data_folder)\"\n OUTPUT_FOLDER = \"$(output_data_folder)\"\n STORAGE_PATH = \"resources/storage/llmstore.yaml\"\n ret = openquestion_filter_layer(snapshot, variables=variables, INPUT_FOLDER=INPUT_FOLDER, OUTPUT_FOLDER=OUTPUT_FOLDER, STORAGE_PATH=STORAGE_PATH)\n print(ret)\n" }, { "path": "DomainSpecific/core/layers/transform/tokenize_article_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport traceback\nimport sentencepiece as spm\n\n\ntokenizer = None\n\ndef tokenize_article_layer(article, variables=dict(), SPM_MODEL_PATH=\"./dependency/models/sentencepiece.bpe.model\"):\n ret = None\n try:\n global tokenizer\n if tokenizer is None:\n tokenizer = spm.SentencePieceProcessor(SPM_MODEL_PATH)\n tokens = tokenizer.encode(article, out_type=str)\n ret = tokens\n except KeyboardInterrupt:\n sys.exit()\n except Exception as ex:\n traceback.print_exc()\n return ret\n\n\nif __name__ == \"__main__\":\n article = \"2021-12-03 记者： 来源：中国网地产\\n\\n杭州市1日对外发布《杭州市共有产权保障住房管理办法》，其中明确，共有产权保障住房面向符合条件的市区户籍和稳定就业的非市区户籍家庭供应，购买共有产权保障住房享有与购买商品住房同等的公共服务权益。\\n\\n根据办法，市区户籍家庭要在符合限购政策前提下同时满足户籍、住房等相关条件，非市区户籍家庭需满足居住证、住房以及累计缴纳社保或个税年限等相关条件。\\n\\n办法明确，共有产权保障住房销售基准价按同地段、同类型商品住房市场价格合理优惠后确定。单套销售价格按照销售基准价及其浮动幅度确定，在售房阶段向社会公布。划拨土地共有产权保障住房，购房家庭可根据支付能力在50%至80%范围内选择产权份额比例，按照单套销售价格对应的不同比例支付购房款。出让土地共有产权保障住房，购房家庭产权份额为土地出让时已确定的份额比例，按照单套销售价格对应的产权比例支付购房款。\\n\\n在使用管理方面，杭州提出共有产权保障住房购房家庭取得不动产权证满5年的，可向代持机构提出一次性增购政府份额的申请，增购后住房性质转为商品住房，划拨土地权利性质调整为出让。取得不动产权证满10年后，方可通过买卖等方式上市交易。\"\n tokens = tokenize_article_layer(article)\n print(tokens)\n" }, { "path": "DomainSpecific/core/layers/transform/warc_encode_layer.py", "content": "#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n#\n# coding=utf-8\nimport os\nimport sys\nos.sys.path.append(f\"{os.path.dirname(os.path.realpath(__file__))}/..\")\nimport re\nimport codecs\nimport logging\nimport traceback\nimport requests\nfrom pathlib import Path\nfrom urllib.parse import urlparse\nfrom io import BytesIO\nfrom warcio.limitreader import LimitReader\nfrom warcio.warcwriter import WARCWriter\nfrom warcio.archiveiterator import ArchiveIterator\nimport lxml.etree as ET\nimport lxml.html as HT\nfrom py_asciimath.translator.translator import MathML2Tex\nfrom pylatexenc.latexwalker import LatexWalker\nfrom charset_normalizer import detect\nimport util\n\ndef tex_in_script_tag(text):\n return text.startswith(''),\n (b\"'),\n (b\"'),\n (b\"'),\n (b\"