[
  {
    "path": ".github/pull_request_template.md",
    "content": "# Description\n\n- Related issues:\n  - #\n\n# Changes in this PR\n\n# How has this been tested?\n\n# Checklist\n- [ ] This PR follows the coding-style of this project\n- [ ] I have tested these changes\n- [ ] I have commented hard-to-understand codes\n"
  },
  {
    "path": ".github/workflows/ci.yml",
    "content": "name: CI\n\non: pull_request\n\njobs:\n  black:\n    runs-on: ubuntu-latest\n    steps:\n    - name: Checkout\n      uses: actions/checkout@v2\n    - name: Setup python\n      uses: actions/setup-python@v2\n      with:\n        python-version: 3.9\n    - name: Upgrade pip\n      run: pip install --upgrade pip\n    - name: Install black\n      run: pip install --upgrade black==23.1.0\n    - name: Run black\n      run: black --check .\n\n  isort:\n    runs-on: ubuntu-latest\n    steps:\n    - name: Checkout\n      uses: actions/checkout@v2\n    - name: Setup python\n      uses: actions/setup-python@v2\n      with:\n        python-version: 3.9\n    - name: Upgrade pip\n      run: pip install --upgrade pip\n    - name: Install isort\n      run: pip install --upgrade isort==5.12.0\n    - name: Run isort\n      working-directory: ./cleval\n      run: isort --profile black --check .\n\n  pytest:\n    runs-on: ubuntu-latest\n    steps:\n    - name: Checkout\n      uses: actions/checkout@v2\n    - name: Setup python\n      uses: actions/setup-python@v2\n      with:\n        python-version: 3.9\n    - name: Upgrade pip\n      run: pip install --upgrade pip && pip install -U setuptools wheel\n    - name: Update apt\n      run: sudo apt update\n    - name: Install pre-requirements\n      run: sudo apt install -y libyajl2 libyajl-dev libleveldb-dev libgl1-mesa-glx libglib2.0-0\n    - name: Install cleval\n      run: pip install six && pip install --force-reinstall --no-cache-dir cleval opencv-python-headless\n    - name: Install pytest\n      run: pip install --upgrade pytest\n    - name: Run pytest\n      run: pytest\n"
  },
  {
    "path": ".gitignore",
    "content": "__pycache__/\n.vscode\n.DS_Store\n.idea\noutput/\n.pytest_cache\n.mypy_cache\nbuild/\ndist/\n*.egg-info/\n\nvenv\ndebug*\ntmp*\nprofile.txt\n"
  },
  {
    "path": "LICENSE",
    "content": "Copyright (c) 2020-present NAVER Corp.\n\n Permission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\n The above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\n THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n"
  },
  {
    "path": "NOTICE",
    "content": "CLEval\nCopyright (c) 2020-present NAVER Corp.\n\nThis project contains subcomponents with separate copyright notices and license terms. \nYour use of the source code for these subcomponents is subject to the terms and conditions of the following licenses.\n\n=====\n\n=====\n\nCLEval solves the drawbacks of previous detection and end-to-end metrics such as IoU and DetEval. \nThis code is based on ICDAR15 official evaluation code from https://rrc.cvc.uab.es/.\n\n=====\n\njquery/jquery\nhttp://jquery.com/\n\n\nCopyright JS Foundation and other contributors, https://js.foundation/\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n\"Software\"), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\nNONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE\nLIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\n=====\n\njquery/jquery-ui\nhttps://github.com/jquery/jquery-ui\n\n\nCopyright jQuery Foundation and other contributors, https://jquery.org/\n\nThis software consists of voluntary contributions made by many\nindividuals. For exact contribution history, see the revision history\navailable at https://github.com/jquery/jquery-ui\n\nThe following license applies to all parts of this software except as\ndocumented below:\n\n====\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n\"Software\"), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\nNONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE\nLIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\n====\n\nCopyright and related rights for sample code are waived via CC0. Sample\ncode is defined as all source code contained within the demos directory.\n\nCC0: http://creativecommons.org/publicdomain/zero/1.0/\n\n====\n\nAll files located in the node_modules and external directories are\nexternally maintained libraries used by this software which have their\nown licenses; we recommend you read them, as their terms may differ from\nthe terms above.\n\n=====\n\nmalsup/form\nhttps://github.com/malsup/form\n\n\nCopyright 2006-2013 (c) M. Alsup\n\nAll versions, present and past, of the jQuery Form plugin are dual licensed under the MIT and GPL licenses:\n\nMIT\nGPL\nYou may use either license. The MIT License is recommended for most projects because it is simple and easy to understand and it places almost no restrictions on what you can do with the plugin.\n\nIf the GPL suits your project better you are also free to use the plugin under that license.\n\nYou don't have to do anything special to choose one license or the other and you don't have to notify anyone which license you are using. You are free to use the jQuery Form Plugin in commercial projects as long as the copyright header is left intact.\n\n-----\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\n\n=====\n\ncs-chan/Total-Text-Dataset\nhttps://github.com/cs-chan/Total-Text-Dataset\n\nCopyright (c) 2018, Chee Seng Chan\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n  list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n  this list of conditions and the following disclaimer in the documentation\n  and/or other materials provided with the distribution.\n\n* Neither the name of Total-Text nor the names of its\n  contributors may be used to endorse or promote products derived from\n  this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n=====\n\nICDAR 2013, ICDAR 2015 ground-truth annotation. (gt/gt_IC13.zip, gt/gt_IC15.zip)\nhttps://rrc.cvc.uab.es/?ch=2&com=tasks, https://rrc.cvc.uab.es/?ch=4&com=tasks\n\nThe \"Incidental Scene Text(ICDAR2015)\" dataset and corresponding annotations are licensed under\na Creative Commons Attribution 4.0 License(https://creativecommons.org/licenses/by/4.0/).\n\n====="
  },
  {
    "path": "README.md",
    "content": "# CLEval: Character-Level Evaluation for Text Detection and Recognition Tasks\n\nOfficial implementation of CLEval | [paper](https://arxiv.org/abs/2006.06244)\n\n## Overview\nWe propose a Character-Level Evaluation metric (CLEval). To perform fine-grained assessment of the results, *instance matching* process handles granularity difference and *scoring process* conducts character-level evaluation. Please refer to the paper for more details. This code is based on [ICDAR15 official evaluation code](http://rrc.cvc.uab.es/).\n\n### 2023.10.16 Huge Update\n- **Much More Faster Version** of CLEval has been Uploaded!!\n- Support CLI \n- Support torchmetric\n- Support scale-wise evaluation\n\n\n### Simplified Method Description\n![Explanation](resources/screenshots/explanation.gif)\n\n## Supported annotation types\n* **LTRB**(xmin, ymin, xmax, ymax)\n* **QUAD**(x1, y1, x2, y2, x3, y3, x4, y4)\n* **POLY**(x1, y1, x2, y2, ..., x_2n, y_2n)\n\n## Supported datasets\n* ICDAR 2013 Focused Scene Text [Link](https://rrc.cvc.uab.es/?ch=2)\n* ICDAR 2015 Incidental Scene Text [Link](https://rrc.cvc.uab.es/?ch=4)\n* TotalText [Link](https://github.com/cs-chan/Total-Text-Dataset)\n* Any other datasets that have a similar format with the datasets mentioned above\n\n## Installation\n\n### Build from pip\ndownload from Clova OCR pypi\n```bash\n$ pip install cleval\n```\n\nor build with url\n```bash\n$ pip install git+https://github.com/clovaai/CLEval.git --user\n```\n\n### Build from source\n\n```bash\n$ git clone https://github.com/clovaai/CLEval.git\n$ cd cleval\n$ python setup.py install --user\n```\n\n## How to use\nYou can replace `cleval` with `PYTHONPATH=$PWD python cleval/main.py` for evaluation using source.\n```bash\n$ PYTHONPATH=$PWD python cleval/main.py -g=gt/gt_IC13.zip -s=[result.zip] --BOX_TYPE=LTRB \n```\n\n### Detection evaluation (CLI)\n```bash\n$ cleval -g=gt/gt_IC13.zip -s=[result.zip] --BOX_TYPE=LTRB          # IC13\n$ cleval -g=gt/gt_IC15.zip -s=[result.zip]                          # IC15\n$ cleval -g=gt/gt_TotalText.zip -s=[result.zip] --BOX_TYPE=POLY     # TotalText\n```\n* Notes\n  * The default value of ```BOX_TYPE``` is set to ```QUAD```. It can be explicitly set to ```--BOX_TYPE=QUAD``` when running evaluation on IC15 dataset.\n  * Add ```--TANSCRIPTION``` option if the result file contains transcription.\n  * Add ```--CONFIDENCES``` option if the result file contains confidence.\n\n### End-to-end evaluation (CLI)\n```bash\n$ cleval -g=gt/gt_IC13.zip -s=[result.zip] --E2E --BOX_TYPE=LTRB        # IC13\n$ cleval -g=gt/gt_IC15.zip -s=[result.zip] --E2E                        # IC15\n$ cleval -g=gt/gt_TotalText.zip -s=[result.zip] --E2E --BOX_TYPE=POLY   # TotalText\n```\n* Notes\n  * Adding ```--E2E``` also automatically adds ```--TANSCRIPTION``` option. Make sure that the transcriptions are included in the result file.  \n  * Add ```--CONFIDENCES``` option if the result file contains confidence.\n\n### TorchMetric\n```python\nfrom cleval import CLEvalMetric\nmetric = CLEvalMetric()\n\nfor gt, det in zip(gts, dets):\n    # your fancy algorithm\n    # ...\n    # gt_quads = ...\n    # det_quads = ...\n    # ...\n    _ = metric(det_quads, gt_quads, det_letters, gt_letters, gt_is_dcs)\n\nmetric_out = metric.compute()\nmetric.reset()\n```\n\n### Profiling\n```bash\n$ cleval -g=resources/test_data/gt/gt_eval_doc_v1_kr_single.zip -s=resources/test_data/pred/res_eval_doc_v1_kr_single.zip --E2E -v --DEBUG --PPROFILE > profile.txt\n$ PYTHONPATH=$PWD python cleval/main.py -g resources/test_data/gt/dummy_dataset_val.json -s resources/test_data/pred/dummy_dataset_val.json --SCALE_WISE --DOMAIN_WISE --ORIENTATION --E2E --ORIENTATION -v --PROFILE --DEBUG > profile.txt\n```\n\n### Paramters for evaluation script\n| name | type | default | description |\n| ---- | ---- | ------- | ---- |\n| -g | ```string``` | | path to ground truth zip file |\n| -s | ```string``` | | path to result zip file |\n| -o | ```string``` | | path to save per-sample result file 'results.zip' |\n\n| name | type | default | description |\n| ---- | ---- | ------- | ---- |\n| --BOX_TYPE | ```string``` | ```QUAD``` | annotation type of box (LTRB, QUAD, POLY) |\n| --TRANSCRIPTION | ```boolean``` | ```False``` | set True if result file has transcription |\n| --CONFIDENCES | ```boolean``` | ```False``` | set True if result file has confidence |\n| --E2E | ```boolean``` | ```False``` | to measure end-to-end evaluation (if not, detection evalution only) |\n| --CASE_SENSITIVE | ```boolean``` | ```True``` | set True to evaluate case-sensitively. (only used in end-to-end evaluation) |\n* Note : Please refer to ```arg_parser.py``` file for additional parameters and default settings used internally.\n\n* Note : For scalewise evaluation, we measure the ratio of the shorter length (text height) of the text-box to the longer length of the image. \nThrough this, evaluation for each ratio can be performed. To adjust the scales, please use SCALE_BINS argument.\n\n## Citation\n```\n@article{baek2020cleval,\n  title={CLEval: Character-Level Evaluation for Text Detection and Recognition Tasks},\n  author={Youngmin Baek, Daehyun Nam, Sungrae Park, Junyeop Lee, Seung Shin, Jeonghun Baek, Chae Young Lee and Hwalsuk Lee},\n  journal={arXiv preprint arXiv:2006.06244},\n  year={2020}\n}\n```\n\n## Contact us\nCLEval has been proposed to make fair evaluation in the OCR community, so we want to hear from many researchers. We welcome any feedbacks to our metric, and appreciate pull requests if you have any comments or improvements.\n\n## License\n```\nCopyright (c) 2020-present NAVER Corp.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n```\n\n### Contribute\nPlease use pre-commit which uses Black and Isort.\n```\n$ pip install pre-commit\n$ pre-commit install\n```\n\n##### Step By Step\n1. Write an issue.\n2. Match code style (black, isort)\n3. Wirte test code.\n4. Delete branch after Squash&Merge.\n\nRequired Approve: 1\n\n## Code Maintainer\n- Donghyun Kim (artit.anthony@gmail.com)\n"
  },
  {
    "path": "cleval/__init__.py",
    "content": "from .torchmetric import CLEvalMetric\n\n__version__ = [\"0.1.1\"]\n\n__all__ = [\"CLEvalMetric\"]\n"
  },
  {
    "path": "cleval/arg_parser.py",
    "content": "import argparse\nimport os\n\nfrom cleval.utils import cpu_count\n\n\ndef str2bool(v):\n    if isinstance(v, bool):\n        return v\n    if v.lower() in (\"yes\", \"true\", \"t\", \"y\", \"1\"):\n        return True\n    elif v.lower() in (\"no\", \"false\", \"f\", \"n\", \"0\"):\n        return False\n    else:\n        raise argparse.ArgumentTypeError(\"Boolean value expected.\")\n\n\ndef get_params():\n    parser = argparse.ArgumentParser(description=\"test global argument parser\")\n\n    # script parameters\n    parser.add_argument(\"-g\", \"--GT_PATHS\", nargs=\"+\", help=\"Path of the Ground Truth files.\")\n    parser.add_argument(\"-s\", \"--SUBMIT_PATHS\", nargs=\"+\", help=\"Path of your method's results file.\")\n\n    # webserver parameters\n    parser.add_argument(\n        \"-o\",\n        \"--OUTPUT_PATH\",\n        default=\"output/\",\n        help=\"Path to a directory where to copy the file that\" \" contains per-sample results.\",\n    )\n    parser.add_argument(\"--DUMP_SAMPLE_RESULT\", action=\"store_true\")\n    parser.add_argument(\"-p\", \"--PORT\", default=8080, help=\"port number to show\")\n\n    # result format related parameters\n    parser.add_argument(\"--BOX_TYPE\", default=\"QUAD\", choices=[\"LTRB\", \"QUAD\", \"POLY\"])\n    parser.add_argument(\"--TRANSCRIPTION\", action=\"store_true\")\n    parser.add_argument(\"--CONFIDENCES\", action=\"store_true\")\n    parser.add_argument(\"--CRLF\", action=\"store_true\")\n\n    # end-to-end related parameters\n    parser.add_argument(\"--E2E\", action=\"store_true\")\n    parser.add_argument(\"--CASE_SENSITIVE\", default=True, type=str2bool)\n    parser.add_argument(\"--RECOG_SCORE\", default=True, type=str2bool)\n\n    # evaluation related parameters\n    parser.add_argument(\"--AREA_PRECISION_CONSTRAINT\", type=float, default=0.3)\n    parser.add_argument(\"--RECALL_GRANULARITY_PENALTY_WEIGHT\", type=float, default=1.0)\n    parser.add_argument(\"--PRECISION_GRANULARITY_PENALTY_WEIGHT\", type=float, default=1.0)\n    parser.add_argument(\"--VERTICAL_ASPECT_RATIO_THRESH\", default=0.5)\n\n    # orientation evaluation\n    parser.add_argument(\"--ORIENTATION\", action=\"store_true\")\n\n    # scale-wise evaluation  (\n    parser.add_argument(\"--SCALE_WISE\", action=\"store_true\")  # scale-wise evaluation\n    parser.add_argument(\"--SCALE_BINS\", default=(0.0, 0.005, 0.01, 0.015, 0.02, 0.025, 0.1, 0.5, 1.0))\n\n    # other parameters\n    parser.add_argument(\"-t\", \"--NUM_WORKERS\", default=-1, type=int, help=\"number of threads to use\")\n    parser.add_argument(\n        \"-v\",\n        \"--VERBOSE\",\n        default=False,\n        action=\"store_true\",\n        help=\"print evaluation progress or not\",\n    )\n    parser.add_argument(\"--DEBUG\", action=\"store_true\")\n    parser.add_argument(\"--PROFILE\", action=\"store_true\")\n\n    args = parser.parse_args()\n    assert len(args.GT_PATHS) == len(args.SUBMIT_PATHS) == 1\n\n    if args.NUM_WORKERS == -1:\n        args.NUM_WORKERS = cpu_count()\n\n    # We suppose there always exist transcription information on end-to-end evaluation\n    if args.E2E:\n        args.TRANSCRIPTION = True\n\n    os.makedirs(args.OUTPUT_PATH, exist_ok=True)\n    return args\n\n\nif __name__ == \"__main__\":\n    from pprint import pprint\n\n    args = get_params()\n    pprint(args)\n"
  },
  {
    "path": "cleval/box_types.py",
    "content": "import abc\nimport math\n\nimport cv2\nimport numpy as np\nimport Polygon as polygon3\nfrom shapely.geometry import Point\nfrom shapely.geometry import Polygon as shapely_poly\n\nMAX_FIDUCIAL_POINTS = 50\n\n\ndef get_midpoints(p1, p2):\n    return (p1[0] + p2[0]) / 2, (p1[1] + p2[1]) / 2\n\n\ndef point_distance(p1, p2):\n    distx = math.fabs(p1[0] - p2[0])\n    disty = math.fabs(p1[1] - p2[1])\n    return math.sqrt(distx * distx + disty * disty)\n\n\nclass Box(metaclass=abc.ABCMeta):\n    def __init__(\n        self,\n        points,\n        confidence,\n        transcription,\n        orientation=None,\n        is_dc=None,\n    ):\n        self.points = points\n        self.confidence = confidence\n        self.transcription = transcription\n        self.orientation = orientation\n        self.is_dc = transcription == \"###\" if is_dc is None else is_dc\n\n    @abc.abstractmethod\n    def __and__(self, other) -> float:\n        \"\"\"Returns intersection between two objects\"\"\"\n        pass\n\n    @abc.abstractmethod\n    def subtract(self, other):\n        \"\"\"polygon subtraction\"\"\"\n        pass\n\n    @abc.abstractmethod\n    def center(self):\n        pass\n\n    @abc.abstractmethod\n    def center_distance(self, other):\n        \"\"\"center distance between each box\"\"\"\n\n    @abc.abstractmethod\n    def diagonal_length(self) -> float:\n        \"\"\"Returns diagonal length for box-level\"\"\"\n        pass\n\n    @abc.abstractmethod\n    def is_inside(self, x, y) -> bool:\n        \"\"\"Returns point (x, y) is inside polygon.\"\"\"\n        pass\n\n    @abc.abstractmethod\n    def make_polygon_obj(self):\n        # TODO: docstring 좀 더 자세히 적기\n        \"\"\"Make polygon object to calculate for future\"\"\"\n        pass\n\n    @abc.abstractmethod\n    def pseudo_character_center(self, *args) -> list:\n        \"\"\"get character level boxes for TedEval pseudo center\"\"\"\n        pass\n\n\nclass QUAD(Box):\n    \"\"\"Points should be x1,y1,...,x4,y4 (8 points) format\"\"\"\n\n    def __init__(\n        self,\n        points,\n        confidence=0.0,\n        transcription=\"\",\n        orientation=None,\n        is_dc=None,\n        scale=None,\n    ):\n        super().__init__(points, confidence, transcription, orientation, is_dc)\n        self.polygon = self.make_polygon_obj()\n        self.scale = scale\n        if self.is_dc:\n            self.transcription = \"#\" * self.pseudo_transcription_length()\n\n    def __and__(self, other) -> float:\n        \"\"\"Get intersection between two area\"\"\"\n        poly_intersect = self.polygon & other.polygon\n        if len(poly_intersect) == 0:\n            return 0.0\n        return poly_intersect.area()\n\n    def subtract(self, other):\n        self.polygon = self.polygon - other.polygon\n\n    def center(self):\n        return self.polygon.center()\n\n    def center_distance(self, other):\n        return point_distance(self.center(), other.center())\n\n    def area(self):\n        return self.polygon.area()\n\n    def __or__(self, other):\n        return self.polygon.area() + other.polygon.area() - (self & other)\n\n    def make_polygon_obj(self):\n        point_matrix = np.empty((4, 2), np.int32)\n        point_matrix[0][0] = int(self.points[0])\n        point_matrix[0][1] = int(self.points[1])\n        point_matrix[1][0] = int(self.points[2])\n        point_matrix[1][1] = int(self.points[3])\n        point_matrix[2][0] = int(self.points[4])\n        point_matrix[2][1] = int(self.points[5])\n        point_matrix[3][0] = int(self.points[6])\n        point_matrix[3][1] = int(self.points[7])\n        return polygon3.Polygon(point_matrix)\n\n    def aspect_ratio(self):\n        top_side = point_distance((self.points[0], self.points[1]), (self.points[2], self.points[3]))\n        right_side = point_distance((self.points[2], self.points[3]), (self.points[4], self.points[5]))\n        bottom_side = point_distance((self.points[4], self.points[5]), (self.points[6], self.points[7]))\n        left_side = point_distance((self.points[6], self.points[7]), (self.points[0], self.points[1]))\n        avg_hor = (top_side + bottom_side) / 2\n        avg_ver = (right_side + left_side) / 2\n\n        return (avg_hor + 1e-5) / (avg_ver + 1e-5)\n\n    def pseudo_transcription_length(self):\n        return min(round(0.5 + (max(self.aspect_ratio(), 1 / self.aspect_ratio()))), 10)\n\n    def pseudo_character_center(self, vertical_aspect_ratio_threshold):\n        chars = list()\n        length = len(self.transcription)\n        aspect_ratio = self.aspect_ratio()\n\n        if length == 0:\n            return chars\n\n        if aspect_ratio >= vertical_aspect_ratio_threshold:\n            left_top = self.points[0], self.points[1]\n            right_top = self.points[2], self.points[3]\n            right_bottom = self.points[4], self.points[5]\n            left_bottom = self.points[6], self.points[7]\n        else:\n            left_top = self.points[6], self.points[7]\n            right_top = self.points[0], self.points[1]\n            right_bottom = self.points[2], self.points[3]\n            left_bottom = self.points[4], self.points[5]\n\n        p1 = get_midpoints(left_top, left_bottom)\n        p2 = get_midpoints(right_top, right_bottom)\n\n        unit_x = (p2[0] - p1[0]) / length\n        unit_y = (p2[1] - p1[1]) / length\n\n        for i in range(length):\n            x = p1[0] + unit_x / 2 + unit_x * i\n            y = p1[1] + unit_y / 2 + unit_y * i\n            chars.append((x, y))\n        return chars\n\n    def diagonal_length(self) -> float:\n        left_top = self.points[0], self.points[1]\n        right_top = self.points[2], self.points[3]\n        right_bottom = self.points[4], self.points[5]\n        left_bottom = self.points[6], self.points[7]\n        diag1 = point_distance(left_top, right_bottom)\n        diag2 = point_distance(right_top, left_bottom)\n        return (diag1 + diag2) / 2\n\n    def is_inside(self, x, y) -> bool:\n        return self.polygon.isInside(x, y)\n\n\nclass POLY(Box):\n    \"\"\"Points should be x1,y1,...,xn,yn (2*n points) format\"\"\"\n\n    def __init__(self, points, confidence=0.0, transcription=\"\", orientation=None, is_dc=None):\n        super().__init__(points, confidence, transcription, orientation, is_dc)\n        self.num_points = len(self.points) // 2\n        self.polygon = self.make_polygon_obj()\n        self._aspect_ratio = self.make_aspect_ratio()\n        if self.is_dc:\n            self.transcription = \"#\" * self.pseudo_transcription_length()\n\n    def __and__(self, other):\n        \"\"\"Get intersection between two area\"\"\"\n        poly_intersect = self.polygon.intersection(other.polygon)\n        return poly_intersect.area\n\n    def subtract(self, other):\n        \"\"\"get substraction\"\"\"\n        self.polygon = self.polygon.difference(self.polygon.intersection(other.polygon))\n\n    def __or__(self, other):\n        return 1.0\n\n    def area(self):\n        return self.polygon.area\n\n    def center(self):\n        return self.polygon.centroid.coords[0]\n\n    def center_distance(self, other):\n        try:\n            return point_distance(self.center(), other.center())\n        except:\n            return 0.0001\n\n    def diagonal_length(self):\n        left_top = self.points[0], self.points[1]\n        right_top = self.points[self.num_points - 2], self.points[self.num_points - 1]\n        right_bottom = self.points[self.num_points], self.points[self.num_points + 1]\n        left_bottom = (\n            self.points[self.num_points * 2 - 2],\n            self.points[self.num_points * 2 - 1],\n        )\n\n        diag1 = point_distance(left_top, right_bottom)\n        diag2 = point_distance(right_top, left_bottom)\n\n        return (diag1 + diag2) / 2\n\n    def is_inside(self, x, y) -> bool:\n        return self.polygon.contains(Point(x, y))\n\n    def check_corner_points_are_continuous(self, lt, rt, rb, lb):\n        counter = 0\n        while lt != rt:\n            lt = (lt + 1) % self.num_points\n            counter += 1\n\n        while rb != lb:\n            rb = (rb + 1) % self.num_points\n            counter += 1\n\n        return True\n\n    def get_four_max_distance_from_center(self):\n        center_x, center_y = self.center()\n        distance_from_center = list()\n        point_x = self.points[0::2]\n        point_y = self.points[1::2]\n\n        for px, py in zip(point_x, point_y):\n            distance_from_center.append(point_distance((center_x, center_y), (px, py)))\n\n        distance_idx_max_order = np.argsort(distance_from_center)[::-1]\n        return distance_idx_max_order[:4]\n\n    def make_polygon_obj(self):\n        point_x = self.points[0::2]\n        point_y = self.points[1::2]\n        # In TotalText dataset, there are under 4 points annotation for Polygon shape.\n        # so, we have to deal with it\n\n        # if points are given 3, fill last quad points with left bottom coordinates\n        if len(point_x) == len(point_y) == 3:\n            point_x.append(point_x[0])\n            point_y.append(point_y[2])\n            self.points.append(point_x[0])\n            self.points.append(point_y[2])\n            self.num_points = len(self.points) // 2\n\n        # if points are given 2, copy value 2 times\n        elif len(point_x) == len(point_y) == 2:\n            point_x *= 2\n            point_y *= 2\n            self.points.append(point_x[1])\n            self.points.append(point_y[0])\n            self.points.append(point_x[0])\n            self.points.append(point_y[1])\n            self.num_points = len(self.points) // 2\n\n        # if points are given 1, copy value 4 times\n        elif len(point_x) == len(point_y) == 1:\n            point_x *= 4\n            point_y *= 4\n            for _ in range(3):\n                self.points.append(point_x[0])\n                self.points.append(point_x[0])\n            self.num_points = len(self.points) // 2\n        return shapely_poly(np.stack([point_x, point_y], axis=1)).buffer(0)\n\n    def aspect_ratio(self):\n        return self._aspect_ratio\n\n    def pseudo_transcription_length(self):\n        return min(round(0.5 + (max(self._aspect_ratio, 1 / self._aspect_ratio))), 10)\n\n    def make_aspect_ratio(self):\n        np.array(np.reshape(self.points, [-1, 2]))\n        rect = cv2.minAreaRect(np.array(np.reshape(self.points, [-1, 2]), dtype=np.float32))\n        width = rect[1][0]\n        height = rect[1][1]\n\n        width += 1e-6\n        height += 1e-6\n\n        return min(10, height / width) + 1e5\n\n    def pseudo_character_center(self):\n        chars = list()\n        length = len(self.transcription)\n\n        # Prepare polygon line estimation with interpolation\n        point_x = self.points[0::2]\n        point_y = self.points[1::2]\n        points_x_top = point_x[: self.num_points // 2]\n        points_x_bottom = point_x[self.num_points // 2 :]\n        points_y_top = point_y[: self.num_points // 2]\n        points_y_bottom = point_y[self.num_points // 2 :]\n\n        # reverse bottom point order from left to right\n        points_x_bottom = points_x_bottom[::-1]\n        points_y_bottom = points_y_bottom[::-1]\n\n        num_interpolation_section = (self.num_points // 2) - 1\n        num_points_to_interpolate = length\n\n        new_point_x_top, new_point_x_bottom = list(), list()\n        new_point_y_top, new_point_y_bottom = list(), list()\n\n        for sec_idx in range(num_interpolation_section):\n            start_x_top, end_x_top = points_x_top[sec_idx], points_x_top[sec_idx + 1]\n            start_y_top, end_y_top = points_y_top[sec_idx], points_y_top[sec_idx + 1]\n            start_x_bottom, end_x_bottom = (\n                points_x_bottom[sec_idx],\n                points_x_bottom[sec_idx + 1],\n            )\n            start_y_bottom, end_y_bottom = (\n                points_y_bottom[sec_idx],\n                points_y_bottom[sec_idx + 1],\n            )\n\n            diff_x_top = (end_x_top - start_x_top) / num_points_to_interpolate\n            diff_y_top = (end_y_top - start_y_top) / num_points_to_interpolate\n            diff_x_bottom = (end_x_bottom - start_x_bottom) / num_points_to_interpolate\n            diff_y_bottom = (end_y_bottom - start_y_bottom) / num_points_to_interpolate\n\n            new_point_x_top.append(start_x_top)\n            new_point_x_bottom.append(start_x_bottom)\n            new_point_y_top.append(start_y_top)\n            new_point_y_bottom.append(start_y_bottom)\n\n            for num_pt in range(1, num_points_to_interpolate):\n                new_point_x_top.append(int(start_x_top + diff_x_top * num_pt))\n                new_point_x_bottom.append(int(start_x_bottom + diff_x_bottom * num_pt))\n                new_point_y_top.append(int(start_y_top + diff_y_top * num_pt))\n                new_point_y_bottom.append(int(start_y_bottom + diff_y_bottom * num_pt))\n        new_point_x_top.append(points_x_top[-1])\n        new_point_y_top.append(points_y_top[-1])\n        new_point_x_bottom.append(points_x_bottom[-1])\n        new_point_y_bottom.append(points_y_bottom[-1])\n\n        len_section_for_single_char = (len(new_point_x_top) - 1) / len(self.transcription)\n\n        for c in range(len(self.transcription)):\n            center_x = (\n                new_point_x_top[int(c * len_section_for_single_char)]\n                + new_point_x_top[int((c + 1) * len_section_for_single_char)]\n                + new_point_x_bottom[int(c * len_section_for_single_char)]\n                + new_point_x_bottom[int((c + 1) * len_section_for_single_char)]\n            ) / 4\n\n            center_y = (\n                new_point_y_top[int(c * len_section_for_single_char)]\n                + new_point_y_top[int((c + 1) * len_section_for_single_char)]\n                + new_point_y_bottom[int(c * len_section_for_single_char)]\n                + new_point_y_bottom[int((c + 1) * len_section_for_single_char)]\n            ) / 4\n\n            chars.append((center_x, center_y))\n        return chars\n"
  },
  {
    "path": "cleval/data.py",
    "content": "from dataclasses import dataclass, field\nfrom typing import Dict, List, Union\n\nfrom cleval.utils import harmonic_mean\n\n\nclass MatchReleation:\n    ONE_TO_ONE = \"one-to-one\"\n    MANY_TO_ONE = \"many-to-one\"\n    ONE_TO_MANY = \"one-to-many\"\n\n\n@dataclass\nclass CoreStats:\n    recall: float = 0.0\n    precision: float = 0.0\n    hmean: float = 0.0\n\n    num_char_gt: int = 0  # TotalNum for Recall\n    num_char_det: int = 0  # TotalNum for Precisiion\n    gran_score_recall: float = 0.0\n    num_char_tp_recall: int = 0\n    gran_score_precision: float = 0.0\n    num_char_tp_precision: int = 0\n\n    num_char_fp: int = 0  # false positive\n\n\n@dataclass\nclass MatchResult:\n    gt_ids: List[int]\n    det_ids: List[int]\n    match_relation: str  # from MatchRelation\n\n    det: CoreStats = field(default_factory=CoreStats)\n    e2e: CoreStats = field(default_factory=CoreStats)\n\n\n@dataclass\nclass Point:\n    x: int\n    y: int\n\n\n@dataclass\nclass GTBoxResult:\n    id: int\n    points: List[Point]\n    pccs: List[Point]\n    orientation: Union[None, str]\n    letters: str\n    is_dc: bool\n\n\n@dataclass\nclass DetBoxResult:\n    id: int\n    points: List[Point]\n    orientation: Union[None, str]\n    letters: str\n\n\n@dataclass\nclass Stats:\n    det: CoreStats = field(default_factory=CoreStats)\n    e2e: CoreStats = field(default_factory=CoreStats)\n\n    # split-merge cases\n    num_splitted: int = 0\n    num_merged: int = 0\n    num_char_overlapped: int = 0\n\n    # orientation evaluation\n    ori_acc: float = 0.0\n    num_ori_total: int = 0\n    num_ori_correct: int = 0\n\n\n@dataclass\nclass SampleResult:\n    matches: List[MatchResult]\n    gts: List[GTBoxResult]\n    preds: List[DetBoxResult]\n    stats: Stats = field(default_factory=Stats)\n    image_id: Union[int, None] = None\n\n\n@dataclass\nclass GlobalResult:\n    \"\"\"Object that holds each record of all samples.\"\"\"\n\n    dataset_inform: Dict = field(default_factory=dict)\n    sample_results: List[SampleResult] = field(default_factory=list)\n    stats: Stats = field(default_factory=Stats)\n\n\ndef accumulate_result(\n    global_res: GlobalResult,\n    sample_res: SampleResult,\n    is_e2e: bool,\n    dump_sample_res: bool = False,\n):\n    if dump_sample_res:\n        global_res.sample_results.append(sample_res)\n    accumulate_stats(global_res.stats, sample_res.stats, is_e2e)\n\n\ndef accumulate_stats(stats1: Stats, stats2: Stats, is_e2e: bool):\n    \"\"\"Accumulate core stats exclude ori_acc.\"\"\"\n    stats1.num_splitted += stats2.num_splitted\n    stats1.num_merged += stats2.num_merged\n    stats1.num_char_overlapped += stats2.num_char_overlapped\n    stats1.num_ori_total += stats2.num_ori_total\n    stats1.num_ori_correct += stats2.num_ori_correct\n\n    accumulate_core_stats(stats1.det, stats2.det)\n    if is_e2e:\n        accumulate_core_stats(stats1.e2e, stats2.e2e)\n\n\ndef accumulate_core_stats(stats1: CoreStats, stats2: CoreStats):\n    \"\"\"Accumulate core stats exclude recall, precision, and hmean.\"\"\"\n    stats1.num_char_gt += stats2.num_char_gt\n    stats1.num_char_det += stats2.num_char_det\n    stats1.gran_score_recall += stats2.gran_score_recall\n    stats1.num_char_tp_recall += stats2.num_char_tp_recall\n    stats1.gran_score_precision += stats2.gran_score_precision\n    stats1.num_char_tp_precision += stats2.num_char_tp_precision\n    stats1.num_char_fp += stats2.num_char_fp\n\n\ndef calculate_global_rph(res: GlobalResult, is_e2e: bool):\n    calculate_rph(res.stats.det)\n    if is_e2e:\n        calculate_rph(res.stats.e2e)\n\n\ndef calculate_rph(stats: CoreStats):\n    total_gt = stats.num_char_gt\n    total_det = stats.num_char_det\n    tp_gt = stats.num_char_tp_recall\n    gran_gt = stats.gran_score_recall\n    tp_det = stats.num_char_tp_precision\n    gran_det = stats.gran_score_precision\n\n    # Sample Score : Character correct length - Granularity Penalty\n    recall = 0.0 if total_gt == 0 else max(0.0, tp_gt - gran_gt) / total_gt\n    precision = 0.0 if total_det == 0 else max(0.0, tp_det - gran_det) / total_det\n    hmean = harmonic_mean(recall, precision)\n    stats.recall = recall\n    stats.precision = precision\n    stats.hmean = hmean\n"
  },
  {
    "path": "cleval/eval_functions.py",
    "content": "from dataclasses import dataclass\nfrom typing import List\n\nimport numpy as np\nfrom numba import njit\nfrom numpy.typing import NDArray\n\nfrom cleval.data import (\n    DetBoxResult,\n    GTBoxResult,\n    MatchReleation,\n    MatchResult,\n    Point,\n    SampleResult,\n)\nfrom cleval.utils import harmonic_mean, lcs\n\n\n@dataclass\nclass EvalMaterial:\n    \"\"\"EvalMaterial Dataclass\n    These are used for calculating eval results.\n    \"\"\"\n\n    gt_pcc_points: List[List]  # [gt_idx][pcc_idx] nested list which has variable length\n    pcc_mat_list: List[NDArray]  # list of pcc_mat which has (len_det, len_pcc) shape.\n    pcc_mat_sum: NDArray[np.int16]  # (len_gt, len_det)\n    ap_mat: NDArray[np.float32]  # (len_gt, len_det)\n    ap_mat_binary: NDArray[bool]  # (len_gt, len_det)\n    ap_constraint: float\n    gt_valid_indices: set\n    det_valid_indices: set\n    len_gt: int\n    len_det: int\n\n\ndef evaluation(args, gt_boxes, det_boxes, scale_range=(0.0, 1.0)):\n    \"\"\"main evaluation function\n\n    Notes:\n        Abbreviations for variable names.\n         - ap: area precision (not average precision)\n         - thresh: threshold\n         - pcc: pseudo char center\n         - mat: matrix\n         - res: result\n         - dc: don't care\n         - fp: false positive\n         - tran: transcription\n    \"\"\"\n    # prepare gt, det\n    gt_dc_indices, gt_pcc_points = prepare_gt(\n        gt_boxes, args.CASE_SENSITIVE, args.VERTICAL_ASPECT_RATIO_THRESH, scale_range\n    )\n    prepare_det(det_boxes, args.CASE_SENSITIVE)\n    len_gt = len(gt_boxes)\n    len_det = len(det_boxes)\n\n    # calc area_precision\n    ap_constraint = args.AREA_PRECISION_CONSTRAINT\n    ap_mat, ap_mat_binary = calc_area_precision(gt_boxes, det_boxes, ap_constraint)\n\n    # calc pcc inclusion\n    pcc_mat_list, pcc_mat_sum = calc_pcc_inclusion(det_boxes, gt_pcc_points)\n\n    # prepare valid indices\n    det_dc_indices = get_det_dc_indices(gt_dc_indices, pcc_mat_sum, ap_mat, ap_mat_binary, ap_constraint, len_det)\n    gt_valid_indices = set(range(len_gt)) - gt_dc_indices\n    det_valid_indices = set(range(len_det)) - det_dc_indices\n\n    # construct eval material\n    eval_material = EvalMaterial(\n        gt_pcc_points,\n        pcc_mat_list,\n        pcc_mat_sum,\n        ap_mat,\n        ap_mat_binary,\n        ap_constraint,\n        gt_valid_indices,\n        det_valid_indices,\n        len_gt,\n        len_det,\n    )\n\n    # Matching process\n    match_mat, match_results = calc_match_matrix(eval_material)\n\n    # Prepare sample_result\n    gt_results, det_results = get_box_results(gt_boxes, gt_pcc_points, det_boxes)\n    sample_res = SampleResult(match_results, gt_results, det_results)\n\n    # Evaluation Process\n    eval_det(args, sample_res, gt_boxes, det_boxes, eval_material, match_mat)\n\n    if args.E2E:\n        eval_e2e(args, sample_res, gt_boxes, det_boxes, eval_material, match_mat)\n\n    if args.ORIENTATION:\n        eval_orientation(sample_res, gt_boxes, det_boxes, gt_valid_indices, match_mat)\n\n    return sample_res\n\n\ndef prepare_gt(gt_boxes, is_case_sensitive, vertical_aspect_ratio_thresh, scale_range):\n    \"\"\"prepare ground-truth boxes in evaluation format.\"\"\"\n    gt_dc_indices = set()  # fast check via using set (hash-table)\n    gt_pcc_points = []\n    for gt_idx, gt_box in enumerate(gt_boxes):\n        if not is_case_sensitive:\n            gt_box.transcription = gt_box.transcription.upper()\n\n        if gt_box.is_dc or (gt_box.scale is not None and not scale_range[0] <= gt_box.scale <= scale_range[1]):\n            gt_dc_indices.add(gt_idx)\n        gt_pcc_point = gt_box.pseudo_character_center(vertical_aspect_ratio_thresh)\n        gt_pcc_points.append(gt_pcc_point)\n\n    # subtract overlapping gt area from don't care boxes\n    # Area(Don't care) - Area(Ground Truth):\n    for dc_idx in gt_dc_indices:\n        for idx in range(len(gt_boxes)):\n            if idx in gt_dc_indices:\n                continue\n            if gt_boxes[idx] & gt_boxes[dc_idx] > 0:\n                # TODO: Consider PCC exclusion for area overlapped with don't care.\n                gt_boxes[dc_idx].subtract(gt_boxes[idx])\n    return gt_dc_indices, gt_pcc_points\n\n\ndef prepare_det(det_boxes, is_case_sensitive):\n    \"\"\"prepare detection results in evaluation format.\"\"\"\n    for det_idx, det_box in enumerate(det_boxes):\n        if not is_case_sensitive:\n            det_box.transcription = det_box.transcription.upper()\n\n\ndef calc_area_precision(gt_boxes, det_boxes, ap_constraint):\n    \"\"\"calculate area precision between each GTbox and DETbox\n    Args:\n        gt_boxes(List[Box]): list of gt boxes\n        det_boxes(List[Box]): list of det boxes\n        ap_constraint(float): area precision contstraint\n\n    Returns:\n        ap_mat(NDArray[float32]): area precision matrix\n        ap_mat_binary(NDArray[bool]): boolean mat that area precision >= ap_constraint\n\n    \"\"\"\n    ap_mat = np.zeros([len(gt_boxes), len(det_boxes)], dtype=np.float32)\n\n    for gt_idx, gt_box in enumerate(gt_boxes):\n        for det_idx, det_box in enumerate(det_boxes):\n            intersected_area = gt_box & det_box\n            det_area = det_box.area()\n            if det_area > 0.0:\n                ap_mat[gt_idx, det_idx] = intersected_area / det_area\n    ap_mat_binary = ap_mat >= ap_constraint\n    return ap_mat, ap_mat_binary\n\n\ndef calc_pcc_inclusion(det_boxes, gt_pcc_points):\n    \"\"\"fill PCC counting matrix by iterating each GTbox and DETbox\"\"\"\n    len_gt = len(gt_pcc_points)\n    len_det = len(det_boxes)\n    pcc_mat_list = []\n    pcc_mat_sum = np.zeros((len_gt, len_det), dtype=np.int16)\n\n    for gt_idx, gt_word_pccs in enumerate(gt_pcc_points):\n        len_pcc = len(gt_word_pccs)\n        pcc_mat = np.zeros((len_det, len_pcc), dtype=bool)\n\n        for det_idx, det_box in enumerate(det_boxes):\n            for pcc_idx, pcc_point in enumerate(gt_word_pccs):\n                if det_box.is_inside(pcc_point[0], pcc_point[1]):\n                    pcc_mat[det_idx, pcc_idx] = True\n                    pcc_mat_sum[gt_idx, det_idx] += 1\n\n        pcc_mat_list.append(pcc_mat)\n    return pcc_mat_list, pcc_mat_sum\n\n\ndef get_det_dc_indices(gt_dc_indices, pcc_mat_sum, ap_mat, ap_mat_binary, ap_constraint, len_det):\n    \"\"\"Filter detection Don't care boxes\"\"\"\n    det_dc_indices = set()\n    if len(gt_dc_indices) > 0:\n        for det_idx in range(len_det):\n            ap_sum = 0\n            for gt_idx in gt_dc_indices:\n                if ap_mat_binary[gt_idx, det_idx]:\n                    det_dc_indices.add(det_idx)\n                    break\n                if pcc_mat_sum[gt_idx, det_idx] > 0:\n                    ap_sum += ap_mat[gt_idx, det_idx]\n            if ap_sum >= ap_constraint:\n                det_dc_indices.add(det_idx)\n    return det_dc_indices\n\n\ndef calc_match_matrix(eval_material):\n    \"\"\"Calculate match matrix with PCC counting matrix information.\"\"\"\n    em = eval_material\n    match_results = []\n    match_mat = np.zeros([em.len_gt, em.len_det], dtype=bool)\n\n    # one-to-one match\n    for gt_idx in em.gt_valid_indices:\n        for det_idx in em.det_valid_indices:\n            is_matched = one_to_one_match(em.pcc_mat_sum, gt_idx, det_idx, em.ap_mat_binary, em.len_gt, em.len_det)\n            if is_matched:\n                match_result = MatchResult(\n                    gt_ids=[gt_idx],\n                    det_ids=[det_idx],\n                    match_relation=MatchReleation.ONE_TO_ONE,\n                )\n                match_results.append(match_result)\n\n    # one-to-many match\n    for gt_idx in em.gt_valid_indices:\n        det_valid_indices_np = np.array(list(em.det_valid_indices), dtype=np.int16)\n        is_matched, matched_det_indices = one_to_many_match(\n            em.pcc_mat_sum, gt_idx, em.ap_mat_binary, det_valid_indices_np\n        )\n        if is_matched:\n            match_result = MatchResult(\n                gt_ids=[gt_idx],\n                det_ids=matched_det_indices,\n                match_relation=MatchReleation.ONE_TO_MANY,\n            )\n            match_results.append(match_result)\n\n    # many-to-one match\n    for det_idx in em.det_valid_indices:\n        gt_valid_indices_np = np.array(list(em.gt_valid_indices), dtype=np.int16)\n        is_matched, matched_gt_indices = many_to_one_match(\n            em.pcc_mat_sum, det_idx, em.ap_mat, em.ap_constraint, gt_valid_indices_np\n        )\n        if is_matched:\n            match_result = MatchResult(\n                gt_ids=matched_gt_indices,\n                det_ids=[det_idx],\n                match_relation=MatchReleation.MANY_TO_ONE,\n            )\n            match_results.append(match_result)\n\n    for match_result in match_results:\n        match_mat[match_result.gt_ids, match_result.det_ids] = True\n\n    # clear pcc count flag for not matched pairs\n    for gt_idx in range(em.len_gt):\n        for det_idx in range(em.len_det):\n            if match_mat[gt_idx, det_idx]:\n                continue\n            for pcc_idx in range(len(em.gt_pcc_points[gt_idx])):\n                em.pcc_mat_sum[gt_idx, det_idx] -= em.pcc_mat_list[gt_idx][det_idx, pcc_idx]\n                em.pcc_mat_list[gt_idx][det_idx, pcc_idx] = 0\n    return match_mat, match_results\n\n\n@njit\ndef one_to_one_match(pcc_mat_sum, gt_idx, det_idx, ap_mat_binary, len_gt, len_det):\n    \"\"\"One-to-One match condition\"\"\"\n    match_counter = 0\n    for i in range(len_det):\n        if ap_mat_binary[gt_idx, i] and pcc_mat_sum[gt_idx, i] > 0:\n            match_counter += 1\n            if match_counter >= 2:\n                break\n    if match_counter != 1:\n        return False\n\n    match_counter = 0\n    for i in range(len_gt):\n        if ap_mat_binary[i, det_idx] and pcc_mat_sum[i, det_idx] > 0:\n            match_counter += 1\n            if match_counter >= 2:\n                break\n    if match_counter != 1:\n        return False\n\n    if ap_mat_binary[gt_idx, det_idx] and pcc_mat_sum[gt_idx, det_idx] > 0:\n        return True\n    return False\n\n\n@njit\ndef one_to_many_match(pcc_mat_sum, gt_idx, ap_mat_binary, det_valid_indices):\n    \"\"\"One-to-Many match condition\"\"\"\n    many_sum = 0\n    matched_det_indices = []\n    for det_idx in det_valid_indices:\n        if ap_mat_binary[gt_idx, det_idx] and pcc_mat_sum[gt_idx, det_idx] > 0:\n            many_sum += pcc_mat_sum[gt_idx, det_idx]\n            matched_det_indices.append(det_idx)\n\n    if many_sum > 0 and len(matched_det_indices) >= 2:\n        return True, matched_det_indices\n    else:\n        return False, matched_det_indices\n\n\n@njit\ndef many_to_one_match(pcc_mat_sum, det_idx, ap_mat, ap_constraint, gt_valid_indices):\n    \"\"\"Many-to-One match condition\"\"\"\n    many_sum = 0\n    matched_gt_indices = []\n    for gt_idx in gt_valid_indices:\n        if pcc_mat_sum[gt_idx, det_idx] > 0:\n            many_sum += ap_mat[gt_idx, det_idx]\n            matched_gt_indices.append(gt_idx)\n    if many_sum >= ap_constraint and len(matched_gt_indices) >= 2:\n        return True, matched_gt_indices\n    else:\n        return False, matched_gt_indices\n\n\ndef get_box_results(gt_boxes, gt_pcc_points, det_boxes):\n    gt_results = []\n    for gt_idx, gt_box in enumerate(gt_boxes):\n        gt = GTBoxResult(\n            id=gt_idx,\n            points=__points_to_result(gt_box.points),\n            pccs=__pccs_to_result(gt_pcc_points[gt_idx]),\n            orientation=gt_box.orientation,\n            letters=gt_box.transcription,\n            is_dc=gt_box.is_dc,\n        )\n        gt_results.append(gt)\n\n    det_results = []\n    for det_idx, det_box in enumerate(det_boxes):\n        det = DetBoxResult(\n            id=det_idx,\n            points=__points_to_result(det_box.points),\n            orientation=det_box.orientation,\n            letters=det_box.transcription,\n        )\n        det_results.append(det)\n\n    return gt_results, det_results\n\n\ndef __points_to_result(points):\n    points = np.array(points, dtype=np.int16).reshape(-1, 2)\n    new_points = [Point(int(round(pt[0])), int(round(pt[1]))) for pt in points]\n    return new_points\n\n\ndef __pccs_to_result(pcc_points):\n    return [Point(int(round(pt[0])), int(round(pt[1]))) for pt in pcc_points]\n\n\ndef eval_det(args, sample_res, gt_boxes, det_boxes, eval_material, match_mat):\n    stats = sample_res.stats\n    em = eval_material\n\n    # res_mat has +2 size for granuarity penalty and summation of matrix\n    res_mat = np.zeros([em.len_gt + 2, em.len_det + 2], dtype=np.float32)\n\n    match_mat_gts_sum = match_mat.sum(axis=0)\n    match_mat_dets_sum = match_mat.sum(axis=1)\n    pcc_checked = [np.zeros(len(pccs), dtype=bool) for pccs in em.gt_pcc_points]\n\n    # Precision score\n    for det_idx in em.det_valid_indices:\n        if match_mat_gts_sum[det_idx] > 0:\n            matched_gt_indices = np.where(match_mat[:, det_idx])[0]\n            if len(matched_gt_indices) > 1:\n                stats.num_merged += 1\n\n            for gt_idx in matched_gt_indices:\n                pcc_indices = np.where(em.pcc_mat_list[gt_idx][det_idx])[0]\n                for pcc_idx in pcc_indices:\n                    if not pcc_checked[gt_idx][pcc_idx]:\n                        pcc_checked[gt_idx][pcc_idx] = True\n                        res_mat[-2, det_idx] += 1  # for total score\n                        res_mat[gt_idx, det_idx] += 1\n                    else:\n                        stats.num_char_overlapped += 1\n            gran_weight = args.PRECISION_GRANULARITY_PENALTY_WEIGHT\n            res_mat[-1, det_idx] = get_gran_score(len(matched_gt_indices), gran_weight)\n\n    # Recall score\n    for gt_idx in em.gt_valid_indices:\n        found_gt_chars = 0\n        if match_mat_dets_sum[gt_idx] > 0:\n            matched_det_indices = np.where(match_mat[gt_idx] > 0)[0]\n            if len(matched_det_indices) > 1:\n                stats.num_splitted += 1\n\n            found_gt_chars = np.sum(pcc_checked[gt_idx])\n            gran_weight = args.RECALL_GRANULARITY_PENALTY_WEIGHT\n            res_mat[gt_idx, -1] = get_gran_score(len(matched_det_indices), gran_weight)\n        res_mat[gt_idx, -2] = found_gt_chars\n\n    # Calculate precision / recall\n    num_char_gt, num_char_det = get_num_total_char(gt_boxes, em.pcc_mat_sum, em.gt_valid_indices, em.det_valid_indices)\n    num_char_fp = get_num_fp_char(det_boxes, em.det_valid_indices, match_mat_gts_sum)\n    num_char_det += num_char_fp\n    extract_stats(sample_res.stats.det, num_char_fp, num_char_gt, num_char_det, res_mat)\n\n    # Calculate match-wise eval out\n    if args.DUMP_SAMPLE_RESULT:\n        for match_res in sample_res.matches:\n            gt_ids = match_res.gt_ids\n            det_ids = match_res.det_ids\n            num_char_gt, num_char_det = get_num_total_char(gt_boxes, em.pcc_mat_sum, gt_ids, det_ids)\n            num_char_fp = get_num_fp_char(det_boxes, det_ids, match_mat_gts_sum)\n            num_char_det += num_char_fp\n            extract_stats(match_res.det, num_char_fp, num_char_gt, num_char_det, res_mat)\n\n\ndef get_num_total_char(gt_boxes, pcc_mat_sum, gt_valid_indices, det_valid_indices):\n    \"\"\"get TotalNum for detection evaluation.\"\"\"\n    num_char_gt = 0\n    num_char_det = 0\n    for gt_idx, gt_box in enumerate(gt_boxes):\n        if gt_idx in gt_valid_indices:\n            num_char_gt += len(gt_box.transcription)\n        num_char_det += np.sum(pcc_mat_sum[gt_idx][list(det_valid_indices)])\n    return num_char_gt, num_char_det\n\n\ndef get_num_fp_char(det_boxes, det_valid_indices, match_mat_gts_sum):\n    \"\"\"get FalsePositive for detection evaluation.\"\"\"\n    fp_char_counts = 0\n    for det_idx in det_valid_indices:\n        # no match with any GTs && not matched with don't care\n        if match_mat_gts_sum[det_idx] == 0:\n            fp_char_count = round(0.5 + 1 / (1e-5 + det_boxes[det_idx].aspect_ratio()))\n            fp_char_counts += min(fp_char_count, 10)\n    return fp_char_counts\n\n\ndef eval_e2e(args, sample_res, gt_boxes, det_boxes, eval_material, match_mat):\n    gt_trans = [box.transcription for box in gt_boxes]\n    det_trans = [box.transcription for box in det_boxes]\n    gt_trans_not_found = [box.transcription for box in gt_boxes]\n    det_trans_not_found = [box.transcription for box in det_boxes]\n\n    em = eval_material\n    stats = sample_res.stats\n\n    # +2 size for granuarity penalty and summation of matrix\n    res_mat = np.zeros([em.len_gt + 2, em.len_det + 2], dtype=np.float32)\n\n    match_mat_gts_sum = match_mat.sum(axis=0)\n    match_mat_dets_sum = match_mat.sum(axis=1)\n\n    # Recall score\n    for gt_idx in em.gt_valid_indices:\n        if match_mat_dets_sum[gt_idx] > 0:\n            matched_det_indices = np.where(match_mat[gt_idx])[0]\n            sorted_det_indices = sort_detbox_order_by_pcc(\n                gt_idx, matched_det_indices, em.gt_pcc_points, em.pcc_mat_list\n            )\n            corrected_num_chars = lcs_elimination(\n                gt_trans,\n                gt_trans_not_found,\n                det_trans_not_found,\n                gt_idx,\n                sorted_det_indices,\n            )\n            res_mat[gt_idx, -2] = corrected_num_chars\n            gran_weight = args.RECALL_GRANULARITY_PENALTY_WEIGHT\n            res_mat[gt_idx, -1] = get_gran_score(len(matched_det_indices), gran_weight)\n\n    # Precision score\n    for det_idx in em.det_valid_indices:\n        if match_mat_gts_sum[det_idx] > 0:\n            matched_gt_indices = np.where(match_mat[:, det_idx])[0]\n            gran_weight = args.PRECISION_GRANULARITY_PENALTY_WEIGHT\n            res_mat[-1, det_idx] = get_gran_score(len(matched_gt_indices), gran_weight)\n        res_mat[-2, det_idx] = len(det_trans[det_idx]) - len(det_trans_not_found[det_idx])\n\n    num_char_det = sum([len(det_trans[i]) for i in em.det_valid_indices])\n    num_char_fp = num_char_det - np.sum(res_mat[-2])\n    extract_stats(stats.e2e, num_char_fp, stats.det.num_char_gt, num_char_det, res_mat)\n\n    if args.DUMP_SAMPLE_RESULT:\n        for match_res in sample_res.matches:\n            det_ids = match_res.det_ids\n            num_char_det = sum([len(det_trans[i]) for i in det_ids])\n            num_char_fp = num_char_det - np.sum(res_mat[-2][det_ids])\n            num_char_gt = match_res.det.num_char_gt\n            extract_stats(match_res.e2e, num_char_fp, num_char_gt, num_char_det, res_mat)\n\n\ndef sort_detbox_order_by_pcc(gt_idx, matched_det_indices, gt_pcc_points, pcc_mat_list):\n    \"\"\"sort detected box order by pcc information.\"\"\"\n    unordered = matched_det_indices.tolist()  # deepcopy\n    ordered_indices = []\n\n    char_len = len(gt_pcc_points[gt_idx])\n    for pcc_idx in range(char_len):\n        if len(unordered) == 1:\n            break\n\n        for det_idx in unordered:\n            if pcc_mat_list[gt_idx][det_idx, pcc_idx]:\n                ordered_indices.append(det_idx)\n                unordered.remove(det_idx)\n                break\n\n    ordered_indices.append(unordered[0])\n    return ordered_indices\n\n\ndef lcs_elimination(gt_trans, gt_trans_not_found, det_trans_not_found, gt_idx, sorted_det_indices):\n    \"\"\"longest common sequence elimination by sorted detection boxes\"\"\"\n    target_string = \"\".join(det_trans_not_found[i] for i in sorted_det_indices)\n    lcs_length, lcs_string = lcs(gt_trans[gt_idx], target_string)\n\n    for char in lcs_string:\n        gt_trans_not_found[gt_idx] = gt_trans_not_found[gt_idx].replace(char, \"\", 1)\n\n        for det_idx in sorted_det_indices:\n            det_tran = det_trans_not_found[det_idx]\n            if not det_tran.find(char) < 0:\n                det_trans_not_found[det_idx] = det_tran.replace(char, \"\", 1)\n                break\n    return lcs_length\n\n\ndef eval_orientation(sample_res, gt_boxes, det_boxes, gt_valid_indices, match_mat):\n    gt_query = [box.orientation for box in gt_boxes]\n    det_query = [box.orientation for box in det_boxes]\n\n    match_mat_dets_sum = match_mat.sum(axis=1)\n    counter = 0\n    num_ori_correct = 0\n    stats = sample_res.stats\n\n    for gt_idx in gt_valid_indices:\n        if match_mat_dets_sum[gt_idx] > 0:\n            matched_det_indices = np.where(match_mat[gt_idx])[0]\n            counter += 1\n            count_size = 0 if len(matched_det_indices) else 1 / len(matched_det_indices)\n            for det_idx in matched_det_indices:\n                if gt_query[gt_idx] == det_query[det_idx]:\n                    num_ori_correct += count_size\n    if counter != 0:\n        stats.num_ori_total = counter\n        stats.num_ori_correct = num_ori_correct\n        stats.ori_acc = num_ori_correct / counter\n\n\ndef extract_stats(core_stats, num_char_fp, num_char_gt, num_char_det, res_mat):\n    core_stats.num_char_fp = int(num_char_fp)\n    core_stats.num_char_gt = total_gt = int(num_char_gt)\n    core_stats.num_char_det = total_det = int(num_char_det)\n    core_stats.num_char_tp_recall = tp_gt = int(np.sum(res_mat[-2]))\n    core_stats.gran_score_recall = gran_gt = float(np.sum(res_mat[:, -1]))\n    core_stats.num_char_tp_precision = tp_det = int(np.sum(res_mat[-2]))\n    core_stats.gran_score_precision = gran_det = float(np.sum(res_mat[-1]))\n\n    # Sample Score : Character correct length - Granularity Penalty\n    recall = 0.0 if total_gt == 0 else max(0.0, tp_gt - gran_gt) / total_gt\n    precision = 0.0 if total_det == 0 else max(0.0, tp_det - gran_det) / total_det\n    hmean = harmonic_mean(recall, precision)\n    core_stats.recall = recall\n    core_stats.precision = precision\n    core_stats.hmean = hmean\n\n\n@njit\ndef get_gran_score(num_splitted, penalty_weight):\n    \"\"\"get granularity penalty given number of how many splitted\"\"\"\n    return max(num_splitted - 1, 0) * penalty_weight\n"
  },
  {
    "path": "cleval/main.py",
    "content": "import os\nimport re\nimport time\nfrom concurrent.futures import ProcessPoolExecutor, as_completed\nfrom dataclasses import asdict\nfrom pprint import pprint\n\nfrom tqdm import tqdm\n\nfrom cleval.arg_parser import get_params\nfrom cleval.box_types import POLY, QUAD, Box\nfrom cleval.data import GlobalResult, accumulate_result, calculate_global_rph\nfrom cleval.eval_functions import evaluation\nfrom cleval.utils import (\n    convert_ltrb2quad,\n    decode_utf8,\n    dump_json,\n    load_zip_file,\n    ltrb_regex_match,\n    quad_regex_match,\n)\nfrom cleval.validation import (\n    validate_data,\n    validate_min_max_bounds,\n    validate_point_inside_bounds,\n)\n\n\ndef main():\n    \"\"\"Also used by cli\"\"\"\n    start_t = time.perf_counter()\n    args = get_params()\n\n    if args.PROFILE:\n        assert args.DEBUG, \"DEBUG mode should be turned on for PPROFILE.\"\n        import pprofile\n\n        prof = pprofile.Profile()\n        with prof():\n            res_dict = cleval(args)\n        prof.print_stats()\n    else:\n        res_dict = cleval(args)\n    end_t = time.perf_counter()\n    print(f\"CLEval total duration...{end_t - start_t}s\")\n    pprint(res_dict)\n\n\ndef cleval(args):\n    \"\"\"This process validates a method, evaluates it.\n    If it succeeds, generates a ZIP file with a JSON entry for each sample.\n    \"\"\"\n    validate_data(args.GT_PATHS[0], args.SUBMIT_PATHS[0], args.CRLF)\n\n    global_res = GlobalResult()\n    gt_zipfile = args.GT_PATHS[0]\n    submit_zipfile = args.SUBMIT_PATHS[0]\n    gt_files, det_files, file_indices = get_file_paths(gt_zipfile, submit_zipfile)\n\n    with tqdm(total=len(gt_files), disable=not args.VERBOSE) as pbar:\n        pbar.set_description(\"Integrating results...\")\n        if args.DEBUG or args.NUM_WORKERS <= 1:\n            for gt_file, det_file, file_idx in zip(gt_files, det_files, file_indices):\n                sample_res = eval_single(args, gt_file, det_file, file_idx)\n                accumulate_result(global_res, sample_res, args.E2E, args.DUMP_SAMPLE_RESULT)\n                pbar.update(1)\n        else:\n            futures = []\n            executor = ProcessPoolExecutor(max_workers=args.NUM_WORKERS)\n            for gt_file, det_file, file_idx in zip(gt_files, det_files, file_indices):\n                future = executor.submit(eval_single, args, gt_file, det_file, file_idx)\n                futures.append(future)\n\n            for future in as_completed(futures):\n                sample_res = future.result()\n                accumulate_result(global_res, sample_res, args.E2E, args.DUMP_SAMPLE_RESULT)\n                pbar.update(1)\n\n            executor.shutdown()\n\n    # Calculate global recall, precision, hmean after accumulate all sample-results.\n    calculate_global_rph(global_res, args.E2E)\n\n    res_dict = {\"all\": asdict(global_res.stats)}\n    dump_path = os.path.join(args.OUTPUT_PATH, \"results.json\")\n    dump_json(dump_path, res_dict)\n\n    if args.DUMP_SAMPLE_RESULT:\n        dump_path = os.path.join(args.OUTPUT_PATH, f\"sample_wise.json\")\n        dump_json(dump_path, asdict(global_res))\n\n    if args.VERBOSE:\n        pprint(\"Calculated!\")\n        pprint(res_dict)\n\n    return res_dict\n\n\ndef get_file_paths(gt_zipfile, submit_zipfile):\n    gt_zipfile_loaded = load_zip_file(gt_zipfile)\n    submission_zipfile_loaded = load_zip_file(submit_zipfile)\n    gt_files, det_files, file_indices = [], [], []\n\n    for file_idx in gt_zipfile_loaded:\n        gt_file = decode_utf8(gt_zipfile_loaded[file_idx])\n\n        if file_idx in submission_zipfile_loaded:\n            det_file = decode_utf8(submission_zipfile_loaded[file_idx])\n            if det_file is None:\n                det_file = \"\"\n        else:\n            det_file = \"\"\n\n        gt_files.append(gt_file)\n        det_files.append(det_file)\n        file_indices.append(file_idx)\n    return gt_files, det_files, file_indices\n\n\ndef eval_single(args, gt_file, det_file, file_id):\n    gt_boxes = parse_single_file(gt_file, args.CRLF, True, False, box_type=args.BOX_TYPE)\n    det_boxes = parse_single_file(\n        det_file,\n        args.CRLF,\n        args.TRANSCRIPTION,\n        args.CONFIDENCES,\n        box_type=args.BOX_TYPE,\n    )\n    sample_res = evaluation(args, gt_boxes, det_boxes)\n    sample_res.img_id = file_id\n    return sample_res\n\n\ndef parse_single_file(\n    content,\n    has_crlf=True,\n    with_transcription=False,\n    with_confidence=False,\n    img_width=0,\n    img_height=0,\n    sort_by_confidences=True,\n    box_type=\"QUAD\",\n):\n    \"\"\"Returns all points, confindences and transcriptions of a file in lists.\n\n    valid line formats:\n        xmin,ymin,xmax,ymax,[confidence],[transcription]\n        x1,y1,x2,y2,x3,y3,x4,y4,[confidence],[transcription]\n    \"\"\"\n    result_boxes = []\n    lines = content.split(\"\\r\\n\" if has_crlf else \"\\n\")\n    for line in lines:\n        line = line.replace(\"\\r\", \"\").replace(\"\\n\", \"\")\n        if line != \"\":\n            result_box = parse_values_from_single_line(\n                line,\n                with_transcription,\n                with_confidence,\n                img_width,\n                img_height,\n                box_type=box_type,\n            )\n            result_boxes.append(result_box)\n\n    if with_confidence and len(result_boxes) and sort_by_confidences:\n        result_boxes.sort(key=lambda x: x.confidence, reverse=True)\n\n    return result_boxes\n\n\ndef parse_values_from_single_line(\n    line,\n    with_transcription=False,\n    with_confidence=False,\n    img_width=0,\n    img_height=0,\n    box_type=\"QUAD\",\n) -> Box:\n    \"\"\"\n    Validate the format of the line.\n    If the line is not valid an ValueError will be raised.\n    If maxWidth and maxHeight are specified, all points must be inside the image bounds.\n    Posible values are:\n    LTRB=True: xmin,ymin,xmax,ymax[,confidence][,transcription]\n    LTRB=False: x1,y1,x2,y2,x3,y3,x4,y4[,confidence][,transcription]\n    LTRB=\"POLY\": x1,y1,x2,y2,x3,y3,x4,y4[,confidence][,transcription]\n\n    box_type:\n        - LTRB: add description\n        - QUAD: add description\n        - POLY: add description\n\n    Returns values from a textline. Points , [Confidences], [Transcriptions]\n    \"\"\"\n    confidence = 0.0\n    transcription = \"\"\n\n    if box_type == \"LTRB\":\n        box_type = QUAD\n        num_points = 4\n        m = ltrb_regex_match(line, with_transcription, with_confidence)\n        xmin = int(m.group(1))\n        ymin = int(m.group(2))\n        xmax = int(m.group(3))\n        ymax = int(m.group(4))\n\n        validate_min_max_bounds(lower_val=xmin, upper_val=xmax)\n        validate_min_max_bounds(lower_val=ymin, upper_val=ymax)\n\n        points = [float(m.group(i)) for i in range(1, (num_points + 1))]\n        points = convert_ltrb2quad(points)\n\n        if img_width > 0 and img_height > 0:\n            validate_point_inside_bounds(xmin, ymin, img_width, img_height)\n            validate_point_inside_bounds(xmax, ymax, img_width, img_height)\n\n    elif box_type == \"QUAD\":\n        box_type = QUAD\n\n        num_points = 8\n        m = quad_regex_match(line, with_transcription, with_confidence)\n        points = [float(m.group(i)) for i in range(1, (num_points + 1))]\n\n        # validate_clockwise_points(points)\n        if img_width > 0 and img_height > 0:\n            validate_point_inside_bounds(points[0], points[1], img_width, img_height)\n            validate_point_inside_bounds(points[2], points[3], img_width, img_height)\n            validate_point_inside_bounds(points[4], points[5], img_width, img_height)\n            validate_point_inside_bounds(points[6], points[7], img_width, img_height)\n\n    elif box_type == \"POLY\":\n        # TODO: TotalText GT보고 정하기\n        # TODO: 이렇게 리턴하는 건 굉장히 위험\n        splitted_line = line.split(\",\")\n        tmp_transcription = list()\n\n        if with_transcription:\n            tmp_transcription.append(splitted_line.pop())\n            while not len(\"\".join(tmp_transcription)):\n                tmp_transcription.append(splitted_line.pop())\n\n        if with_confidence:\n            if len(splitted_line) % 2 != 0:\n                confidence = float(splitted_line.pop())\n                points = [float(x) for x in splitted_line]\n            else:\n                backward_idx = len(splitted_line) - 1\n                while backward_idx > 0:\n                    if splitted_line[backward_idx].isdigit() and len(splitted_line) % 2 != 0:\n                        break\n                    tmp_transcription.append(splitted_line.pop())\n                    backward_idx -= 1\n                confidence = float(splitted_line.pop())\n                points = [float(x) for x in splitted_line]\n        else:\n            if len(splitted_line) % 2 == 0:\n                points = [float(x) for x in splitted_line]\n            else:\n                backward_idx = len(splitted_line) - 1\n                while backward_idx > 0:\n                    if splitted_line[backward_idx].isdigit():\n                        break\n                    tmp_transcription.append(splitted_line.pop())\n                    backward_idx -= 1\n                points = [float(x) for x in splitted_line]\n\n        transcription = \",\".join(tmp_transcription)\n        return POLY(points, confidence=confidence, transcription=transcription)\n    else:\n        raise RuntimeError(f\"Something is wrong with configuration. Box Type: [{box_type}]\")\n\n    # QUAD or LTRB format\n    if with_confidence:\n        try:\n            confidence = float(m.group(num_points + 1))\n        except ValueError:\n            raise ValueError(\"Confidence value must be a float\")\n\n    if with_transcription:\n        pos_transcription = num_points + (2 if with_confidence else 1)\n        transcription = m.group(pos_transcription)\n        m2 = re.match(r\"^\\s*\\\"(.*)\\\"\\s*$\", transcription)\n\n        # Transcription with double quotes\n        # We extract the value and replace escaped characters\n        if m2 is not None:\n            transcription = m2.group(1).replace(\"\\\\\\\\\", \"\\\\\").replace('\\\\\"', '\"')\n\n    result_box = box_type(points, confidence=confidence, transcription=transcription)\n    return result_box\n\n\ndef parse_jylee_annot(quad, transcription, box_type):\n    assert box_type == \"QUAD\"\n    points = [\n        quad[\"x1\"],\n        quad[\"y1\"],\n        quad[\"x2\"],\n        quad[\"y2\"],\n        quad[\"x3\"],\n        quad[\"y3\"],\n        quad[\"x4\"],\n        quad[\"y4\"],\n    ]\n    result_box = QUAD(points, confidence=0.0, transcription=transcription)\n    return result_box\n\n\ndef parse_clova_ocr(quad, transcription, box_type):\n    assert box_type == \"QUAD\"\n    result_box = QUAD(quad, confidence=0.0, transcription=transcription)\n    return result_box\n\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "cleval/torchmetric.py",
    "content": "\"\"\"\nTODO: Support scalewise eval\nTODO: Support orientation accuracy\n\"\"\"\n\nimport cv2\nimport numpy as np\nimport torch\nfrom torchmetrics import Metric\n\nfrom cleval.box_types import QUAD\nfrom cleval.data import SampleResult\nfrom cleval.eval_functions import evaluation\n\n\nclass Options:\n    def __init__(\n        self,\n        case_sensitive,\n        recall_gran_penalty,\n        precision_gran_penalty,\n        vertical_aspect_ratio_thresh,\n        ap_constraint,\n    ):\n        self.DUMP_SAMPLE_RESULT = False\n        self.E2E = (True,)  # change in runtime. See update function.\n        self.ORIENTATION = False\n        self.CASE_SENSITIVE = case_sensitive\n        self.RECALL_GRANULARITY_PENALTY_WEIGHT = recall_gran_penalty\n        self.PRECISION_GRANULARITY_PENALTY_WEIGHT = precision_gran_penalty\n        self.VERTICAL_ASPECT_RATIO_THRESH = vertical_aspect_ratio_thresh\n        self.AREA_PRECISION_CONSTRAINT = ap_constraint\n\n\nclass CLEvalMetric(Metric):\n    full_state_update: bool = False\n\n    def __init__(\n        self,\n        dist_sync_on_step=False,\n        case_sensitive=True,\n        recall_gran_penalty=1.0,\n        precision_gran_penalty=1.0,\n        vertical_aspect_ratio_thresh=0.5,\n        ap_constraint=0.3,\n        scale_wise=False,\n        scale_bins=(0.0, 0.005, 0.01, 0.015, 0.02, 0.025, 0.1, 0.5, 1.0),\n        scale_range=(0.0, 1.0),\n    ):\n        super().__init__(dist_sync_on_step=dist_sync_on_step)\n        self.options = Options(\n            case_sensitive,\n            recall_gran_penalty,\n            precision_gran_penalty,\n            vertical_aspect_ratio_thresh,\n            ap_constraint,\n        )\n        self.scale_range = scale_range\n\n        self.scalewise_metric = {}\n        if scale_wise:\n            bin_ranges = [scale_bins[i : i + 2] for i in range(len(scale_bins) - 1)]\n            for bin_range in bin_ranges:\n                self.scalewise_metric[bin_range] = CLEvalMetric(\n                    dist_sync_on_step=dist_sync_on_step,\n                    case_sensitive=case_sensitive,\n                    recall_gran_penalty=recall_gran_penalty,\n                    precision_gran_penalty=precision_gran_penalty,\n                    vertical_aspect_ratio_thresh=vertical_aspect_ratio_thresh,\n                    ap_constraint=ap_constraint,\n                    scale_wise=False,\n                    scale_range=bin_range,\n                )\n\n        # Detection\n        self.add_state(\"det_num_char_gt\", torch.tensor(0, dtype=torch.int32), dist_reduce_fx=\"sum\")\n        self.add_state(\"det_num_char_det\", torch.tensor(0, dtype=torch.int32), dist_reduce_fx=\"sum\")\n        self.add_state(\n            \"det_gran_score_recall\",\n            torch.tensor(0, dtype=torch.float32),\n            dist_reduce_fx=\"sum\",\n        )\n        self.add_state(\n            \"det_num_char_tp_recall\",\n            torch.tensor(0, dtype=torch.int32),\n            dist_reduce_fx=\"sum\",\n        )\n        self.add_state(\n            \"det_gran_score_precision\",\n            torch.tensor(0, dtype=torch.float32),\n            dist_reduce_fx=\"sum\",\n        )\n        self.add_state(\n            \"det_num_char_tp_precision\",\n            torch.tensor(0, dtype=torch.int32),\n            dist_reduce_fx=\"sum\",\n        )\n\n        self.add_state(\"det_num_char_fp\", torch.tensor(0, dtype=torch.int32), dist_reduce_fx=\"sum\")\n\n        # E2E\n        self.add_state(\"e2e_num_char_gt\", torch.tensor(0, dtype=torch.int32), dist_reduce_fx=\"sum\")\n        self.add_state(\"e2e_num_char_det\", torch.tensor(0, dtype=torch.int32), dist_reduce_fx=\"sum\")\n        self.add_state(\n            \"e2e_gran_score_recall\",\n            torch.tensor(0, dtype=torch.float32),\n            dist_reduce_fx=\"sum\",\n        )\n        self.add_state(\n            \"e2e_num_char_tp_recall\",\n            torch.tensor(0, dtype=torch.int32),\n            dist_reduce_fx=\"sum\",\n        )\n        self.add_state(\n            \"e2e_gran_score_precision\",\n            torch.tensor(0, dtype=torch.float32),\n            dist_reduce_fx=\"sum\",\n        )\n        self.add_state(\n            \"e2e_num_char_tp_precision\",\n            torch.tensor(0, dtype=torch.int32),\n            dist_reduce_fx=\"sum\",\n        )\n\n        self.add_state(\"e2e_num_char_fp\", torch.tensor(0, dtype=torch.int32), dist_reduce_fx=\"sum\")\n\n        # split-merge cases\n        self.add_state(\"num_splitted\", torch.tensor(0, dtype=torch.int32), dist_reduce_fx=\"sum\")\n        self.add_state(\"num_merged\", torch.tensor(0, dtype=torch.int32), dist_reduce_fx=\"sum\")\n        self.add_state(\n            \"num_char_overlapped\",\n            torch.tensor(0, dtype=torch.int32),\n            dist_reduce_fx=\"sum\",\n        )\n\n    def to(self, *args, **kwargs):\n        super().to(*args, **kwargs)\n        for key, metric in self.scalewise_metric.items():\n            self.scalewise_metric[key] = metric.to(*args, **kwargs)\n        return self\n\n    def update(\n        self,\n        det_quads,\n        gt_quads,\n        det_letters=None,\n        gt_letters=None,\n        gt_is_dcs=None,\n        img_longer_length=None,\n    ):\n        \"\"\"\n        Args:\n            det_quads (NDArray[float32]): (N, 8) detected quads\n            gt_quads (NDArray[float32]): (N, 8) target quads\n            det_letters (List[str]): detected letters\n            gt_letters (List[str]): target letters\n            gt_is_dcs (List[bool]): is dc gt quad?\n            img_longer_length (int): longer length of images\n        \"\"\"\n        gt_inps = self.__make_eval_input(gt_quads, gt_letters, gt_is_dcs, img_longer_length)\n        det_inps = self.__make_eval_input(det_quads, det_letters)\n        self.options.E2E = False if gt_letters is None and det_letters is None else True\n        sample_res = evaluation(self.options, gt_inps, det_inps, scale_range=self.scale_range)\n        self.__accumulate(sample_res)\n\n        for metric in self.scalewise_metric.values():\n            if img_longer_length is None:\n                raise ValueError(\"[img_longer_length] argument should be \" \"given for scalewise evaluation.\")\n            metric(\n                det_quads,\n                gt_quads,\n                det_letters,\n                gt_letters,\n                gt_is_dcs,\n                img_longer_length,\n            )\n\n    def __make_eval_input(self, quads, letters, is_dcs=None, img_longer_length=None):\n        eval_inps = []\n        for i in range(len(quads)):\n            box_scale = None\n            if img_longer_length is not None:\n                box_scale = self.__check_box_scale(quads[i], img_longer_length)\n\n            eval_inp = QUAD(\n                quads[i],\n                confidence=0.0,\n                transcription=None if letters is None else letters[i],\n                is_dc=None if is_dcs is None else is_dcs[i],\n                scale=box_scale,\n            )\n            eval_inps.append(eval_inp)\n        return eval_inps\n\n    @staticmethod\n    def __check_box_scale(quad, img_longer_length):\n        \"\"\"The method calculates box scale\n        Box scale is defined using the equation: char-height / image-longer size\n        The size of a box is defined w.r.t image size, allowing us to judge how sensitive\n        the model is to the box scale.\n        \"\"\"\n        rect = cv2.minAreaRect(quad.reshape(4, 2))\n        quad = cv2.boxPoints(rect)\n        quad = np.around(quad)\n        box_w = np.linalg.norm(quad[1] - quad[0]) + np.linalg.norm(quad[3] - quad[2])\n        box_h = np.linalg.norm(quad[2] - quad[1]) + np.linalg.norm(quad[0] - quad[3])\n        box_scale = min(box_w, box_h) / 2 / img_longer_length\n        return box_scale\n\n    def __accumulate(self, sample_res: SampleResult):\n        self.num_splitted += sample_res.stats.num_splitted\n        self.num_merged += sample_res.stats.num_merged\n        self.num_char_overlapped += sample_res.stats.num_char_overlapped\n\n        self.det_num_char_gt += sample_res.stats.det.num_char_gt\n        self.det_num_char_det += sample_res.stats.det.num_char_det\n        self.det_gran_score_recall += sample_res.stats.det.gran_score_recall\n        self.det_num_char_tp_recall += sample_res.stats.det.num_char_tp_recall\n        self.det_gran_score_precision += sample_res.stats.det.gran_score_precision\n        self.det_num_char_tp_precision += sample_res.stats.det.num_char_tp_precision\n        self.det_num_char_fp += sample_res.stats.det.num_char_fp\n\n        self.e2e_num_char_gt += sample_res.stats.e2e.num_char_gt\n        self.e2e_num_char_det += sample_res.stats.e2e.num_char_det\n        self.e2e_gran_score_recall += sample_res.stats.e2e.gran_score_recall\n        self.e2e_num_char_tp_recall += sample_res.stats.e2e.num_char_tp_recall\n        self.e2e_gran_score_precision += sample_res.stats.e2e.gran_score_precision\n        self.e2e_num_char_tp_precision += sample_res.stats.e2e.num_char_tp_precision\n        self.e2e_num_char_fp += sample_res.stats.e2e.num_char_fp\n\n    def compute(self):\n        det_r, det_p, det_h = self.__calculate_rph(\n            self.det_num_char_gt,\n            self.det_num_char_det,\n            self.det_gran_score_recall,\n            self.det_num_char_tp_recall,\n            self.det_gran_score_precision,\n            self.det_num_char_tp_precision,\n        )\n        e2e_r, e2e_p, e2e_h = self.__calculate_rph(\n            self.e2e_num_char_gt,\n            self.e2e_num_char_det,\n            self.e2e_gran_score_recall,\n            self.e2e_num_char_tp_recall,\n            self.e2e_gran_score_precision,\n            self.e2e_num_char_tp_precision,\n        )\n        return_dict = {\n            \"det_r\": det_r,\n            \"det_p\": det_p,\n            \"det_h\": det_h,\n            \"e2e_r\": e2e_r,\n            \"e2e_p\": e2e_p,\n            \"e2e_h\": e2e_h,\n            \"num_splitted\": self.num_splitted,\n            \"num_merged\": self.num_merged,\n            \"num_char_overlapped\": self.num_char_overlapped,\n            \"scale_wise\": {},\n        }\n\n        for scale_bin, metric in self.scalewise_metric.items():\n            return_dict[\"scale_wise\"][scale_bin] = metric.compute()\n\n        return return_dict\n\n    def reset(self):\n        super().reset()\n        for metric in self.scalewise_metric.values():\n            metric.reset()\n\n    def __calculate_rph(\n        self,\n        num_char_gt,\n        num_char_det,\n        gran_score_recall,\n        num_char_tp_recall,\n        gran_score_precision,\n        num_char_tp_precision,\n    ):\n        total_gt = num_char_gt\n        total_det = num_char_det\n        gran_gt = gran_score_recall\n        tp_gt = num_char_tp_recall\n        gran_det = gran_score_precision\n        tp_det = num_char_tp_precision\n\n        # Sample Score : Character correct length - Granularity Penalty\n        recall = 0.0 if total_gt == 0 else max(0.0, tp_gt - gran_gt) / total_gt\n        precision = 0.0 if total_det == 0 else max(0.0, tp_det - gran_det) / total_det\n        hmean = self.harmonic_mean(recall, precision)\n        return recall, precision, hmean\n\n    def harmonic_mean(self, score1, score2):\n        \"\"\"get harmonic mean value\"\"\"\n        if score1 + score2 == 0:\n            return torch.tensor(0, dtype=torch.float32, device=self.device)\n        else:\n            return (2 * score1 * score2) / (score1 + score2)\n"
  },
  {
    "path": "cleval/utils.py",
    "content": "import codecs\nimport json\nimport re\nimport subprocess\nimport zipfile\n\nfrom numba import njit\n\n\ndef load_zip_file(file):\n    \"\"\"\n    Returns an array with the contents (filtered by fileNameRegExp) of a ZIP file.\n    all_entries validates that all entries in the ZIP file pass the fileNameRegExp\n    \"\"\"\n    archive = zipfile.ZipFile(file, mode=\"r\", allowZip64=True)\n\n    pairs = dict()\n    for name in archive.namelist():\n        key_name = (\n            name.replace(\"gt_\", \"\").replace(\"res_\", \"\").replace(\".txt\", \"\").replace(\".json\", \"\").replace(\".jpg\", \"\")\n        )\n        pairs[key_name] = archive.read(name)\n    return pairs\n\n\ndef decode_utf8(raw):\n    \"\"\"\n    Returns a Unicode object\n    \"\"\"\n    raw = codecs.decode(raw, \"utf-8\", \"replace\")\n\n    # extracts BOM if exists\n    raw = raw.encode(\"utf8\")\n    if raw.startswith(codecs.BOM_UTF8):\n        raw = raw.replace(codecs.BOM_UTF8, b\"\", 1)\n    return raw.decode(\"utf-8\")\n\n\ndef dump_json(json_file_path, json_data):\n    with open(json_file_path, \"w\", encoding=\"utf-8\") as f:\n        json.dump(json_data, f)\n\n\ndef read_json(json_file_path):\n    with open(json_file_path, \"r\", encoding=\"utf-8\") as f:\n        json_data = json.load(f)\n    return json_data\n\n\ndef convert_ltrb2quad(points):\n    \"\"\"Convert point format from LTRB to QUAD\"\"\"\n    new_points = [\n        points[0],\n        points[1],\n        points[2],\n        points[1],\n        points[2],\n        points[3],\n        points[0],\n        points[3],\n    ]\n    return new_points\n\n\ndef ltrb_regex_match(line, with_transcription, with_confidence):\n    if with_transcription and with_confidence:\n        m = re.match(\n            r\"^\\s*(-?[0-9]+)\\s*\"\n            r\",\\s*(-?[0-9]+)\\s*\"\n            r\",\\s*([0-9]+)\\s*\"\n            r\",\\s*([0-9]+)\\s*\"\n            r\",\\s*([0-1].?[0-9]*)\\s*,(.*)$\",\n            line,\n        )\n        if m is None:\n            raise ValueError(\"Format incorrect. \" \"Should be: xmin,ymin,xmax,ymax,confidence,transcription\")\n    elif with_confidence:\n        m = re.match(\n            r\"^\\s*(-?[0-9]+)\\s*,\" r\"\\s*(-?[0-9]+)\\s*,\" r\"\\s*([0-9]+)\\s*,\" r\"\\s*([0-9]+)\\s*,\" r\"\\s*([0-1].?[0-9]*)\\s*$\",\n            line,\n        )\n        if m is None:\n            raise ValueError(\"Format incorrect. Should be: xmin,ymin,xmax,ymax,confidence\")\n    elif with_transcription:\n        m = re.match(\n            r\"^\\s*(-?[0-9]+)\\s*,\" r\"\\s*(-?[0-9]+)\\s*,\" r\"\\s*([0-9]+)\\s*,\" r\"\\s*([0-9]+)\\s*,(.*)$\",\n            line,\n        )\n        if m is None:\n            raise ValueError(\"Format incorrect. Should be: xmin,ymin,xmax,ymax,transcription\")\n    else:\n        m = re.match(\n            r\"^\\s*(-?[0-9]+)\\s*,\" r\"\\s*(-?[0-9]+)\\s*,\" r\"\\s*([0-9]+)\\s*,\" r\"\\s*([0-9]+)\\s*,?\\s*$\",\n            line,\n        )\n        if m is None:\n            raise ValueError(\"Format incorrect. Should be: xmin,ymin,xmax,ymax\")\n    return m\n\n\ndef quad_regex_match(line, with_transcription, with_confidence):\n    if with_transcription and with_confidence:\n        m = re.match(\n            r\"^\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*([0-1].?[0-9]*)\\s*,(.*)$\",\n            line,\n        )\n        if m is None:\n            raise ValueError(\"Format incorrect. \" \"Should be: x1,y1,x2,y2,x3,y3,x4,y4,confidence,transcription\")\n    elif with_confidence:\n        m = re.match(\n            r\"^\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*([0-1].?[0-9]*)\\s*$\",\n            line,\n        )\n        if m is None:\n            raise ValueError(\"Format incorrect. Should be: x1,y1,x2,y2,x3,y3,x4,y4,confidence\")\n    elif with_transcription:\n        m = re.match(\n            r\"^\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,(.*)$\",\n            line,\n        )\n        if m is None:\n            raise ValueError(\"Format incorrect. Should be: x1,y1,x2,y2,x3,y3,x4,y4,transcription\")\n    else:\n        if line[-1] == \",\":\n            line = line[:-1]\n        m = re.match(\n            r\"^\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*,\"\n            r\"\\s*(-?[0-9]+)\\s*$\",\n            line,\n        )\n        if m is None:\n            raise ValueError(\"Format incorrect. Should be: x1,y1,x2,y2,x3,y3,x4,y4\")\n    return m\n\n\n@njit\ndef lcs(s1, s2):\n    \"\"\"Longeset Common Sequence between s1 & s2\"\"\"\n    # https://stackoverflow.com/questions/48651891/longest-common-subsequence-in-python\n    if len(s1) == 0 or len(s2) == 0:\n        return 0, \"\"\n    matrix = [[\"\" for _ in range(len(s2))] for _ in range(len(s1))]\n    for i in range(len(s1)):\n        for j in range(len(s2)):\n            if s1[i] == s2[j]:\n                if i == 0 or j == 0:\n                    matrix[i][j] = s1[i]\n                else:\n                    matrix[i][j] = matrix[i - 1][j - 1] + s1[i]\n            else:\n                if len(matrix[i - 1][j]) > len(matrix[i][j - 1]):\n                    matrix[i][j] = matrix[i - 1][j]\n                else:\n                    matrix[i][j] = matrix[i][j - 1]\n    cs = matrix[-1][-1]\n    return len(cs), cs\n\n\n@njit\ndef harmonic_mean(score1, score2):\n    \"\"\"get harmonic mean value\"\"\"\n    if score1 + score2 == 0:\n        return 0\n    else:\n        return (2 * score1 * score2) / (score1 + score2)\n\n\ndef cpu_count():\n    \"\"\"Get number of cpu\n    os.cpu_count() has a problem with docker container.\n    For example, we have 72 cpus. os.cpu_count() always return 72\n    even if we allocate only 4 cpus for container.\n    \"\"\"\n    return int(subprocess.check_output(\"nproc\").decode().strip())\n"
  },
  {
    "path": "cleval/validation.py",
    "content": "from cleval.utils import decode_utf8, load_zip_file\n\n\ndef validate_data(gt_file, submit_file, has_crlf):\n    gt = load_zip_file(gt_file)\n    subm = load_zip_file(submit_file)\n\n    # Validate format of GroundTruth\n    for k in gt:\n        validate_lines_in_file(k, gt[k], has_crlf)\n\n    # Validate format of results\n    for k in subm:\n        if k not in gt:\n            raise ValueError(\"The sample %s not present in GT\" % k)\n        validate_lines_in_file(k, subm[k], has_crlf)\n\n\ndef validate_lines_in_file(file_name, file_contents, has_crlf=True):\n    \"\"\"This function validates that all lines of the file.\n    Execute line validation function for each line.\n    \"\"\"\n    utf8file = decode_utf8(file_contents)\n    if utf8file is None:\n        raise ValueError(\"The file %s is not UTF-8\" % file_name)\n\n    lines = utf8file.split(\"\\r\\n\" if has_crlf else \"\\n\")\n    for line in lines:\n        _ = line.replace(\"\\r\", \"\").replace(\"\\n\", \"\")\n\n\ndef validate_point_inside_bounds(x, y, img_width, img_height):\n    if x < 0 or x > img_width:\n        raise ValueError(\"X value (%s) not valid. Image dimensions: (%s,%s)\" % (x, img_width, img_height))\n    if y < 0 or y > img_height:\n        raise ValueError(\"Y value (%s)  not valid. Image dimensions: (%s,%s)\" % (y, img_width, img_height))\n\n\ndef validate_min_max_bounds(lower_val, upper_val):\n    if lower_val > upper_val:\n        raise ValueError(f\"Value {lower_val} should be smaller than value {upper_val}.\")\n\n\ndef validate_clockwise_points(points):\n    \"\"\"\n    Validates that the points are in clockwise order.\n    \"\"\"\n\n    if len(points) != 8:\n        raise ValueError(\"Points list not valid.\" + str(len(points)))\n\n    point = [\n        [int(points[0]), int(points[1])],\n        [int(points[2]), int(points[3])],\n        [int(points[4]), int(points[5])],\n        [int(points[6]), int(points[7])],\n    ]\n    edge = [\n        (point[1][0] - point[0][0]) * (point[1][1] + point[0][1]),\n        (point[2][0] - point[1][0]) * (point[2][1] + point[1][1]),\n        (point[3][0] - point[2][0]) * (point[3][1] + point[2][1]),\n        (point[0][0] - point[3][0]) * (point[0][1] + point[3][1]),\n    ]\n\n    summatory = edge[0] + edge[1] + edge[2] + edge[3]\n    if summatory > 0:\n        raise ValueError(\n            \"Points are not clockwise. \" \"The coordinates of bounding quads have to be given in clockwise order.\"\n        )\n"
  },
  {
    "path": "pyproject.toml",
    "content": "[tool.isort]\nprofile = \"black\"\n\n[tool.black]\nline-length = 120\ntarget-version = ['py38']\ninclude = '\\.pyi?$'\n\n[tool.pytest.ini_options]\naddopts = \"-s\"\n"
  },
  {
    "path": "setup.py",
    "content": "import setuptools\n\nwith open(\"README.md\", \"r\") as fh:\n    long_description = fh.read()\n\nsetuptools.setup(\n    name=\"cleval\",\n    version=\"0.1.1\",\n    author=\"dong.hyun\",\n    author_email=\"dong.hyun@navercorp.com\",\n    description=\"cleval\",\n    long_description=long_description,\n    long_description_content_type=\"text/markdown\",\n    url=\"https://oss.navercorp.com/CLOVA-AI-OCR/cleval\",\n    packages=setuptools.find_packages(),\n    install_requires=[\n        \"bottle\",\n        \"requests\",\n        \"Pillow\",\n        \"Polygon3\",\n        \"Shapely\",\n        \"tqdm\",\n        \"pprofile\",\n        \"numba>=0.58.0\",\n        \"six\",\n        \"torchmetrics>=1.2.0\",\n        \"numpy\",\n    ],\n    classifiers=[\n        \"Programming Language :: Python :: 3\",\n        \"License :: OSI Approved :: MIT License\",\n        \"Operating System :: OS Independent\",\n    ],\n    entry_points={\n        \"console_scripts\": [\n            \"cleval = cleval.main:main\",\n        ],\n    },\n    python_requires=\">=3.7\",\n)\n"
  },
  {
    "path": "tests/__init__.py",
    "content": ""
  },
  {
    "path": "tests/test_scores.py",
    "content": "import sys\n\nimport pytest\n\n\ndef test_output_score():\n    sys.argv[1:] = [\n        \"-g=resources/test_data/gt/gt_eval_doc_v1_kr.zip\",\n        \"-s=resources/test_data/pred/res_eval_doc_v1_kr.zip\",\n        \"--E2E\",\n        \"--DUMP_SAMPLE_RESULT\",\n        \"--DEBUG\",\n    ]\n    from cleval.arg_parser import get_params\n    from cleval.main import cleval\n\n    args = get_params()\n    result = cleval(args)\n    det_hmean = 0.977989360950786\n    e2e_hmean = 0.9165773847119407\n    pred_det_hmean = result[\"all\"][\"det\"][\"hmean\"]\n    pred_e2e_hmean = result[\"all\"][\"e2e\"][\"hmean\"]\n    assert pred_det_hmean == pytest.approx(det_hmean), pred_det_hmean\n    assert pred_e2e_hmean == pytest.approx(e2e_hmean), pred_e2e_hmean\n\n\ndef test_output_score_torchmetric():\n    sys.argv[1:] = [\n        \"-g=resources/test_data/gt/gt_eval_doc_v1_kr.zip\",\n        \"-s=resources/test_data/pred/res_eval_doc_v1_kr.zip\",\n        \"--E2E\",\n        \"--DUMP_SAMPLE_RESULT\",\n        \"--DEBUG\",\n    ]\n\n    import numpy as np\n\n    from cleval import CLEvalMetric\n    from cleval.arg_parser import get_params\n    from cleval.main import get_file_paths, parse_single_file\n\n    args = get_params()\n    gt_zipfile = args.GT_PATHS[0]\n    submit_zipfile = args.SUBMIT_PATHS[0]\n    gt_files, det_files, file_indices = get_file_paths(gt_zipfile, submit_zipfile)\n    metric = CLEvalMetric()\n\n    for gt_file, det_file, file_idx in zip(gt_files, det_files, file_indices):\n        gt_boxes = parse_single_file(gt_file, args.CRLF, True, False, box_type=args.BOX_TYPE)\n        det_boxes = parse_single_file(\n            det_file,\n            args.CRLF,\n            args.TRANSCRIPTION,\n            args.CONFIDENCES,\n            box_type=args.BOX_TYPE,\n        )\n        gt_quads = np.array([gt_box.points for gt_box in gt_boxes])\n        gt_letters = [gt_box.transcription for gt_box in gt_boxes]\n        gt_is_dcs = [gt_box.is_dc for gt_box in gt_boxes]\n        det_quads = np.array([det_box.points for det_box in det_boxes])\n        det_letters = [det_box.transcription for det_box in det_boxes]\n        _ = metric(det_quads, gt_quads, det_letters, gt_letters, gt_is_dcs)\n\n    metric_out = metric.compute()\n    metric.reset()\n\n    det_hmean = 0.977989360950786\n    e2e_hmean = 0.9165773847119407\n    assert metric_out[\"det_h\"].item() == pytest.approx(det_hmean), metric_out[\"det_h\"]\n    assert metric_out[\"e2e_h\"].item() == pytest.approx(e2e_hmean), metric_out[\"e2e_h\"]\n"
  }
]